From c8f80bc752b0a322297df3c4ebe41f8f85a9bc88 Mon Sep 17 00:00:00 2001 From: abdulsaheel Date: Wed, 28 Oct 2026 22:04:19 +0530 Subject: [PATCH 01/33] finalized day lookup scanned every finalized bundle for the imported marker I keep an imported flag on day_result now, set by trigger from the same LIKE test and backfilled in pages off the open path, and read the finalized set off a covering index. (cherry picked from commit 1a1fb4f31576348631cc982886f09334ac9fc893) --- lib/data/db.dart | 97 +++++++++++++++++++- test/finalized_day_ids_index_test.dart | 119 +++++++++++++++++++++++++ 2 files changed, 214 insertions(+), 2 deletions(-) create mode 100644 test/finalized_day_ids_index_test.dart diff --git a/lib/data/db.dart b/lib/data/db.dart index 6dfe313f2..7012beb44 100644 --- a/lib/data/db.dart +++ b/lib/data/db.dart @@ -1252,6 +1252,7 @@ class LocalDb { await _createRawArchive(db); await _createDerived(db); await _createDayResult(db); + await _ensureDayResultImportedColumn(db); await _createUserTables(db); await _createSyncState(db); await _createSyncCursor(db); @@ -9311,18 +9312,110 @@ class LocalDb { /// no rows for yet, and band rows for it arriving later must still derive /// and replace the snapshot ([isMeasuredDayRow]). Every caller intersects /// this with days that have raw, so a raw-less import is still left alone. + /// + /// Read off [_dayResultFinalizedIndex] once [_ensureDayResultImportedFlag] + /// has filled `day_result.imported`. The LIKE it replaces read every + /// finalized day's whole bundle — and `finalized` itself sits after + /// `payload_json` in the row, so even the plain column test walked each + /// bundle's overflow pages — several times per derive pass. static Future> finalizedDayIds(int algoVersion) async { final db = await instance; + final indexed = await _ensureDayResultImportedFlag(db); final rows = await db.query( 'day_result', columns: ['day_id'], - where: 'algo_version = ? AND finalized = 1 ' - "AND payload_json NOT LIKE '%\"imported\":true%'", + where: indexed + ? 'algo_version = ? AND finalized = 1 AND imported = 0' + : 'algo_version = ? AND finalized = 1 ' + 'AND payload_json NOT LIKE $_importedPayloadLike', whereArgs: [algoVersion], ); return {for (final r in rows) r['day_id'] as String}; } + /// The one test for "this bundle is an imported snapshot". The `imported` + /// column is computed from it by trigger, so the column and the old scan + /// cannot disagree about a row. + static const String _importedPayloadLike = "'%\"imported\":true%'"; + + /// Covering index for [finalizedDayIds]. Created only once every existing + /// row's `imported` is filled, so its presence is also the "backfill done" + /// marker: a database restored from a file without it backfills again. + static const String _dayResultFinalizedIndex = 'idx_day_result_finalized'; + + /// `day_result.imported` and the triggers that keep it equal to + /// [_importedPayloadLike] on every insert and payload update — whichever + /// path writes the row (derive, import, restore merge, raw SQL). Cheap and + /// idempotent: ADD COLUMN with a constant default rewrites nothing, so this + /// is safe on the every-open repair path. The backfill of existing rows is + /// NOT here; see [_ensureDayResultImportedFlag]. + static Future _ensureDayResultImportedColumn(Database db) async { + if ((await _columnsOf(db, 'day_result')).isEmpty) return; + await _addColumnIfMissing( + db, 'day_result', 'imported', 'INTEGER NOT NULL DEFAULT 0', + ); + await db.execute( + 'CREATE TRIGGER IF NOT EXISTS trg_day_result_imported_ins ' + 'AFTER INSERT ON day_result BEGIN ' + 'UPDATE day_result ' + 'SET imported = (NEW.payload_json LIKE $_importedPayloadLike) ' + 'WHERE rowid = NEW.rowid; END', + ); + await db.execute( + 'CREATE TRIGGER IF NOT EXISTS trg_day_result_imported_upd ' + 'AFTER UPDATE OF payload_json ON day_result BEGIN ' + 'UPDATE day_result ' + 'SET imported = (NEW.payload_json LIKE $_importedPayloadLike) ' + 'WHERE rowid = NEW.rowid; END', + ); + } + + /// True once `day_result.imported` is trustworthy for every row and + /// [_dayResultFinalizedIndex] exists. The first call on a database that + /// predates the column fills it in pages of 200 rows, each its own short + /// statement, then builds the index — off `openDatabase`, so a long history + /// never runs it under the launch watchdog. Rows written meanwhile are + /// already right (the triggers), and re-filling one recomputes the same + /// value, so a concurrent derive cannot race it. Any failure leaves the + /// index absent and the caller on the old scan; the next call resumes. + static Future _ensureDayResultImportedFlag(Database db) async { + Future ready() async => (await db.rawQuery( + "SELECT 1 FROM sqlite_master WHERE type = 'index' AND name = ?", + [_dayResultFinalizedIndex], + )) + .isNotEmpty; + if (await ready()) return true; + try { + if (!(await _columnsOf(db, 'day_result')).contains('imported')) { + await _ensureDayResultImportedColumn(db); + } + var after = 0; + while (true) { + final page = await db.rawQuery( + 'SELECT rowid AS r FROM day_result WHERE rowid > ? ' + 'ORDER BY rowid LIMIT 200', + [after], + ); + if (page.isEmpty) break; + final hi = (page.last['r'] as num).toInt(); + await db.rawUpdate( + 'UPDATE day_result ' + 'SET imported = (payload_json LIKE $_importedPayloadLike) ' + 'WHERE rowid > ? AND rowid <= ?', + [after, hi], + ); + after = hi; + } + await db.execute( + 'CREATE INDEX IF NOT EXISTS $_dayResultFinalizedIndex ' + 'ON day_result(algo_version, finalized, imported, day_id)', + ); + return true; + } catch (_) { + return false; + } + } + /// Normalize a day_result row to also carry a `date` key (== day_id) so legacy /// readers that keyed on `date` keep working. static Map _withDate(Map row) { diff --git a/test/finalized_day_ids_index_test.dart b/test/finalized_day_ids_index_test.dart new file mode 100644 index 000000000..230a96645 --- /dev/null +++ b/test/finalized_day_ids_index_test.dart @@ -0,0 +1,119 @@ +// `finalizedDayIds` reads an indexed `day_result.imported` flag instead of +// LIKE-scanning every finalized bundle. The answer must be the same set the +// scan gave, for rows written before the column existed (backfill) and after +// (triggers), on every write path. + +import 'package:flutter_test/flutter_test.dart'; +import 'package:path/path.dart' as p; +import 'package:sqflite_common_ffi/sqflite_ffi.dart'; +import 'package:openstrap_edge/data/db.dart'; + +Future _useFreshDb(String name) async { + await LocalDb.close(); + LocalDb.dbName = name; + final dir = await databaseFactory.getDatabasesPath(); + await databaseFactory.deleteDatabase(p.join(dir, name)); +} + +const _algo = 7; + +Future> _likeScan(Database db) async => { + for (final r in await db.query( + 'day_result', + columns: ['day_id'], + where: + 'algo_version = ? AND finalized = 1 ' + "AND payload_json NOT LIKE '%\"imported\":true%'", + whereArgs: [_algo], + )) + r['day_id'] as String, +}; + +Future _indexed(Database db) async => (await db.rawQuery( + "SELECT 1 FROM sqlite_master WHERE type = 'index' " + "AND name = 'idx_day_result_finalized'", +)).isNotEmpty; + +Future _put(String day, String payload, {bool finalized = true}) => + LocalDb.putDayResult( + dayId: day, + algoVersion: _algo, + payloadJson: payload, + windowJson: '{}', + finalized: finalized, + ); + +void main() { + setUpAll(() { + sqfliteFfiInit(); + databaseFactory = databaseFactoryFfi; + }); + + tearDown(() async { + await LocalDb.close(); + }); + + test('same set as the LIKE scan, through the triggers', () async { + await _useFreshDb('finalized_ids_triggers.db'); + await _put('2026-01-01', '{"scalars":{}}'); + await _put('2026-01-02', '{"imported":true,"source":"whoop_export"}'); + await _put('2026-01-03', '{"scalars":{}}', finalized: false); + await _put('2026-01-04', '{"imported":false}'); + final db = await LocalDb.instance; + expect(await LocalDb.finalizedDayIds(_algo), {'2026-01-01', '2026-01-04'}); + expect(await _indexed(db), isTrue); + + // A measured re-derive replacing the import, and an import replacing a + // measured row, both flip the flag. + await _put('2026-01-02', '{"scalars":{"rhr":50}}'); + await _put('2026-01-04', '{"imported":true}'); + // A raw payload rewrite (the shape `_scrubImportedSkinTempZ` uses). + await db.rawUpdate( + "UPDATE day_result SET payload_json = '{\"imported\":true}' " + "WHERE day_id = '2026-01-01'", + ); + final ids = await LocalDb.finalizedDayIds(_algo); + expect(ids, {'2026-01-02'}); + expect(ids, await _likeScan(db)); + }); + + test('backfills rows written before the column existed', () async { + await _useFreshDb('finalized_ids_backfill.db'); + final db = await LocalDb.instance; + // Emulate a pre-column database: no triggers, no index, stale flags. + await db.execute('DROP TRIGGER IF EXISTS trg_day_result_imported_ins'); + await db.execute('DROP TRIGGER IF EXISTS trg_day_result_imported_upd'); + await db.execute('DROP INDEX IF EXISTS idx_day_result_finalized'); + for (var i = 0; i < 450; i++) { + final day = + '2025-${(1 + i ~/ 28).toString().padLeft(2, '0')}-' + '${(1 + i % 28).toString().padLeft(2, '0')}'; + await db.insert('day_result', { + 'day_id': day, + 'algo_version': _algo, + 'payload_json': i % 3 == 0 ? '{"imported":true}' : '{"x":$i}', + 'window_json': '{}', + 'computed_at': 0, + 'finalized': i % 5 == 0 ? 0 : 1, + // Deliberately wrong, so only a real backfill passes. + 'imported': i % 3 == 0 ? 0 : 1, + }); + } + final expected = await _likeScan(db); + expect(expected, isNotEmpty); + // Reopen: the repair pass restores the triggers, the first read backfills. + await LocalDb.close(); + expect(await LocalDb.finalizedDayIds(_algo), expected); + final reopened = await LocalDb.instance; + expect(await _indexed(reopened), isTrue); + final plan = await reopened.rawQuery( + 'EXPLAIN QUERY PLAN SELECT day_id FROM day_result ' + 'WHERE algo_version = ? AND finalized = 1 AND imported = 0', + [_algo], + ); + expect( + plan.map((r) => '${r['detail']}').join(' '), + contains('COVERING INDEX idx_day_result_finalized'), + ); + }); +} From 807121e64025e103d3549c58fd77a7463f85263a Mon Sep 17 00:00:00 2001 From: abdulsaheel Date: Wed, 28 Oct 2026 22:22:07 +0530 Subject: [PATCH 02/33] the substrate archive kept a year of raw readings with no way to change it The default is 90 days now, with 30 / 90 / 1 year / forever in the data settings. Shortening it asks first and evicts the days it no longer covers. A day older than the window can't be re-scored after a future algorithm change. A day that never finalizes now holds raw for 7 days instead of 14. (cherry picked from commit 06277918a224159489f9e26a02fcb7113956aab2) --- lib/compute/derivation_engine.dart | 10 +++- lib/data/substrate_archive.dart | 30 +++++++++-- lib/l10n/app_de.arb | 33 ++++++++++++- lib/l10n/app_en.arb | 55 +++++++++++++++++++++ lib/l10n/app_es.arb | 33 ++++++++++++- lib/l10n/app_fr.arb | 33 ++++++++++++- lib/l10n/app_hi.arb | 33 ++++++++++++- lib/l10n/app_ru.arb | 33 ++++++++++++- lib/l10n/app_zh.arb | 33 ++++++++++++- lib/state/app_state.dart | 15 ++++++ lib/ui2/profile/data.dart | 66 +++++++++++++++++++++++++ test/raw_prune_gating_test.dart | 10 ++-- test/substrate_archive_window_test.dart | 38 ++++++++++++++ 13 files changed, 407 insertions(+), 15 deletions(-) create mode 100644 test/substrate_archive_window_test.dart diff --git a/lib/compute/derivation_engine.dart b/lib/compute/derivation_engine.dart index c90151123..30ac6a0d4 100644 --- a/lib/compute/derivation_engine.dart +++ b/lib/compute/derivation_engine.dart @@ -7212,7 +7212,15 @@ class DerivationEngine { /// filled in), and a transient skip never finalizes, so a day whose second /// half fails every single time never becomes finalized. One such day used /// to latch pruning off for the whole install, forever, at ~12 MB/day. - static const int _maxRawHoldDays = 14; + /// + /// Seven days behind the data edge, not fourteen. Nothing that can still + /// finalize needs longer: a day locks once the edge is [_finalizationSec] + /// past it, and a strain score it is owed comes from the days BEFORE it, + /// which [_quietLevelUnreachable] settles as soon as those have derived. A + /// day still unlocked a week on is one whose derive keeps failing, and the + /// second week of retries only cost another week of substrate (~12 MB a + /// day on a WHOOP 4, several times that on an MG). + static const int _maxRawHoldDays = 7; /// Cursor holding the highest `rec_ts` cutoff the raw prune has applied. static const String _prunedBeforeCursor = diff --git a/lib/data/substrate_archive.dart b/lib/data/substrate_archive.dart index e27a4125b..4a3ac0d2b 100644 --- a/lib/data/substrate_archive.dart +++ b/lib/data/substrate_archive.dart @@ -39,9 +39,33 @@ import 'substrate_archive_codec.dart'; class SubstrateArchivePolicy { const SubstrateArchivePolicy(this.keepDays); - /// One year. Chosen before a measurement on real exports; revisit with the - /// bytes/day the size probe reports (see substrate_archive_size_probe_test). - static const SubstrateArchivePolicy defaults = SubstrateArchivePolicy(365); + /// Ninety days. The archive costs ~1.6 MB per UTC day of WHOOP substrate, + /// so a year was ~600 MB on disk for history nothing re-derives from yet. + /// What the window gives up: a day older than it has no substrate left + /// anywhere, so a future algorithm change can never re-score it — it keeps + /// the score of the version that derived it. Users who want that history + /// kept pick a longer window ([choices]). + static const SubstrateArchivePolicy defaults = SubstrateArchivePolicy(90); + + /// The windows the data settings offer, shortest first; -1 is forever. + static const List choices = [30, 90, 365, -1]; + + /// The setting after this one in [choices], wrapping to the shortest. A + /// stored value that is not a choice (0 = off, or a hand-set count) moves + /// to the first longer choice. + SubstrateArchivePolicy get next { + int rank(int d) => d < 0 ? 1 << 30 : d; + for (final d in choices) { + if (rank(d) > rank(keepDays)) return SubstrateArchivePolicy(d); + } + return SubstrateArchivePolicy(choices.first); + } + + /// Whether switching to [other] deletes archived days this one keeps. + bool keepsLongerThan(SubstrateArchivePolicy other) { + int rank(int d) => d < 0 ? 1 << 30 : d; + return other.enabled && rank(other.keepDays) < rank(keepDays); + } /// 0 = off (and purged), -1 = forever, otherwise days behind the data edge. final int keepDays; diff --git a/lib/l10n/app_de.arb b/lib/l10n/app_de.arb index cee3a5a9e..17d983394 100644 --- a/lib/l10n/app_de.arb +++ b/lib/l10n/app_de.arb @@ -2904,5 +2904,36 @@ "notifBatteryForecastReserveBody": "Bei {rate} %/h ist der Akku gegen {time} leer, kurz nach deiner üblichen Aufstehzeit – beim Aufwachen bleiben etwa {left} %. Lade ihn jetzt, damit die Nacht aufgezeichnet wird.", "notifBatteryForecastAfterWakeBody": "Bei {rate} %/h ist der Akku gegen {time} leer, nach deiner üblichen Aufstehzeit.", "notifWindDownBody": "Deine Schlafenszeit ist gegen {time}. Fang an, runterzufahren.", - "notifWorkoutIdleBody": "Seit {minutes} Minuten wurde nichts über Ruheniveau aufgezeichnet. Wenn die Einheit vorbei ist, beende sie im Tab „Training“." + "notifWorkoutIdleBody": "Seit {minutes} Minuten wurde nichts über Ruheniveau aufgezeichnet. Wenn die Einheit vorbei ist, beende sie im Tab „Training“.", + "dataStorageGroup": "Speicher", + "dataKeepHistory": "Rohmesswerte behalten", + "dataKeepHistorySub": "Eine komprimierte Kopie der sekündlichen Messwerte jedes Tages, die bleibt, nachdem die App sie gelöscht hat. Ältere Tage behalten ihre Werte, können aber nach einer verbesserten Auswertung nicht neu berechnet werden", + "dataKeepDays": "{days} Tage", + "@dataKeepDays": { + "placeholders": { + "days": { + "type": "int" + } + } + }, + "dataKeepYear": "1 Jahr", + "dataKeepForever": "Für immer", + "dataKeepShorterTitle": "Rohmesswerte für {window} behalten?", + "@dataKeepShorterTitle": { + "placeholders": { + "window": { + "type": "String" + } + } + }, + "dataKeepShorterBody": "Messwerte, die älter als {window} sind, werden jetzt gelöscht. Diese Tage behalten ihre Werte, können nach künftigen Updates aber nicht neu berechnet werden.", + "@dataKeepShorterBody": { + "placeholders": { + "window": { + "type": "String" + } + } + }, + "dataKeepShorterConfirm": "Ältere Messwerte löschen", + "dataKeepShorterCancel": "Behalten" } diff --git a/lib/l10n/app_en.arb b/lib/l10n/app_en.arb index f2cdfad98..80d9d9fe5 100644 --- a/lib/l10n/app_en.arb +++ b/lib/l10n/app_en.arb @@ -16082,5 +16082,60 @@ "type": "int" } } + }, + "dataStorageGroup": "Storage", + "@dataStorageGroup": { + "description": "Data screen group title: storage settings and compaction." + }, + "dataKeepHistory": "Keep raw readings", + "@dataKeepHistory": { + "description": "Row title: how long the compressed raw-reading archive is kept." + }, + "dataKeepHistorySub": "A compressed copy of each day's second-by-second readings, kept after the app clears them. A day older than this keeps its scores but can't be re-scored when scoring improves", + "@dataKeepHistorySub": { + "description": "Row sub for the archive window setting." + }, + "dataKeepDays": "{days} days", + "@dataKeepDays": { + "description": "Archive window value: a number of days.", + "placeholders": { + "days": { + "type": "int" + } + } + }, + "dataKeepYear": "1 year", + "@dataKeepYear": { + "description": "Archive window value: one year." + }, + "dataKeepForever": "Forever", + "@dataKeepForever": { + "description": "Archive window value: never delete." + }, + "dataKeepShorterTitle": "Keep {window} of raw readings?", + "@dataKeepShorterTitle": { + "description": "Confirm title when the archive window is shortened.", + "placeholders": { + "window": { + "type": "String" + } + } + }, + "dataKeepShorterBody": "Readings older than {window} are deleted now. Those days keep their scores, but can't be re-scored after future updates.", + "@dataKeepShorterBody": { + "description": "Confirm body when the archive window is shortened.", + "placeholders": { + "window": { + "type": "String" + } + } + }, + "dataKeepShorterConfirm": "Delete older readings", + "@dataKeepShorterConfirm": { + "description": "Confirm button: shorten the window and delete older archive." + }, + "dataKeepShorterCancel": "Keep them", + "@dataKeepShorterCancel": { + "description": "Cancel button: keep the current window." } } diff --git a/lib/l10n/app_es.arb b/lib/l10n/app_es.arb index 72a83d045..bd934d814 100644 --- a/lib/l10n/app_es.arb +++ b/lib/l10n/app_es.arb @@ -2816,5 +2816,36 @@ "notifBatteryForecastReserveBody": "A {rate} %/h se agota hacia las {time}, poco después de tu hora habitual de despertar: te quedará alrededor del {left} % al levantarte. Cárgala ahora para no perder el sueño de esta noche.", "notifBatteryForecastAfterWakeBody": "A {rate} %/h se agota hacia las {time}, después de tu hora habitual de despertar.", "notifWindDownBody": "Te sueles acostar hacia las {time}. Empieza a bajar el ritmo.", - "notifWorkoutIdleBody": "No se ha registrado nada por encima del reposo en {minutes} minutos. Si la sesión terminó, finalízala desde la pestaña Entrenamiento." + "notifWorkoutIdleBody": "No se ha registrado nada por encima del reposo en {minutes} minutos. Si la sesión terminó, finalízala desde la pestaña Entrenamiento.", + "dataStorageGroup": "Almacenamiento", + "dataKeepHistory": "Conservar lecturas sin procesar", + "dataKeepHistorySub": "Una copia comprimida de las lecturas segundo a segundo de cada día, que se conserva después de que la app las borre. Los días más antiguos mantienen sus puntuaciones, pero no se pueden recalcular cuando mejore el cálculo", + "dataKeepDays": "{days} días", + "@dataKeepDays": { + "placeholders": { + "days": { + "type": "int" + } + } + }, + "dataKeepYear": "1 año", + "dataKeepForever": "Siempre", + "dataKeepShorterTitle": "¿Conservar {window} de lecturas sin procesar?", + "@dataKeepShorterTitle": { + "placeholders": { + "window": { + "type": "String" + } + } + }, + "dataKeepShorterBody": "Las lecturas de hace más de {window} se borran ahora. Esos días mantienen sus puntuaciones, pero no se podrán recalcular tras futuras actualizaciones.", + "@dataKeepShorterBody": { + "placeholders": { + "window": { + "type": "String" + } + } + }, + "dataKeepShorterConfirm": "Borrar lecturas antiguas", + "dataKeepShorterCancel": "Conservarlas" } diff --git a/lib/l10n/app_fr.arb b/lib/l10n/app_fr.arb index 06997c709..d7b7be690 100644 --- a/lib/l10n/app_fr.arb +++ b/lib/l10n/app_fr.arb @@ -2787,5 +2787,36 @@ "notifBatteryForecastReserveBody": "À {rate} %/h, il sera vide vers {time}, juste après votre heure de réveil habituelle : il restera environ {left} % au lever. Rechargez-le maintenant pour garder la nuit.", "notifBatteryForecastAfterWakeBody": "À {rate} %/h, il sera vide vers {time}, après votre heure de réveil habituelle.", "notifWindDownBody": "Vous vous couchez vers {time}. Commencez à ralentir.", - "notifWorkoutIdleBody": "Rien au-dessus du repos n'a été enregistré depuis {minutes} minutes. Si la séance est finie, terminez-la depuis l'onglet Entraînement." + "notifWorkoutIdleBody": "Rien au-dessus du repos n'a été enregistré depuis {minutes} minutes. Si la séance est finie, terminez-la depuis l'onglet Entraînement.", + "dataStorageGroup": "Stockage", + "dataKeepHistory": "Conserver les mesures brutes", + "dataKeepHistorySub": "Une copie compressée des mesures seconde par seconde de chaque jour, conservée après que l'app les a effacées. Un jour plus ancien garde ses scores mais ne peut pas être recalculé quand le calcul s'améliore", + "dataKeepDays": "{days} jours", + "@dataKeepDays": { + "placeholders": { + "days": { + "type": "int" + } + } + }, + "dataKeepYear": "1 an", + "dataKeepForever": "Toujours", + "dataKeepShorterTitle": "Conserver {window} de mesures brutes ?", + "@dataKeepShorterTitle": { + "placeholders": { + "window": { + "type": "String" + } + } + }, + "dataKeepShorterBody": "Les mesures de plus de {window} sont supprimées maintenant. Ces jours gardent leurs scores, mais ne pourront pas être recalculés après les prochaines mises à jour.", + "@dataKeepShorterBody": { + "placeholders": { + "window": { + "type": "String" + } + } + }, + "dataKeepShorterConfirm": "Supprimer les anciennes mesures", + "dataKeepShorterCancel": "Les garder" } diff --git a/lib/l10n/app_hi.arb b/lib/l10n/app_hi.arb index 324870bab..f2530c44f 100644 --- a/lib/l10n/app_hi.arb +++ b/lib/l10n/app_hi.arb @@ -2787,5 +2787,36 @@ "notifBatteryForecastReserveBody": "{rate}%/घंटा की दर से यह लगभग {time} तक खत्म हो जाएगी, आपके सामान्य जागने के समय के ठीक बाद — उठते समय लगभग {left}% बचेगी। आज रात की नींद बचाने के लिए अभी चार्ज करें।", "notifBatteryForecastAfterWakeBody": "{rate}%/घंटा की दर से यह लगभग {time} तक खत्म हो जाएगी, आपके सामान्य जागने के समय के बाद।", "notifWindDownBody": "आपके सोने का समय लगभग {time} है। धीमे होना शुरू करें।", - "notifWorkoutIdleBody": "{minutes} मिनट से आराम से ऊपर की कोई मेहनत दर्ज नहीं हुई है। अगर सेशन खत्म हो गया है, तो उसे वर्कआउट टैब से खत्म करें।" + "notifWorkoutIdleBody": "{minutes} मिनट से आराम से ऊपर की कोई मेहनत दर्ज नहीं हुई है। अगर सेशन खत्म हो गया है, तो उसे वर्कआउट टैब से खत्म करें।", + "dataStorageGroup": "स्टोरेज", + "dataKeepHistory": "कच्ची रीडिंग रखें", + "dataKeepHistorySub": "हर दिन की सेकंड-दर-सेकंड रीडिंग की एक संपीड़ित कॉपी, जो ऐप के उन्हें हटाने के बाद भी रहती है। इससे पुराने दिन अपने स्कोर रखते हैं, पर स्कोरिंग बेहतर होने पर दोबारा नहीं आंके जा सकते", + "dataKeepDays": "{days} दिन", + "@dataKeepDays": { + "placeholders": { + "days": { + "type": "int" + } + } + }, + "dataKeepYear": "1 साल", + "dataKeepForever": "हमेशा", + "dataKeepShorterTitle": "{window} की कच्ची रीडिंग रखें?", + "@dataKeepShorterTitle": { + "placeholders": { + "window": { + "type": "String" + } + } + }, + "dataKeepShorterBody": "{window} से पुरानी रीडिंग अभी हटा दी जाएंगी। उन दिनों के स्कोर बने रहेंगे, पर आगे के अपडेट के बाद उन्हें दोबारा नहीं आंका जा सकेगा।", + "@dataKeepShorterBody": { + "placeholders": { + "window": { + "type": "String" + } + } + }, + "dataKeepShorterConfirm": "पुरानी रीडिंग हटाएं", + "dataKeepShorterCancel": "उन्हें रखें" } diff --git a/lib/l10n/app_ru.arb b/lib/l10n/app_ru.arb index c6fa2177b..3d835323d 100644 --- a/lib/l10n/app_ru.arb +++ b/lib/l10n/app_ru.arb @@ -7732,5 +7732,36 @@ "type": "int" } } - } + }, + "dataStorageGroup": "Хранилище", + "dataKeepHistory": "Хранить исходные показания", + "dataKeepHistorySub": "Сжатая копия посекундных показаний за каждый день, которая остаётся после того, как приложение их удалит. Более старые дни сохраняют свои оценки, но их нельзя пересчитать после улучшения расчётов", + "dataKeepDays": "{days} дн.", + "@dataKeepDays": { + "placeholders": { + "days": { + "type": "int" + } + } + }, + "dataKeepYear": "1 год", + "dataKeepForever": "Всегда", + "dataKeepShorterTitle": "Хранить исходные показания за {window}?", + "@dataKeepShorterTitle": { + "placeholders": { + "window": { + "type": "String" + } + } + }, + "dataKeepShorterBody": "Показания старше {window} будут удалены сейчас. Эти дни сохранят свои оценки, но их нельзя будет пересчитать после будущих обновлений.", + "@dataKeepShorterBody": { + "placeholders": { + "window": { + "type": "String" + } + } + }, + "dataKeepShorterConfirm": "Удалить старые показания", + "dataKeepShorterCancel": "Оставить" } diff --git a/lib/l10n/app_zh.arb b/lib/l10n/app_zh.arb index 7e27582d6..242e27dfa 100644 --- a/lib/l10n/app_zh.arb +++ b/lib/l10n/app_zh.arb @@ -2787,5 +2787,36 @@ "notifBatteryForecastReserveBody": "按每小时 {rate}% 的耗电速度，电量将在 {time} 左右耗尽，略晚于你平时起床的时间，起床时约剩 {left}%。现在充电，才能记录今晚的睡眠。", "notifBatteryForecastAfterWakeBody": "按每小时 {rate}% 的耗电速度，电量将在 {time} 左右耗尽，晚于你平时起床的时间。", "notifWindDownBody": "你通常在 {time} 左右就寝。开始放慢节奏吧。", - "notifWorkoutIdleBody": "已有 {minutes} 分钟没有记录到高于静息水平的运动。如果训练已结束，请在“锻炼”标签页中结束它。" + "notifWorkoutIdleBody": "已有 {minutes} 分钟没有记录到高于静息水平的运动。如果训练已结束，请在“锻炼”标签页中结束它。", + "dataStorageGroup": "存储", + "dataKeepHistory": "保留原始读数", + "dataKeepHistorySub": "每天逐秒读数的压缩副本，在应用清除原始数据后仍会保留。超过此期限的日子保留其评分，但在评分改进后无法重新计算", + "dataKeepDays": "{days} 天", + "@dataKeepDays": { + "placeholders": { + "days": { + "type": "int" + } + } + }, + "dataKeepYear": "1 年", + "dataKeepForever": "永久", + "dataKeepShorterTitle": "保留 {window} 的原始读数？", + "@dataKeepShorterTitle": { + "placeholders": { + "window": { + "type": "String" + } + } + }, + "dataKeepShorterBody": "超过 {window} 的读数将立即删除。这些日子会保留评分，但在以后的更新后无法重新计算。", + "@dataKeepShorterBody": { + "placeholders": { + "window": { + "type": "String" + } + } + }, + "dataKeepShorterConfirm": "删除较早的读数", + "dataKeepShorterCancel": "保留" } diff --git a/lib/state/app_state.dart b/lib/state/app_state.dart index 494f96ee5..6ff7c76c4 100644 --- a/lib/state/app_state.dart +++ b/lib/state/app_state.dart @@ -91,6 +91,7 @@ import 'prefs.dart'; import '../ble/adapters/signals.dart' show InputSignal; import '../ui2/profile/devices.dart' show liveSources, rankSources; import '../data/db.dart'; +import '../data/substrate_archive.dart' show SubstrateArchivePolicy; import '../data/export_retention.dart' show purgeSharedHealthCopies; import '../ecg/ble_ecg_transport.dart'; import '../ecg/ecg_controller.dart'; @@ -2449,6 +2450,20 @@ class AppState extends ChangeNotifier { } } + /// Set how long pruned raw readings stay in `substrate_archive`. A shorter + /// window evicts the days it no longer covers right away, behind the derive + /// latch so a prune that is archiving a bucket finishes first. + Future setSubstrateArchiveWindow(SubstrateArchivePolicy next) async { + final current = await LocalDb.substrateArchivePolicy(); + await LocalDb.setSubstrateArchivePolicy(next); + if (!current.keepsLongerThan(next)) return; + final edge = await LocalDb.lastDecodedRecTs(); + if (edge == null) return; + final evicted = await DerivationEngine.whileIdle( + () => LocalDb.evictSubstrateArchive(next, edge)); + if (evicted > 0) _log('[db] archive window shortened: $evicted bucket(s)'); + } + /// User-initiated "Re-analyze data": force-derive EVERY day that has raw, /// ignoring the derived cursor, then refresh the UI. Returns the days derived, /// the days that could not be, and the error when the pass itself failed (for diff --git a/lib/ui2/profile/data.dart b/lib/ui2/profile/data.dart index bf10e2871..a0fb9fd53 100644 --- a/lib/ui2/profile/data.dart +++ b/lib/ui2/profile/data.dart @@ -27,6 +27,7 @@ import '../../data/csv_export.dart'; import '../../data/day_label.dart' show dayLabelOf; import '../../data/db.dart'; import '../../data/export_retention.dart'; +import '../../data/substrate_archive.dart' show SubstrateArchivePolicy; import '../../import/backup_crypto.dart'; import '../../l10n/app_localizations.dart'; import '../../state/app_state.dart'; @@ -64,6 +65,24 @@ class _DataScreenState extends State { bool _noteFailed = false; ImportOutcome? _outcome; + /// How long pruned raw readings are archived. Null until read. + SubstrateArchivePolicy? _archivePolicy; + + @override + void initState() { + super.initState(); + _loadStorage(); + } + + Future _loadStorage() async { + try { + final policy = await LocalDb.substrateArchivePolicy(); + if (mounted) setState(() => _archivePolicy = policy); + } catch (_) { + // The row stays blank; tapping it re-reads. + } + } + void _say(String s, {bool failed = false}) { if (mounted) { setState(() { @@ -224,6 +243,32 @@ class _DataScreenState extends State { ); } + /// Step to the next archive window. A shorter one deletes archived days + /// for good, so it asks first. + Future<_Note> _stepArchiveWindow(AppState app) async { + final l = AppLocalizations.of(context); + final current = _archivePolicy ?? await LocalDb.substrateArchivePolicy(); + final next = current.next; + if (current.keepsLongerThan(next)) { + if (!mounted) return ('', false); + final window = _windowLabel(l, next.keepDays); + final ok = await confirmRemove( + context, + title: l?.dataKeepShorterTitle(window) ?? + 'Keep $window of raw readings?', + body: l?.dataKeepShorterBody(window) ?? + 'Readings older than $window are deleted now. Those days keep ' + "their scores, but can't be re-scored after future updates.", + remove: l?.dataKeepShorterConfirm ?? 'Delete older readings', + keep: l?.dataKeepShorterCancel ?? 'Keep them', + ); + if (!ok) return ('', false); + } + await app.setSubstrateArchiveWindow(next); + if (mounted) setState(() => _archivePolicy = next); + return ('', false); + } + Future<_Note> _reanalyze(AppState app) async { final l = AppLocalizations.of(context); final ml = MaterialLocalizations.of(context); @@ -463,6 +508,20 @@ class _DataScreenState extends State { onTap: _busy ? null : () => goto(c, const PhoneImport())), ]), const SizedBox(height: S.x5), + settingsGroup(c, l?.dataStorageGroup ?? 'Storage', [ + SetRow(LucideIcons.archive, C.teal, + l?.dataKeepHistory ?? 'Keep raw readings', + sub: l?.dataKeepHistorySub ?? + "A compressed copy of each day's second-by-second " + 'readings, kept after the app clears them. A day ' + "older than this keeps its scores but can't be " + 're-scored when scoring improves', + value: _archivePolicy == null + ? '' + : _windowLabel(l, _archivePolicy!.keepDays), + onTap: _busy ? null : () => _run(() => _stepArchiveWindow(app))), + ]), + const SizedBox(height: S.x5), settingsGroup(c, l?.dataRebuildGroup ?? 'Rebuild', [ // The engine puts days on hold after a ≥3 h timezone jump // "until Re-analyze data runs" — and nothing in the app ran @@ -524,6 +583,13 @@ class _DataScreenState extends State { } } +/// The archive window as the row shows it. +String _windowLabel(AppLocalizations? l, int keepDays) => keepDays < 0 + ? (l?.dataKeepForever ?? 'Forever') + : keepDays == 365 + ? (l?.dataKeepYear ?? '1 year') + : (l?.dataKeepDays(keepDays) ?? '$keepDays days'); + /// Off → Daily → Weekly → Off. Three states cycle in a row; a picker for three /// options is a sheet nobody needs. BackupCadence _nextCadence(BackupCadence c) => BackupCadence diff --git a/test/raw_prune_gating_test.dart b/test/raw_prune_gating_test.dart index 667c63b6a..cf0091e6b 100644 --- a/test/raw_prune_gating_test.dart +++ b/test/raw_prune_gating_test.dart @@ -80,14 +80,14 @@ void main() { final dataNow = _dayStart('2026-05-20') + 12 * 3600; final cutoff = DerivationEngine.rawPruneCutoffSec( dataNowSec: dataNow, - rawDayIds: const ['2026-05-12', '2026-05-13', '2026-05-19'], + rawDayIds: const ['2026-05-12', '2026-05-15', '2026-05-19'], finalizedDayIds: const {'2026-05-12', '2026-05-19'}, ); - // Held at 05-12 (05-13's night search starts 05-12 noon)… - expect(cutoff, _dayStart('2026-05-12')); + // Held at 05-14 (05-15's night search starts 05-14 noon)… + expect(cutoff, _dayStart('2026-05-14')); // …but older finalized days are still reclaimed. The old guard // returned early and kept them too. - expect(cutoff, greaterThan(_dayStart('2026-05-11'))); + expect(cutoff, greaterThan(_dayStart('2026-05-13'))); }); // Day D's night starts the evening before. A day with a result that has @@ -120,7 +120,7 @@ void main() { )!; expect(cutoff, greaterThan(_dayStart('2026-01-05'))); // Never keeps more than the documented ceiling behind the data edge. - expect(dataNow - cutoff, lessThanOrEqualTo(14 * 86400)); + expect(dataNow - cutoff, lessThanOrEqualTo(7 * 86400)); }); test('a day still inside the retention window does not move it', () { diff --git a/test/substrate_archive_window_test.dart b/test/substrate_archive_window_test.dart new file mode 100644 index 000000000..4cb7906c6 --- /dev/null +++ b/test/substrate_archive_window_test.dart @@ -0,0 +1,38 @@ +// The archive window the data settings step through. + +import 'package:flutter_test/flutter_test.dart'; +import 'package:openstrap_edge/data/substrate_archive.dart'; + +void main() { + test('defaults to 90 days', () { + expect(SubstrateArchivePolicy.defaults.keepDays, 90); + expect(SubstrateArchivePolicy.fromJson(null).keepDays, 90); + }); + + test('steps 30 → 90 → 365 → forever → 30', () { + var p = const SubstrateArchivePolicy(30); + final seen = []; + for (var i = 0; i < 4; i++) { + p = p.next; + seen.add(p.keepDays); + } + expect(seen, [90, 365, -1, 30]); + }); + + test('an off or hand-set window moves to the next longer choice', () { + expect(const SubstrateArchivePolicy(0).next.keepDays, 30); + expect(const SubstrateArchivePolicy(45).next.keepDays, 90); + expect(const SubstrateArchivePolicy(400).next.keepDays, -1); + }); + + test('only a shorter window counts as deleting history', () { + const forever = SubstrateArchivePolicy(-1); + const year = SubstrateArchivePolicy(365); + const month = SubstrateArchivePolicy(30); + expect(forever.keepsLongerThan(month), isTrue); + expect(year.keepsLongerThan(month), isTrue); + expect(month.keepsLongerThan(year), isFalse); + expect(month.keepsLongerThan(forever), isFalse); + expect(year.keepsLongerThan(year), isFalse); + }); +} From 9bcc898c64f5c05153f2d16d70a03178ddbad092 Mon Sep 17 00:00:00 2001 From: abdulsaheel Date: Wed, 28 Oct 2026 22:25:00 +0530 Subject: [PATCH 03/33] add a compact database action to the data settings It runs the retention prune and archive eviction a derive pass ends with, then VACUUM and a truncating WAL checkpoint, behind the derive latch. It refuses to start during a sync, an offload or a live session, and only vacuums with twice the database free, read from the platform. It reports what it freed. (cherry picked from commit 0a8946b898d9846e26493463a6cf13a5cea2fde9) --- .../openstrap_edge/NativeChannels.kt | 19 ++++ ios/Runner/AppDelegate.swift | 36 +++++++ lib/compute/derivation_engine.dart | 85 +++++++++++++++ lib/data/db.dart | 25 +++++ lib/data/disk_space.dart | 31 ++++++ lib/l10n/app_de.arb | 38 ++++++- lib/l10n/app_en.arb | 57 ++++++++++ lib/l10n/app_es.arb | 38 ++++++- lib/l10n/app_fr.arb | 38 ++++++- lib/l10n/app_hi.arb | 38 ++++++- lib/l10n/app_ru.arb | 38 ++++++- lib/l10n/app_zh.arb | 38 ++++++- lib/state/app_state.dart | 21 ++++ lib/ui2/profile/data.dart | 80 +++++++++++++- test/compact_database_test.dart | 102 ++++++++++++++++++ 15 files changed, 677 insertions(+), 7 deletions(-) create mode 100644 lib/data/disk_space.dart create mode 100644 test/compact_database_test.dart diff --git a/android/app/src/main/kotlin/wtf/openstrap/openstrap_edge/NativeChannels.kt b/android/app/src/main/kotlin/wtf/openstrap/openstrap_edge/NativeChannels.kt index fdd6cb8e2..c8cd72b84 100644 --- a/android/app/src/main/kotlin/wtf/openstrap/openstrap_edge/NativeChannels.kt +++ b/android/app/src/main/kotlin/wtf/openstrap/openstrap_edge/NativeChannels.kt @@ -37,6 +37,7 @@ object NativeChannels { private const val DEVICE_ACTIONS_CHANNEL = "openstrap/device_actions" private const val ANDROID_BG_CHANNEL = "openstrap/android_background" private const val BLE_NATIVE_CHANNEL = "openstrap/ble_native" + private const val STORAGE_CHANNEL = "openstrap/storage" const val TASKER_CHANNEL = "openstrap/tasker" const val ACTION_DOUBLE_TAP = "wtf.openstrap.openstrap_edge.DOUBLE_TAP" private const val TASKER_TOKEN_KEY = "tasker_auth_token" @@ -183,6 +184,24 @@ object NativeChannels { } } + // Free space on the volume holding the database, for "Compact database". + MethodChannel(engine.dartExecutor.binaryMessenger, STORAGE_CHANNEL) + .setMethodCallHandler { call, result -> + when (call.method) { + "freeBytes" -> { + val path = call.argument("path") ?: app.filesDir.path + result.success( + try { + android.os.StatFs(path).availableBytes + } catch (e: IllegalArgumentException) { + null + } + ) + } + else -> result.notImplemented() + } + } + // Band-gesture actions. All no-risk OS APIs: media-key dispatch (works for any // player, no permission), system media volume, ringtone + vibrate, torch. MethodChannel(engine.dartExecutor.binaryMessenger, DEVICE_ACTIONS_CHANNEL) diff --git a/ios/Runner/AppDelegate.swift b/ios/Runner/AppDelegate.swift index 0d02f683f..384db0ee9 100644 --- a/ios/Runner/AppDelegate.swift +++ b/ios/Runner/AppDelegate.swift @@ -95,6 +95,10 @@ import CoreMotion if let registrar = registry.registrar(forPlugin: "ConfigBridge") { ConfigBridge.register(messenger: registrar.messenger()) } + // Free space on the volume holding the database, for "Compact database". + if let registrar = registry.registrar(forPlugin: "StorageBridge") { + StorageBridge.register(messenger: registrar.messenger()) + } // The phone's OWN step count (CMPedometer), not HealthKit's multi-writer // aggregate. See lib/health/phone_pedometer.dart. if let registrar = registry.registrar(forPlugin: "PedometerBridge") { @@ -123,6 +127,38 @@ import CoreMotion } } +enum StorageBridge { + private static let channelName = "openstrap/storage" + + static func register(messenger: FlutterBinaryMessenger) { + let channel = FlutterMethodChannel(name: channelName, binaryMessenger: messenger) + channel.setMethodCallHandler { call, result in + switch call.method { + case "freeBytes": + // Bytes a write can actually use. The "important usage" figure counts + // purgeable space iOS frees on demand, which is what a VACUUM sees. + let path = (call.arguments as? [String: Any])?["path"] as? String + ?? NSHomeDirectory() + let url = URL(fileURLWithPath: path) + do { + let values = try url.resourceValues( + forKeys: [.volumeAvailableCapacityForImportantUsageKey]) + if let bytes = values.volumeAvailableCapacityForImportantUsage { + result(NSNumber(value: bytes)) + return + } + let attrs = try FileManager.default.attributesOfFileSystem(forPath: path) + result(attrs[.systemFreeSize] as? NSNumber) + } catch { + result(nil) + } + default: + result(FlutterMethodNotImplemented) + } + } + } +} + enum ConfigBridge { private static let channelName = "openstrap/ios_config" diff --git a/lib/compute/derivation_engine.dart b/lib/compute/derivation_engine.dart index 30ac6a0d4..21cb8f262 100644 --- a/lib/compute/derivation_engine.dart +++ b/lib/compute/derivation_engine.dart @@ -2989,6 +2989,32 @@ Future _agingEdgeFor(String? family, int dataNowSec, Future debugAgingEdge(String? family, int dataNowSec) => _agingEdgeFor(family, dataNowSec); +/// Where a [DerivationEngine.compactStorage] has got to. +enum CompactStep { prune, vacuum } + +/// What a [DerivationEngine.compactStorage] did. Sizes are the database file +/// plus its WAL, in bytes. +class CompactResult { + const CompactResult({ + required this.beforeBytes, + required this.afterBytes, + required this.freeBytes, + required this.neededBytes, + required this.vacuumed, + }); + + final int beforeBytes, afterBytes, neededBytes; + + /// Free space on the volume when it was checked; null when the platform + /// could not say. + final int? freeBytes; + + /// False when the free-space gate refused the VACUUM (the prune still ran). + final bool vacuumed; + + int get freedBytes => beforeBytes > afterBytes ? beforeBytes - afterBytes : 0; +} + class DerivationEngine { DerivationEngine({this.log, this.background = false, this.offloadActive}); final void Function(String)? log; @@ -3056,6 +3082,65 @@ class DerivationEngine { } } + /// The user's "Compact database": the retention prune and archive eviction + /// a derive pass ends with, then a full `VACUUM` and a truncating WAL + /// checkpoint, all under the derive latch (waiting, like [whileIdle], for a + /// pass in flight). The prune is the pass's own: it holds unfinalized days + /// exactly as [_pruneOldDecoded] does, and it never touches anything a + /// drain has not committed, so commit-before-ACK is untouched — a drain that + /// starts meanwhile only waits on the database lock, and a commit that + /// cannot land is never acknowledged. + /// + /// The VACUUM runs only when [freeDiskBytes] reports at least twice the + /// database (plus its WAL) free: it writes a whole second copy before it + /// drops the first. Unknown free space counts as not enough. + Future compactStorage({ + required Future Function() freeDiskBytes, + void Function(CompactStep step)? onStep, + }) => + whileIdle(() async { + final before = await LocalDb.databaseBytesOnDisk(); + onStep?.call(CompactStep.prune); + try { + // Every day with substrate, the wearable's included, as the scope + // hands a pass's prune: a pending day missing here is not held. + final rawByDay = await LocalDb.decodedRecTsMaxByDay(); + final wearable = await activeWearable(); + if (wearable != null) { + for (final day in (await wearableRecTsMaxByDay(wearable)).keys) { + rawByDay.putIfAbsent(day, () => 0); + } + } + final rawDays = rawByDay.keys.toList()..sort(); + final dataNowSec = + await _withWearableEdge(await LocalDb.lastDecodedRecTs()) ?? 0; + if (dataNowSec > 0) await _pruneOldDecoded(rawDays, dataNowSec); + } catch (e) { + _log('compact: prune skipped: $e'); + } + await _runStorageHousekeeping(); + final size = await LocalDb.databaseBytesOnDisk(); + final free = await freeDiskBytes(); + if (free == null || free < 2 * size) { + return CompactResult( + beforeBytes: before, + afterBytes: size, + freeBytes: free, + neededBytes: 2 * size, + vacuumed: false, + ); + } + onStep?.call(CompactStep.vacuum); + await LocalDb.vacuumAndCheckpoint(); + return CompactResult( + beforeBytes: before, + afterBytes: await LocalDb.databaseBytesOnDisk(), + freeBytes: free, + neededBytes: 2 * size, + vacuumed: true, + ); + }); + final Map _diag = { 'running': false, 'stage': 'idle', diff --git a/lib/data/db.dart b/lib/data/db.dart index 7012beb44..9aa93bf8b 100644 --- a/lib/data/db.dart +++ b/lib/data/db.dart @@ -9519,6 +9519,31 @@ class LocalDb { return major > 3 || (major == 3 && minor >= 27); } + /// The database file plus its `-wal`: what the user's storage settings + /// show it costing. After a VACUUM in WAL mode the WAL can be as big as the + /// file until a checkpoint truncates it. + static Future databaseBytesOnDisk() async { + final path = p.join(await getDatabasesPath(), dbName); + var total = 0; + for (final f in [File(path), File('$path-wal')]) { + if (await f.exists()) total += await f.length(); + } + return total; + } + + /// Rewrite the file without its free pages, then hand the WAL's copy of + /// that rewrite back to the filesystem. The caller gates it (free space, + /// nothing else in flight); see [vacuumIfBloated] for why this is a one-off + /// VACUUM and not `auto_vacuum`. The checkpoint is a `rawQuery`: the PRAGMA + /// returns a row, which `execute` refuses on iOS's sqflite. + static Future vacuumAndCheckpoint() async { + final db = await instance; + await db.execute('VACUUM'); + // VACUUM renumbers every btree root page; see [vacuumIfBloated]. + await CoachDb.close(); + await db.rawQuery('PRAGMA wal_checkpoint(TRUNCATE)'); + } + static Future databaseFileBytes() async { final dir = await getDatabasesPath(); final path = p.join(dir, dbName); diff --git a/lib/data/disk_space.dart b/lib/data/disk_space.dart new file mode 100644 index 000000000..838158cca --- /dev/null +++ b/lib/data/disk_space.dart @@ -0,0 +1,31 @@ +// Free space on the volume that holds the database, from the platform. +// +// Only "Compact database" asks: a VACUUM writes a full second copy of the file +// before it drops the first, so it must not start on a nearly full phone. + +import 'package:flutter/foundation.dart' show visibleForTesting; +import 'package:flutter/services.dart'; +import 'package:sqflite/sqflite.dart' show getDatabasesPath; + +const MethodChannel _channel = MethodChannel('openstrap/storage'); + +/// Test seam: when set, answers instead of the platform. +@visibleForTesting +Future Function()? debugFreeDiskBytes; + +/// Bytes free for the app on the database's volume, or null when the platform +/// cannot say (no channel, a desktop test, an error). Null is "unknown", and +/// every caller treats unknown as not enough. +Future freeDiskBytes() async { + final override = debugFreeDiskBytes; + if (override != null) return override(); + try { + final v = await _channel.invokeMethod( + 'freeBytes', + {'path': await getDatabasesPath()}, + ); + return v is num ? v.toInt() : null; + } catch (_) { + return null; + } +} diff --git a/lib/l10n/app_de.arb b/lib/l10n/app_de.arb index 17d983394..7585ac1b2 100644 --- a/lib/l10n/app_de.arb +++ b/lib/l10n/app_de.arb @@ -2935,5 +2935,41 @@ } }, "dataKeepShorterConfirm": "Ältere Messwerte löschen", - "dataKeepShorterCancel": "Behalten" + "dataKeepShorterCancel": "Behalten", + "dataCompact": "Datenbank verkleinern", + "dataCompactSub": "Löscht, was die Einstellungen oben nicht mehr behalten, und gibt den freien Platz an das Telefon zurück. Braucht etwa doppelt so viel freien Speicher, wie die Datenbank groß ist", + "dataCompactPruning": "Alte Daten werden gelöscht…", + "dataCompactVacuuming": "Wird verkleinert…", + "dataCompactFreed": "{freed} MB freigegeben. Die Datenbank ist jetzt {size} MB groß.", + "@dataCompactFreed": { + "placeholders": { + "freed": { + "type": "int" + }, + "size": { + "type": "int" + } + } + }, + "dataCompactNoSpace": "Alte Daten wurden gelöscht, aber zum Verkleinern sind {need} MB frei nötig und das Telefon hat {free} MB.", + "@dataCompactNoSpace": { + "placeholders": { + "need": { + "type": "int" + }, + "free": { + "type": "int" + } + } + }, + "dataCompactUnknownSpace": "Alte Daten wurden gelöscht, aber der freie Speicher des Telefons ließ sich nicht lesen, daher wurde die Datei nicht verkleinert.", + "dataCompactBusy": "Eine Synchronisierung oder eine laufende Sitzung ist aktiv. Versuche es danach noch einmal.", + "dataSizeMb": "{mb} MB", + "@dataSizeMb": { + "placeholders": { + "mb": { + "type": "int" + } + } + } } diff --git a/lib/l10n/app_en.arb b/lib/l10n/app_en.arb index 80d9d9fe5..77cdfe72f 100644 --- a/lib/l10n/app_en.arb +++ b/lib/l10n/app_en.arb @@ -16137,5 +16137,62 @@ "dataKeepShorterCancel": "Keep them", "@dataKeepShorterCancel": { "description": "Cancel button: keep the current window." + }, + "dataCompact": "Compact database", + "@dataCompact": { + "description": "Row title: compact the database file." + }, + "dataCompactSub": "Clears what the settings above no longer keep, then gives the unused space back to the phone. Needs free space of about twice the database's size", + "@dataCompactSub": { + "description": "Row sub for Compact database." + }, + "dataCompactPruning": "Clearing old data…", + "@dataCompactPruning": { + "description": "Progress line while old data is cleared before compaction." + }, + "dataCompactVacuuming": "Compacting…", + "@dataCompactVacuuming": { + "description": "Progress line while the database file is rewritten." + }, + "dataCompactFreed": "Freed {freed} MB. The database is now {size} MB.", + "@dataCompactFreed": { + "description": "Compaction result: megabytes freed and the new size.", + "placeholders": { + "freed": { + "type": "int" + }, + "size": { + "type": "int" + } + } + }, + "dataCompactNoSpace": "Old data was cleared, but compacting needs {need} MB free and the phone has {free} MB.", + "@dataCompactNoSpace": { + "description": "Compaction refused: not enough free storage.", + "placeholders": { + "need": { + "type": "int" + }, + "free": { + "type": "int" + } + } + }, + "dataCompactUnknownSpace": "Old data was cleared, but the phone's free space couldn't be read, so the file was not compacted.", + "@dataCompactUnknownSpace": { + "description": "Compaction refused: free storage could not be read." + }, + "dataCompactBusy": "A sync or a live session is running. Try again once it's done.", + "@dataCompactBusy": { + "description": "Compaction refused: a sync or session is running." + }, + "dataSizeMb": "{mb} MB", + "@dataSizeMb": { + "description": "A size in megabytes, shown as the row value.", + "placeholders": { + "mb": { + "type": "int" + } + } } } diff --git a/lib/l10n/app_es.arb b/lib/l10n/app_es.arb index bd934d814..3eb4a5ab8 100644 --- a/lib/l10n/app_es.arb +++ b/lib/l10n/app_es.arb @@ -2847,5 +2847,41 @@ } }, "dataKeepShorterConfirm": "Borrar lecturas antiguas", - "dataKeepShorterCancel": "Conservarlas" + "dataKeepShorterCancel": "Conservarlas", + "dataCompact": "Compactar la base de datos", + "dataCompactSub": "Borra lo que los ajustes de arriba ya no conservan y devuelve el espacio libre al teléfono. Necesita un espacio libre de unas dos veces el tamaño de la base de datos", + "dataCompactPruning": "Borrando datos antiguos…", + "dataCompactVacuuming": "Compactando…", + "dataCompactFreed": "Se liberaron {freed} MB. La base de datos ocupa ahora {size} MB.", + "@dataCompactFreed": { + "placeholders": { + "freed": { + "type": "int" + }, + "size": { + "type": "int" + } + } + }, + "dataCompactNoSpace": "Se borraron los datos antiguos, pero compactar necesita {need} MB libres y el teléfono tiene {free} MB.", + "@dataCompactNoSpace": { + "placeholders": { + "need": { + "type": "int" + }, + "free": { + "type": "int" + } + } + }, + "dataCompactUnknownSpace": "Se borraron los datos antiguos, pero no se pudo leer el espacio libre del teléfono, así que el archivo no se compactó.", + "dataCompactBusy": "Hay una sincronización o una sesión en curso. Inténtalo de nuevo cuando termine.", + "dataSizeMb": "{mb} MB", + "@dataSizeMb": { + "placeholders": { + "mb": { + "type": "int" + } + } + } } diff --git a/lib/l10n/app_fr.arb b/lib/l10n/app_fr.arb index d7b7be690..b35869048 100644 --- a/lib/l10n/app_fr.arb +++ b/lib/l10n/app_fr.arb @@ -2818,5 +2818,41 @@ } }, "dataKeepShorterConfirm": "Supprimer les anciennes mesures", - "dataKeepShorterCancel": "Les garder" + "dataKeepShorterCancel": "Les garder", + "dataCompact": "Compacter la base de données", + "dataCompactSub": "Efface ce que les réglages ci-dessus ne gardent plus, puis rend l'espace inutilisé au téléphone. Nécessite un espace libre d'environ deux fois la taille de la base", + "dataCompactPruning": "Effacement des anciennes données…", + "dataCompactVacuuming": "Compactage…", + "dataCompactFreed": "{freed} Mo libérés. La base fait maintenant {size} Mo.", + "@dataCompactFreed": { + "placeholders": { + "freed": { + "type": "int" + }, + "size": { + "type": "int" + } + } + }, + "dataCompactNoSpace": "Les anciennes données ont été effacées, mais le compactage nécessite {need} Mo libres et le téléphone en a {free}.", + "@dataCompactNoSpace": { + "placeholders": { + "need": { + "type": "int" + }, + "free": { + "type": "int" + } + } + }, + "dataCompactUnknownSpace": "Les anciennes données ont été effacées, mais l'espace libre du téléphone n'a pas pu être lu, donc le fichier n'a pas été compacté.", + "dataCompactBusy": "Une synchronisation ou une session est en cours. Réessayez une fois terminée.", + "dataSizeMb": "{mb} Mo", + "@dataSizeMb": { + "placeholders": { + "mb": { + "type": "int" + } + } + } } diff --git a/lib/l10n/app_hi.arb b/lib/l10n/app_hi.arb index f2530c44f..8b2374bfd 100644 --- a/lib/l10n/app_hi.arb +++ b/lib/l10n/app_hi.arb @@ -2818,5 +2818,41 @@ } }, "dataKeepShorterConfirm": "पुरानी रीडिंग हटाएं", - "dataKeepShorterCancel": "उन्हें रखें" + "dataKeepShorterCancel": "उन्हें रखें", + "dataCompact": "डेटाबेस छोटा करें", + "dataCompactSub": "ऊपर की सेटिंग जो अब नहीं रखतीं उसे हटाता है, फिर खाली जगह फ़ोन को लौटाता है। डेटाबेस के आकार से लगभग दोगुनी खाली जगह चाहिए", + "dataCompactPruning": "पुराना डेटा हटाया जा रहा है…", + "dataCompactVacuuming": "छोटा किया जा रहा है…", + "dataCompactFreed": "{freed} MB खाली हुआ। डेटाबेस अब {size} MB है।", + "@dataCompactFreed": { + "placeholders": { + "freed": { + "type": "int" + }, + "size": { + "type": "int" + } + } + }, + "dataCompactNoSpace": "पुराना डेटा हटा दिया गया, पर छोटा करने के लिए {need} MB खाली जगह चाहिए और फ़ोन में {free} MB है।", + "@dataCompactNoSpace": { + "placeholders": { + "need": { + "type": "int" + }, + "free": { + "type": "int" + } + } + }, + "dataCompactUnknownSpace": "पुराना डेटा हटा दिया गया, पर फ़ोन की खाली जगह पढ़ी नहीं जा सकी, इसलिए फ़ाइल छोटी नहीं की गई।", + "dataCompactBusy": "सिंक या कोई सत्र चल रहा है। उसके खत्म होने पर दोबारा कोशिश करें।", + "dataSizeMb": "{mb} MB", + "@dataSizeMb": { + "placeholders": { + "mb": { + "type": "int" + } + } + } } diff --git a/lib/l10n/app_ru.arb b/lib/l10n/app_ru.arb index 3d835323d..2ce73bbbd 100644 --- a/lib/l10n/app_ru.arb +++ b/lib/l10n/app_ru.arb @@ -7763,5 +7763,41 @@ } }, "dataKeepShorterConfirm": "Удалить старые показания", - "dataKeepShorterCancel": "Оставить" + "dataKeepShorterCancel": "Оставить", + "dataCompact": "Сжать базу данных", + "dataCompactSub": "Удаляет то, что настройки выше больше не хранят, и возвращает свободное место телефону. Нужно свободное место примерно вдвое больше размера базы", + "dataCompactPruning": "Удаление старых данных…", + "dataCompactVacuuming": "Сжатие…", + "dataCompactFreed": "Освобождено {freed} МБ. Теперь база занимает {size} МБ.", + "@dataCompactFreed": { + "placeholders": { + "freed": { + "type": "int" + }, + "size": { + "type": "int" + } + } + }, + "dataCompactNoSpace": "Старые данные удалены, но для сжатия нужно {need} МБ свободного места, а на телефоне {free} МБ.", + "@dataCompactNoSpace": { + "placeholders": { + "need": { + "type": "int" + }, + "free": { + "type": "int" + } + } + }, + "dataCompactUnknownSpace": "Старые данные удалены, но не удалось узнать свободное место на телефоне, поэтому файл не сжат.", + "dataCompactBusy": "Идёт синхронизация или активная сессия. Повторите, когда она закончится.", + "dataSizeMb": "{mb} МБ", + "@dataSizeMb": { + "placeholders": { + "mb": { + "type": "int" + } + } + } } diff --git a/lib/l10n/app_zh.arb b/lib/l10n/app_zh.arb index 242e27dfa..26041466b 100644 --- a/lib/l10n/app_zh.arb +++ b/lib/l10n/app_zh.arb @@ -2818,5 +2818,41 @@ } }, "dataKeepShorterConfirm": "删除较早的读数", - "dataKeepShorterCancel": "保留" + "dataKeepShorterCancel": "保留", + "dataCompact": "压缩数据库", + "dataCompactSub": "清除上面设置不再保留的数据，然后把未使用的空间还给手机。需要约为数据库两倍大小的可用空间", + "dataCompactPruning": "正在清除旧数据…", + "dataCompactVacuuming": "正在压缩…", + "dataCompactFreed": "已释放 {freed} MB。数据库现在为 {size} MB。", + "@dataCompactFreed": { + "placeholders": { + "freed": { + "type": "int" + }, + "size": { + "type": "int" + } + } + }, + "dataCompactNoSpace": "旧数据已清除，但压缩需要 {need} MB 可用空间，手机只有 {free} MB。", + "@dataCompactNoSpace": { + "placeholders": { + "need": { + "type": "int" + }, + "free": { + "type": "int" + } + } + }, + "dataCompactUnknownSpace": "旧数据已清除，但无法读取手机的可用空间，因此未压缩文件。", + "dataCompactBusy": "正在同步或有进行中的会话。完成后再试。", + "dataSizeMb": "{mb} MB", + "@dataSizeMb": { + "placeholders": { + "mb": { + "type": "int" + } + } + } } diff --git a/lib/state/app_state.dart b/lib/state/app_state.dart index 6ff7c76c4..9ba7217aa 100644 --- a/lib/state/app_state.dart +++ b/lib/state/app_state.dart @@ -91,6 +91,7 @@ import 'prefs.dart'; import '../ble/adapters/signals.dart' show InputSignal; import '../ui2/profile/devices.dart' show liveSources, rankSources; import '../data/db.dart'; +import '../data/disk_space.dart' show freeDiskBytes; import '../data/substrate_archive.dart' show SubstrateArchivePolicy; import '../data/export_retention.dart' show purgeSharedHealthCopies; import '../ecg/ble_ecg_transport.dart'; @@ -2464,6 +2465,26 @@ class AppState extends ChangeNotifier { if (evicted > 0) _log('[db] archive window shortened: $evicted bucket(s)'); } + /// The data settings' "Compact database": prune per the retention and + /// archive policies, VACUUM, truncate the WAL. Null when it declined to + /// start because a sync, an offload or a live session is running — a + /// VACUUM holds the write lock for the whole rewrite, and those are the + /// writers that must not wait on it. A derive in flight is waited for + /// rather than refused (see [DerivationEngine.compactStorage]). + Future compactDatabase({ + void Function(CompactStep step)? onStep, + }) async { + if (busy || syncingNow || _liveSessionActive) return null; + if (_deriveScheduler.offloadActive) return null; + final result = await _derive.compactStorage( + freeDiskBytes: freeDiskBytes, + onStep: onStep, + ); + _log('[db] compact: ${result.beforeBytes >> 20} MB -> ' + '${result.afterBytes >> 20} MB (vacuumed=${result.vacuumed})'); + return result; + } + /// User-initiated "Re-analyze data": force-derive EVERY day that has raw, /// ignoring the derived cursor, then refresh the UI. Returns the days derived, /// the days that could not be, and the error when the pass itself failed (for diff --git a/lib/ui2/profile/data.dart b/lib/ui2/profile/data.dart index a0fb9fd53..718803653 100644 --- a/lib/ui2/profile/data.dart +++ b/lib/ui2/profile/data.dart @@ -25,6 +25,7 @@ import 'package:share_plus/share_plus.dart'; import '../../data/auto_backup.dart'; import '../../data/csv_export.dart'; import '../../data/day_label.dart' show dayLabelOf; +import '../../compute/derivation_engine.dart' show CompactStep; import '../../data/db.dart'; import '../../data/export_retention.dart'; import '../../data/substrate_archive.dart' show SubstrateArchivePolicy; @@ -68,6 +69,12 @@ class _DataScreenState extends State { /// How long pruned raw readings are archived. Null until read. SubstrateArchivePolicy? _archivePolicy; + /// Database file plus WAL, for the Compact row. Null until read. + int? _dbBytes; + + /// The compaction step in progress, shown under the spinner. + String? _progress; + @override void initState() { super.initState(); @@ -77,7 +84,13 @@ class _DataScreenState extends State { Future _loadStorage() async { try { final policy = await LocalDb.substrateArchivePolicy(); - if (mounted) setState(() => _archivePolicy = policy); + final bytes = await LocalDb.databaseBytesOnDisk(); + if (mounted) { + setState(() { + _archivePolicy = policy; + _dbBytes = bytes; + }); + } } catch (_) { // The row stays blank; tapping it re-reads. } @@ -269,6 +282,54 @@ class _DataScreenState extends State { return ('', false); } + Future<_Note> _compact(AppState app) async { + final l = AppLocalizations.of(context); + try { + final r = await app.compactDatabase(onStep: (step) { + if (!mounted) return; + setState(() => _progress = switch (step) { + CompactStep.prune => + l?.dataCompactPruning ?? 'Clearing old data…', + CompactStep.vacuum => l?.dataCompactVacuuming ?? 'Compacting…', + }); + }); + if (r == null) { + return ( + l?.dataCompactBusy ?? + "A sync or a live session is running. Try again once it's done.", + true + ); + } + int mb(int bytes) => (bytes / (1 << 20)).round(); + if (mounted) setState(() => _dbBytes = r.afterBytes); + if (!r.vacuumed) { + final free = r.freeBytes; + return free == null + ? ( + l?.dataCompactUnknownSpace ?? + "Old data was cleared, but the phone's free space couldn't " + 'be read, so the file was not compacted.', + true + ) + : ( + l?.dataCompactNoSpace(mb(r.neededBytes), mb(free)) ?? + 'Old data was cleared, but compacting needs ' + '${mb(r.neededBytes)} MB free and the phone has ' + '${mb(free)} MB.', + true + ); + } + return ( + l?.dataCompactFreed(mb(r.freedBytes), mb(r.afterBytes)) ?? + 'Freed ${mb(r.freedBytes)} MB. The database is now ' + '${mb(r.afterBytes)} MB.', + false + ); + } finally { + if (mounted) setState(() => _progress = null); + } + } + Future<_Note> _reanalyze(AppState app) async { final l = AppLocalizations.of(context); final ml = MaterialLocalizations.of(context); @@ -520,6 +581,17 @@ class _DataScreenState extends State { ? '' : _windowLabel(l, _archivePolicy!.keepDays), onTap: _busy ? null : () => _run(() => _stepArchiveWindow(app))), + SetRow(LucideIcons.minimize2, C.blue, + l?.dataCompact ?? 'Compact database', + sub: l?.dataCompactSub ?? + 'Clears what the settings above no longer keep, then ' + 'gives the unused space back to the phone. Needs ' + "free space of about twice the database's size", + value: _dbBytes == null + ? '' + : (l?.dataSizeMb((_dbBytes! / (1 << 20)).round()) ?? + '${(_dbBytes! / (1 << 20)).round()} MB'), + onTap: _busy ? null : () => _run(() => _compact(app))), ]), const SizedBox(height: S.x5), settingsGroup(c, l?.dataRebuildGroup ?? 'Rebuild', [ @@ -541,6 +613,12 @@ class _DataScreenState extends State { if (_busy) ...[ const SizedBox(height: S.x6), Center(child: CircularProgressIndicator(color: p.on(C.blue))), + if (_progress != null) ...[ + const SizedBox(height: S.x3), + Center( + child: Text(_progress!, + style: F.cap.copyWith(color: p.ink2))), + ], ], if (_note != null && _note!.isNotEmpty) ...[ const SizedBox(height: S.x5), diff --git a/test/compact_database_test.dart b/test/compact_database_test.dart new file mode 100644 index 000000000..38383c08a --- /dev/null +++ b/test/compact_database_test.dart @@ -0,0 +1,102 @@ +// "Compact database": prune per policy, VACUUM, truncate the WAL — only with +// twice the database free, and never alongside a derive pass. + +import 'dart:io'; +import 'dart:typed_data'; + +import 'package:flutter_test/flutter_test.dart'; +import 'package:openstrap_edge/compute/derivation_engine.dart'; +import 'package:openstrap_edge/data/db.dart'; +import 'package:path/path.dart' as p; +import 'package:sqflite_common_ffi/sqflite_ffi.dart'; + +Future _useFreshDb(String name) async { + await LocalDb.close(); + LocalDb.dbName = name; + final dir = await databaseFactory.getDatabasesPath(); + await databaseFactory.deleteDatabase(p.join(dir, name)); +} + +/// Fill a scratch table and delete it, leaving free pages behind. +Future _bloat() async { + final db = await LocalDb.instance; + await db.execute('CREATE TABLE IF NOT EXISTS _bloat (x BLOB)'); + final blob = Uint8List(64 * 1024)..fillRange(0, 64 * 1024, 7); + await db.transaction((txn) async { + for (var i = 0; i < 160; i++) { + await txn.insert('_bloat', {'x': blob}); + } + }); + await db.execute('DROP TABLE _bloat'); +} + +Future _walBytes() async { + final path = p.join(await getDatabasesPath(), LocalDb.dbName); + final f = File('$path-wal'); + return await f.exists() ? await f.length() : 0; +} + +void main() { + setUpAll(() { + sqfliteFfiInit(); + databaseFactory = databaseFactoryFfi; + }); + + tearDown(() async { + DerivationEngine.debugRunning = false; + await LocalDb.close(); + }); + + test('vacuums and truncates the WAL when there is room', () async { + await _useFreshDb('compact_room.db'); + await _bloat(); + final steps = []; + final r = await DerivationEngine().compactStorage( + freeDiskBytes: () async => 1 << 40, + onStep: steps.add, + ); + expect(r.vacuumed, isTrue); + expect(steps, [CompactStep.prune, CompactStep.vacuum]); + expect(r.freedBytes, greaterThan(5 << 20)); + expect(r.afterBytes, await LocalDb.databaseBytesOnDisk()); + expect(await _walBytes(), 0); + expect(await LocalDb.freelistBytes(), 0); + }); + + test('refuses the VACUUM under twice the database free', () async { + await _useFreshDb('compact_tight.db'); + await _bloat(); + final size = await LocalDb.databaseBytesOnDisk(); + final r = await DerivationEngine().compactStorage( + freeDiskBytes: () async => 2 * size - 1, + ); + expect(r.vacuumed, isFalse); + expect(r.neededBytes, greaterThanOrEqualTo(r.freeBytes!)); + expect(await LocalDb.freelistBytes(), greaterThan(0)); + }); + + test('unknown free space counts as not enough', () async { + await _useFreshDb('compact_unknown.db'); + await _bloat(); + final r = await DerivationEngine().compactStorage( + freeDiskBytes: () async => null, + ); + expect(r.vacuumed, isFalse); + expect(r.freeBytes, isNull); + }); + + test('waits for a derive pass in flight instead of running beside it', + () async { + await _useFreshDb('compact_latch.db'); + DerivationEngine.debugRunning = true; + var done = false; + final f = DerivationEngine() + .compactStorage(freeDiskBytes: () async => 1 << 40) + .then((_) => done = true); + await Future.delayed(const Duration(milliseconds: 600)); + expect(done, isFalse); + DerivationEngine.debugRunning = false; + await f; + expect(done, isTrue); + }); +} From d90f010fdb465411a8ba40b3f935a560681d99e6 Mon Sep 17 00:00:00 2001 From: abdulsaheel Date: Wed, 28 Oct 2026 22:34:29 +0530 Subject: [PATCH 04/33] undecodable frames cost ~9 KB each in raw_archive The table kept each frame as hex text under a (device_id, hex) primary key, so the hex was stored twice, at two characters a byte. A WHOOP MG banks ~10,500 such frames a day. I store the bytes once now, keyed by their SHA-1, in raw_archive_frames, and raw_archive is a view that still hands every reader the same hex. Text that isn't canonical hex is kept verbatim. An existing archive is renamed aside on open and moved over in pages from the derive pass's housekeeping, each page inserted and deleted in one transaction; the view covers both tables until it's done. The 1-in-60 thinning stays at the retention edge: thinning at commit would drop full-rate frames that are kept today for the whole retention window. (cherry picked from commit 3f1b420f377d9c95efb92a1224901b8e717517f1) --- lib/coach/coach_db.dart | 3 +- lib/compute/derivation_engine.dart | 11 + lib/data/db.dart | 473 ++++++++++++++++++++++++++--- lib/data/recent_slice.dart | 2 + test/raw_archive_frames_test.dart | 262 ++++++++++++++++ 5 files changed, 700 insertions(+), 51 deletions(-) create mode 100644 test/raw_archive_frames_test.dart diff --git a/lib/coach/coach_db.dart b/lib/coach/coach_db.dart index 7d968dac0..ae0a235a9 100644 --- a/lib/coach/coach_db.dart +++ b/lib/coach/coach_db.dart @@ -97,7 +97,8 @@ class CoachDb { 'workout_suggestions', 'workout_route', 'live_coverage', 'session_score_mask', 'session_sensor', // raw / decoded substrate - 'raw_records', 'raw_archive', 'decoded_onehz', 'decoded_rr', 'samples', + 'raw_records', 'raw_archive', 'raw_archive_frames', 'raw_archive_legacy', + 'decoded_onehz', 'decoded_rr', 'samples', 'events', 'band_events', 'band_battery', 'substrate_archive', 'device_coverage', 'signal_priority', // WHOOP MG ECG. `ecg_reading` is a base table of an allowed view, so the diff --git a/lib/compute/derivation_engine.dart b/lib/compute/derivation_engine.dart index 21cb8f262..97b4b8cbf 100644 --- a/lib/compute/derivation_engine.dart +++ b/lib/compute/derivation_engine.dart @@ -7446,6 +7446,17 @@ class DerivationEngine { /// Off the path to a durable commit, and never inside a migration: `onUpgrade` /// runs under iOS's CPU watchdog (invariant 11). Future _runStorageHousekeeping() async { + try { + // The pre-conversion undecodable archive, moved into the frame store a + // bounded slice at a time (see LocalDb.migrateLegacyRawArchive). A + // WHOOP MG's is ~500 MB of hex, so a background pass takes little. + final moved = await LocalDb.migrateLegacyRawArchive( + budget: Duration(seconds: background ? 3 : 15), + ); + if (moved > 0) _log('moved $moved archived frame(s) to the frame store'); + } catch (e) { + _log('raw archive move skipped (will retry): $e'); + } try { final reencoded = await LocalDb.reencodeLegacyDayResults(); if (reencoded > 0) { diff --git a/lib/data/db.dart b/lib/data/db.dart index 9aa93bf8b..36c5d35b4 100644 --- a/lib/data/db.dart +++ b/lib/data/db.dart @@ -17,6 +17,7 @@ import 'dart:io'; import 'dart:isolate'; import 'dart:typed_data'; +import 'package:crypto/crypto.dart' show sha1; import 'package:flutter/foundation.dart' show visibleForTesting; import 'package:openstrap_protocol/openstrap_protocol.dart' as proto; import 'package:path/path.dart' as p; @@ -572,7 +573,7 @@ class LocalDb { await db.execute('CREATE INDEX idx_samples_ts ON samples(ts)'); await _createEvents(db); await _createBandSignals(db); - await _createRawArchive(db); + await _ensureRawArchiveStore(db); await _createDerived(db); await _createDayResult(db); await _createUserTables(db); @@ -770,7 +771,7 @@ class LocalDb { // the thinning in [thinRawArchiveBefore]) so a future firmware's // records can be re-decoded. Also add `millivolts` to band_battery // for the battery-health series. Both additive. - await _createRawArchive(db); + await _createLegacyRawArchive(db); await _ensureBandBatteryMillivolts(db); } if (oldV < 22) { @@ -882,7 +883,7 @@ class LocalDb { 'ALTER TABLE raw_archive RENAME TO _raw_archive_old', ); await db.execute('DROP INDEX IF EXISTS idx_raw_archive_captured'); - await _createRawArchive(db); + await _createLegacyRawArchive(db); await db.execute( 'INSERT OR IGNORE INTO raw_archive ' '(hex, counter, packet_type, rec_ts, captured_at, reason) ' @@ -891,7 +892,7 @@ class LocalDb { ); await db.execute('DROP TABLE _raw_archive_old'); } else { - await _createRawArchive(db); + await _createLegacyRawArchive(db); } } if (oldV < 33) { @@ -1249,7 +1250,7 @@ class LocalDb { await _createBandSignals(db); await _ensureBandBatteryMillivolts(db); await _ensureBandBatteryChargeUnits(db); - await _createRawArchive(db); + await _ensureRawArchiveStore(db); await _createDerived(db); await _createDayResult(db); await _ensureDayResultImportedColumn(db); @@ -4429,15 +4430,19 @@ class LocalDb { // aside BEFORE the caller writes the batch-ACK that lets the band trim. if (archives != null) { for (final a in archives) { - batch.insert('raw_archive', { - 'device_id': deviceId, - 'counter': a.counter, - 'hex': a.hex, - 'packet_type': a.packetType, - 'rec_ts': a.recTs, - 'captured_at': a.capturedAt, - 'reason': a.reason, - }, conflictAlgorithm: ConflictAlgorithm.ignore); + batch.insert( + rawArchiveTable, + rawArchiveRow( + deviceId: deviceId, + hex: a.hex, + counter: a.counter, + packetType: a.packetType, + recTs: a.recTs, + capturedAt: a.capturedAt, + reason: a.reason, + ), + conflictAlgorithm: ConflictAlgorithm.ignore, + ); if (++ops >= chunkOps) await flushChunk(); } } @@ -7311,38 +7316,296 @@ class LocalDb { if (queued > 0) await batch.commit(noResult: true); } + /// The `(device_id, hex)`-keyed table the ladder rungs below v58 build and + /// re-key ([_rekeyByDeviceIdV51]). Only the ladder creates it now; the open + /// pass converts it into [rawArchiveTable] ([_ensureRawArchiveStore]). + static Future _createLegacyRawArchive(Database db) async { + await db.execute(''' + CREATE TABLE IF NOT EXISTS raw_archive ( + device_id TEXT NOT NULL DEFAULT '$kPrimaryDeviceId', + hex TEXT NOT NULL, + counter INTEGER, + packet_type INTEGER NOT NULL, + rec_ts INTEGER, + captured_at INTEGER NOT NULL, + reason TEXT NOT NULL, + PRIMARY KEY (device_id, hex) + ) + '''); + await db.execute( + 'CREATE INDEX IF NOT EXISTS idx_raw_archive_captured ' + 'ON raw_archive(captured_at DESC)', + ); + } + + /// Where archived frames are stored: [_createRawArchiveFrames]. Readers keep + /// reading the `raw_archive` view, which gives the frame back as `hex`. + static const String rawArchiveTable = 'raw_archive_frames'; + + /// A pre-conversion `raw_archive` table, renamed aside on open and moved + /// into [rawArchiveTable] in pages by [migrateLegacyRawArchive]. + static const String _rawArchiveLegacy = 'raw_archive_legacy'; + + /// The frame as the `raw_archive` view's `hex` column: the stored text for a + /// frame kept verbatim, else the bytes in lowercase hex — exactly the + /// string [rawArchiveRow] was handed. + static const String _rawArchiveHexSql = 'COALESCE(hex, lower(hex(frame)))'; + /// Durable archive for historical records we received but could not decode /// (unknown/unsupported version). Thinned to a stated SAMPLE RATE behind the /// retention edge and otherwise kept forever — the point is that a future /// firmware's records survive until we understand the format, and a sample is /// what a decode needs. See [thinRawArchiveBefore] for the numbers. Keyed by - /// frame `hex` (content identity), like `events`/`band_events` — NOT by - /// `counter`. The strap resets its record counter to ~0 on every reboot, so - /// two DISTINCT undecodable frames from different boots can collide on a - /// reused counter; a `counter`-PK + IGNORE silently DROPPED the second, in - /// the one table whose whole purpose is to never lose a frame. Hashing on the - /// bytes means an identical re-flood (missed-ACK redelivery) still dedups, - /// while genuinely distinct frames both survive a counter collision. `counter` - /// is retained as a plain forensic column. - static Future _createRawArchive(Database db) async { + /// frame CONTENT (`hash`), like `events`/`band_events` — NOT by `counter`. + /// The strap resets its record counter to ~0 on every reboot, so two + /// DISTINCT undecodable frames from different boots can collide on a reused + /// counter; a `counter`-PK + IGNORE silently DROPPED the second, in the one + /// table whose whole purpose is to never lose a frame. Keying on the bytes + /// means an identical re-flood (missed-ACK redelivery) still dedups, while + /// genuinely distinct frames both survive a counter collision. `counter` is + /// retained as a plain forensic column. + /// + /// THE BYTES, NOT THEIR HEX, AND KEYED BY A HASH. The table this replaces + /// stored each frame as hex TEXT under `PRIMARY KEY (device_id, hex)` on a + /// rowid table, so every frame was written twice — once in the row and once + /// in the key's autoindex — at two characters a byte: a 2,128-byte WHOOP MG + /// v20 frame cost ~9 KB, ~10,500 of them a day. Here it is one BLOB under a + /// 20-byte SHA-1 of it, ~4.2 KB a frame (a 2.2 KB row cannot share a 4 KB + /// page with a second one, so each takes a page). `frame` is last so a + /// scan of the small columns (thinning, stats) never walks its overflow + /// pages. A rowid table still: [_mergeFromDbFile] pages restores by rowid. + /// + /// `hex` keeps the text verbatim, with `frame` NULL, when it is not the + /// canonical lowercase hex of some bytes — so nothing is ever rewritten on + /// the way in, whatever a caller handed over. + static Future _createRawArchiveFrames(Database db) async { await db.execute(''' - CREATE TABLE IF NOT EXISTS raw_archive ( + CREATE TABLE IF NOT EXISTS $rawArchiveTable ( device_id TEXT NOT NULL DEFAULT '$kPrimaryDeviceId', - hex TEXT NOT NULL, + hash BLOB NOT NULL, counter INTEGER, packet_type INTEGER NOT NULL, rec_ts INTEGER, captured_at INTEGER NOT NULL, reason TEXT NOT NULL, - PRIMARY KEY (device_id, hex) + hex TEXT, + frame BLOB, + UNIQUE (device_id, hash) ) '''); + // Same name as the old table's index. An older build opening this file + // runs `CREATE INDEX IF NOT EXISTS idx_raw_archive_captured ON + // raw_archive`, which the existing name turns into a no-op instead of an + // attempt to index a view (which fails the open). await db.execute( 'CREATE INDEX IF NOT EXISTS idx_raw_archive_captured ' - 'ON raw_archive(captured_at DESC)', + 'ON $rawArchiveTable(captured_at DESC)', + ); + // Every reader filters on reason (the re-drive, the vendor inputs, the + // thinning) and most on device too. + await db.execute( + 'CREATE INDEX IF NOT EXISTS idx_raw_archive_reason ' + 'ON $rawArchiveTable(reason, device_id)', ); } + /// `raw_archive` as every reader has always seen it, over [rawArchiveTable] + /// (and the legacy table while [migrateLegacyRawArchive] is still moving + /// it). The INSTEAD OF triggers keep plain inserts and deletes against the + /// old name working — an older build after a rollback, a test fixture — + /// with the text itself as the key: correct, just without the size saving. + /// Everything in this build writes through [rawArchiveRow] instead. + static Future _ensureRawArchiveView(Database db) async { + final legacy = await _objectType(db, _rawArchiveLegacy) == 'table'; + await db.execute('DROP VIEW IF EXISTS raw_archive'); + await db.execute( + 'CREATE VIEW raw_archive AS ' + 'SELECT device_id, $_rawArchiveHexSql AS hex, counter, packet_type, ' + 'rec_ts, captured_at, reason FROM $rawArchiveTable' + '${legacy ? ' UNION ALL SELECT device_id, hex, counter, packet_type, ' + 'rec_ts, captured_at, reason FROM $_rawArchiveLegacy' : ''}', + ); + await db.execute( + 'CREATE TRIGGER IF NOT EXISTS raw_archive_insert ' + 'INSTEAD OF INSERT ON raw_archive BEGIN ' + 'INSERT OR IGNORE INTO $rawArchiveTable ' + '(device_id, hash, counter, packet_type, rec_ts, captured_at, reason, hex) ' + "VALUES (COALESCE(NEW.device_id, '$kPrimaryDeviceId'), " + "CAST('text:' || NEW.hex AS BLOB), NEW.counter, NEW.packet_type, " + 'NEW.rec_ts, NEW.captured_at, NEW.reason, NEW.hex); END', + ); + await db.execute( + 'CREATE TRIGGER IF NOT EXISTS raw_archive_delete ' + 'INSTEAD OF DELETE ON raw_archive BEGIN ' + 'DELETE FROM $rawArchiveTable WHERE device_id = OLD.device_id ' + 'AND $_rawArchiveHexSql = OLD.hex; ' + '${legacy ? 'DELETE FROM $_rawArchiveLegacy WHERE device_id = ' + 'OLD.device_id AND hex = OLD.hex; ' : ''}END', + ); + } + + /// `'table'`, `'view'`, … for [name] in sqlite_master, or null. + static Future _objectType(DatabaseExecutor db, String name) async { + final rows = await db.rawQuery( + 'SELECT type FROM sqlite_master WHERE name = ? LIMIT 1', + [name], + ); + return rows.isEmpty ? null : rows.first['type'] as String?; + } + + /// The archive store, on every open. A database still holding the old + /// `raw_archive` TABLE has it renamed aside — O(1), no row touched, so safe + /// under the launch watchdog (invariant 11) — and the rows move later, in + /// pages, off the open path ([migrateLegacyRawArchive]). The view unions + /// both until then, so no reader ever sees a frame missing. + static Future _ensureRawArchiveStore(Database db) async { + if (await _objectType(db, 'raw_archive') == 'table') { + if (await _objectType(db, _rawArchiveLegacy) == 'table') { + // Only reachable if a file was hand-assembled with both; fold the + // newer table into the aside one rather than lose either. + await db.execute( + 'INSERT OR IGNORE INTO $_rawArchiveLegacy ' + '(device_id, hex, counter, packet_type, rec_ts, captured_at, reason) ' + 'SELECT device_id, hex, counter, packet_type, rec_ts, captured_at, ' + 'reason FROM raw_archive', + ); + await db.execute('DROP TABLE raw_archive'); + } else { + await db.execute('ALTER TABLE raw_archive RENAME TO $_rawArchiveLegacy'); + } + // Frees the name for [rawArchiveTable]'s index (see there). + await db.execute('DROP INDEX IF EXISTS idx_raw_archive_captured'); + } + await _createRawArchiveFrames(db); + if (await _objectType(db, 'raw_archive') != 'view') { + await _ensureRawArchiveView(db); + } else if ((await _objectType(db, _rawArchiveLegacy) == 'table') != + (await db.rawQuery( + "SELECT 1 FROM sqlite_master WHERE type = 'view' " + "AND name = 'raw_archive' AND sql LIKE '%$_rawArchiveLegacy%'", + )) + .isNotEmpty) { + // The view and the legacy table disagree (a restore, a crash between + // the drop and the rebuild): rebuild the view and its triggers. + await db.execute('DROP TRIGGER IF EXISTS raw_archive_insert'); + await db.execute('DROP TRIGGER IF EXISTS raw_archive_delete'); + await _ensureRawArchiveView(db); + } + } + + /// [hex] as bytes when it is canonical lowercase hex (even length, 0-9a-f), + /// else null — so `lower(hex(frame))` gives back exactly [hex]. + static Uint8List? _canonicalHexBytes(String hex) { + final n = hex.length; + if (n == 0 || n.isOdd) return null; + final out = Uint8List(n >> 1); + int nibble(int c) { + if (c >= 0x30 && c <= 0x39) return c - 0x30; + if (c >= 0x61 && c <= 0x66) return c - 0x61 + 10; + return -1; + } + + for (var i = 0; i < n; i += 2) { + final hi = nibble(hex.codeUnitAt(i)), lo = nibble(hex.codeUnitAt(i + 1)); + if (hi < 0 || lo < 0) return null; + out[i >> 1] = (hi << 4) | lo; + } + return out; + } + + /// The [rawArchiveTable] row for one archived frame. Every writer in this + /// build goes through here (the drain commit, the fallback path, the + /// legacy move, a restore), so the key is always the same function of the + /// frame: SHA-1 of the bytes, or of the text when it is kept verbatim. + static Map rawArchiveRow({ + required String deviceId, + required String hex, + required int? counter, + required int packetType, + required int? recTs, + required int capturedAt, + required String reason, + }) { + final bytes = _canonicalHexBytes(hex); + return { + 'device_id': deviceId, + 'hash': Uint8List.fromList( + bytes != null + ? sha1.convert(bytes).bytes + : sha1.convert(utf8.encode('text:$hex')).bytes, + ), + 'counter': counter, + 'packet_type': packetType, + 'rec_ts': recTs, + 'captured_at': capturedAt, + 'reason': reason, + 'hex': bytes == null ? hex : null, + 'frame': bytes, + }; + } + + /// Move the renamed-aside legacy archive into [rawArchiveTable], 500 rows a + /// transaction: each page is inserted and deleted from the legacy table in + /// the same commit, so a kill between pages loses nothing and repeats + /// nothing. Once it is empty the legacy table is dropped and the view + /// rebuilt over the new table alone. Returns rows moved; stops after the + /// first page past [budget] and resumes on the next call. + static Future migrateLegacyRawArchive({Duration? budget}) async { + final db = await instance; + if (await _objectType(db, _rawArchiveLegacy) != 'table') return 0; + final clock = Stopwatch()..start(); + var moved = 0; + while (true) { + final page = await db.rawQuery( + 'SELECT rowid AS _r, device_id, hex, counter, packet_type, rec_ts, ' + 'captured_at, reason FROM $_rawArchiveLegacy ORDER BY rowid LIMIT 500', + ); + if (page.isEmpty) { + await db.transaction((txn) async { + if ((await txn.rawQuery( + 'SELECT 1 FROM $_rawArchiveLegacy LIMIT 1')) + .isNotEmpty) { + return; // a row landed since; the next call moves it + } + await txn.execute('DROP TRIGGER IF EXISTS raw_archive_insert'); + await txn.execute('DROP TRIGGER IF EXISTS raw_archive_delete'); + await txn.execute('DROP VIEW IF EXISTS raw_archive'); + await txn.execute('DROP TABLE $_rawArchiveLegacy'); + }); + if (await _objectType(db, 'raw_archive') != 'view') { + await _ensureRawArchiveView(db); + } + return moved; + } + final hi = (page.last['_r'] as num).toInt(); + await db.transaction((txn) async { + final batch = txn.batch(); + for (final r in page) { + batch.insert( + rawArchiveTable, + rawArchiveRow( + deviceId: r['device_id'] as String? ?? kPrimaryDeviceId, + hex: r['hex'] as String, + counter: (r['counter'] as num?)?.toInt(), + packetType: (r['packet_type'] as num?)?.toInt() ?? 0, + recTs: (r['rec_ts'] as num?)?.toInt(), + capturedAt: (r['captured_at'] as num?)?.toInt() ?? 0, + reason: r['reason'] as String? ?? 'unknown', + ), + conflictAlgorithm: ConflictAlgorithm.ignore, + ); + } + batch.rawDelete( + 'DELETE FROM $_rawArchiveLegacy WHERE rowid <= ?', + [hi], + ); + await batch.commit(noResult: true); + }); + moved += page.length; + if (budget != null && clock.elapsed > budget) return moved; + } + } + /// One compressed, verified bucket of pruned decoded substrate per /// (device_id, utc_day) — see substrate_archive.dart. `utc_day` is /// `rec_ts ~/ 86400`, an absolute storage partition and NEVER a day label. @@ -7670,15 +7933,19 @@ class LocalDb { String deviceId = kPrimaryDeviceId, }) async { final db = await instance; - await db.insert('raw_archive', { - 'device_id': deviceId, - 'counter': a.counter, - 'hex': a.hex, - 'packet_type': a.packetType, - 'rec_ts': a.recTs, - 'captured_at': a.capturedAt, - 'reason': a.reason, - }, conflictAlgorithm: ConflictAlgorithm.ignore); + await db.insert( + rawArchiveTable, + rawArchiveRow( + deviceId: deviceId, + hex: a.hex, + counter: a.counter, + packetType: a.packetType, + recTs: a.recTs, + capturedAt: a.capturedAt, + reason: a.reason, + ), + conflictAlgorithm: ConflictAlgorithm.ignore, + ); } /// The `raw_archive.reason` values [redriveArchivedRecords] will retry. @@ -7754,6 +8021,9 @@ class LocalDb { /// `compute_freshness` key for [redriveArchiveOnce]. static const String kArchiveRedriveKey = 'archive_redrive_v12_v1'; + /// Marks a [_redrive] resume position that is a [rawArchiveTable] rowid. + static const String _redriveRowidCursor = '\u0000rowid'; + /// One-shot replay of every [redrivableArchiveReasons] row, off the launch /// path (called from a heavy derive). Marked done only after it finishes, /// so an interrupted pass repeats; the replay skips seconds already decoded, @@ -7772,6 +8042,12 @@ class LocalDb { if (v is Map) state = v.cast(); } if (state['done'] == true) return 0; + // Wait for the legacy archive to finish moving: the replay reads the new + // store only, and marking it done over a half-moved archive would skip + // the rest for good. + if (await _objectType(await instance, _rawArchiveLegacy) == 'table') { + return 0; + } final row = await computeFreshness(kRedriveExcludedKey); final excluded = row == null ? const <(int, int)>[] : _parseWindows(row['payload_json']); @@ -7848,12 +8124,23 @@ class LocalDb { // the archive yet (or is mid-ladder) must no-op rather than throw — a throw // in here rolls the WHOLE upgrade back and quarantines the database. for (final t in const ['raw_archive', 'decoded_onehz']) { + // `raw_archive` is a view over [rawArchiveTable] once converted. final present = await db.rawQuery( - "SELECT 1 FROM sqlite_master WHERE type='table' AND name=?", + "SELECT 1 FROM sqlite_master WHERE type IN ('table', 'view') " + 'AND name=?', [t], ); if (present.isEmpty) return (recovered: 0, resume: null); } + // The converted store pages on its own rowid (the hex is computed, so + // paging on it would sort the archive every page). Its cursor travels in + // the same (String, String) slot, marked by [_redriveRowidCursor]; a + // position from the old hex-ordered pass restarts at the top, which the + // replay's skip-what-is-decoded rule makes safe. + final framesLayout = await _objectType(db, 'raw_archive') == 'view'; + var afterRowid = framesLayout && startDevice == _redriveRowidCursor + ? int.tryParse(startHex) ?? 0 + : 0; // SELF-DETECTING, because this runs from BOTH sides of the v47 rung: from // the oldV<44 ladder step, where `decoded_onehz` is still keyed by rec_ts // alone and naming `device_id` would throw inside onUpgrade (quarantining @@ -7874,11 +8161,19 @@ class LocalDb { // Post-v51, `hex` alone is no longer unique // (two devices can share a byte-identical undecodable frame), so a plain // `hex > ?` cursor would silently skip the second device's row. - var afterHex = startHex; - var afterDevice = startDevice; + var afterHex = framesLayout ? '$afterRowid' : startHex; + var afterDevice = framesLayout ? _redriveRowidCursor : startDevice; var recovered = 0; while (true) { - final rows = preArchiveDeviceKey + final rows = framesLayout + ? await db.rawQuery( + 'SELECT rowid AS _r, device_id, $_rawArchiveHexSql AS hex, ' + 'counter, packet_type, rec_ts, captured_at ' + 'FROM $rawArchiveTable WHERE reason IN ($marks) AND rowid > ? ' + 'ORDER BY rowid ASC LIMIT 500', + [...reasons, afterRowid], + ) + : preArchiveDeviceKey ? await db.rawQuery( 'SELECT hex, counter, packet_type, rec_ts, captured_at ' 'FROM raw_archive WHERE reason IN ($marks) AND hex > ? ' @@ -7895,8 +8190,14 @@ class LocalDb { ); if (rows.isEmpty) return (recovered: recovered, resume: null); final pageHex = afterHex, pageDevice = afterDevice; - afterHex = rows.last['hex'] as String; - afterDevice = preArchiveDeviceKey ? '' : rows.last['device_id'] as String; + if (framesLayout) { + afterRowid = (rows.last['_r'] as num).toInt(); + afterHex = '$afterRowid'; + } else { + afterHex = rows.last['hex'] as String; + afterDevice = + preArchiveDeviceKey ? '' : rows.last['device_id'] as String; + } // Decode first, keyed by (device, second) each record claims. Two // archived frames for the SAME device and second collapse here rather @@ -8076,14 +8377,24 @@ class LocalDb { DatabaseExecutor txn, int beforeMs, ) async { - return txn.delete( - 'raw_archive', - // `counter` is NOT NULL in practice but the column is nullable, and - // `NULL % 60` is NULL — so a counter-less row would never match and would - // be kept. Keeping an unsampleable frame is the safe direction. - where: 'reason = ? AND captured_at < ? AND counter % ? != 0', - whereArgs: [_thinnedArchiveReason, beforeMs, rawArchiveKeepEvery], - ); + var deleted = 0; + // The store and, until [migrateLegacyRawArchive] has emptied it, the + // renamed-aside legacy table: the same rule over both. + for (final t in [ + rawArchiveTable, + if (await _objectType(txn, _rawArchiveLegacy) == 'table') + _rawArchiveLegacy, + ]) { + deleted += await txn.delete( + t, + // `counter` is NOT NULL in practice but the column is nullable, and + // `NULL % 60` is NULL — so a counter-less row would never match and + // would be kept. Keeping an unsampleable frame is the safe direction. + where: 'reason = ? AND captured_at < ? AND counter % ? != 0', + whereArgs: [_thinnedArchiveReason, beforeMs, rawArchiveKeepEvery], + ); + } + return deleted; } /// How many undecodable records we've archived, and by reason — for the sync @@ -10431,6 +10742,59 @@ class LocalDb { 'strength_set': (['session_id'], null), }; + /// `raw_archive` out of a backup, whichever layout wrote it: the frame + /// store of this build, the hex-keyed table of an older one, or both (a + /// file saved mid-move). Every row is rebuilt through [rawArchiveRow], so a + /// frame already here, or in the backup twice, lands once. Paged on rowid, + /// like every other table the merge reads. Returns rows offered. + static Future _mergeRawArchive(Database src, Database db) async { + var copied = 0; + Future pageFrom(String table, String hexSql) async { + if (await _objectType(src, table) != 'table') return; + // Pre-v51 files have no device_id: every row is the primary band's. + final hasDevice = (await src.rawQuery('PRAGMA table_info($table)')) + .any((c) => c['name'] == 'device_id'); + final deviceSql = + hasDevice ? 'device_id' : "'$kPrimaryDeviceId' AS device_id"; + var after = 0; + while (true) { + final page = await src.rawQuery( + 'SELECT rowid AS _r, $deviceSql, $hexSql AS hex, counter, ' + 'packet_type, rec_ts, captured_at, reason FROM $table ' + 'WHERE rowid > ? ORDER BY rowid LIMIT 500', + [after], + ); + if (page.isEmpty) return; + after = (page.last['_r'] as num).toInt(); + final batch = db.batch(); + for (final r in page) { + final hex = r['hex']; + if (hex is! String) continue; + batch.insert( + rawArchiveTable, + rawArchiveRow( + deviceId: r['device_id'] as String? ?? kPrimaryDeviceId, + hex: hex, + counter: (r['counter'] as num?)?.toInt(), + packetType: (r['packet_type'] as num?)?.toInt() ?? 0, + recTs: (r['rec_ts'] as num?)?.toInt(), + capturedAt: (r['captured_at'] as num?)?.toInt() ?? 0, + reason: r['reason'] as String? ?? 'unknown', + ), + conflictAlgorithm: ConflictAlgorithm.ignore, + ); + copied++; + } + await batch.commit(noResult: true); + } + } + + await pageFrom('raw_archive', 'hex'); // a table only in older files + await pageFrom(rawArchiveTable, _rawArchiveHexSql); + await pageFrom(_rawArchiveLegacy, 'hex'); + return copied; + } + @visibleForTesting static List get restoreTablesForTest => _restoreTables; @@ -10511,6 +10875,15 @@ class LocalDb { : (srcEdge == null || localEdge > srcEdge ? localEdge : srcEdge); } for (final t in (only ?? tables)) { + if (t == 'raw_archive') { + try { + counts[t] = await _mergeRawArchive(src, db); + } catch (_) { + if (!tolerant) rethrow; + counts[t] = 0; + } + continue; + } try { // PAGED SOURCE READ — never `SELECT *` a whole table. // diff --git a/lib/data/recent_slice.dart b/lib/data/recent_slice.dart index 1440e13d2..64ed6d41f 100644 --- a/lib/data/recent_slice.dart +++ b/lib/data/recent_slice.dart @@ -46,6 +46,8 @@ const Map kSliceTables = { 'decoded_rr': (SliceKeep.highRate, 'rec_ts', 'beat-to-beat intervals'), 'samples': (SliceKeep.highRate, 'ts', 'per-second heart rate'), 'raw_archive': (SliceKeep.highRate, 'captured_at', 'records the app could not decode, by when they arrived'), + 'raw_archive_frames': (SliceKeep.highRate, 'captured_at', 'records the app could not decode, by when they arrived'), + 'raw_archive_legacy': (SliceKeep.highRate, 'captured_at', 'the same records in the older layout, until they are moved'), 'events': (SliceKeep.recent, 'ts', 'strap event stream'), 'band_events': (SliceKeep.recent, 'ts', 'decoded strap events'), 'band_battery': (SliceKeep.recent, 'ts', 'battery readings every few minutes'), diff --git a/test/raw_archive_frames_test.dart b/test/raw_archive_frames_test.dart new file mode 100644 index 000000000..350c918b0 --- /dev/null +++ b/test/raw_archive_frames_test.dart @@ -0,0 +1,262 @@ +// raw_archive stores each undecodable frame once, as bytes, keyed by a hash of +// them. The old table kept it as hex TEXT under PRIMARY KEY (device_id, hex) +// on a rowid table, so the hex was stored twice. This pins: readers still see +// the exact hex they always did, dedup still holds, text that is not +// canonical hex survives verbatim, and an existing archive moves across in +// pages without losing or duplicating a frame. + +import 'dart:io'; +import 'dart:math' as math; + +import 'package:flutter_test/flutter_test.dart'; +import 'package:path/path.dart' as p; +import 'package:shared_preferences/shared_preferences.dart'; +import 'package:sqflite_common_ffi/sqflite_ffi.dart'; +import 'package:openstrap_edge/data/db.dart'; +import 'package:openstrap_edge/data/models.dart'; + +const _frameBytes = 2128; // a WHOOP MG v20 frame + +String _frameHex(int seed) { + final r = math.Random(seed); + final b = StringBuffer(); + for (var i = 0; i < _frameBytes; i++) { + b.write(r.nextInt(256).toRadixString(16).padLeft(2, '0')); + } + return b.toString(); +} + +ArchiveRecord _rec(int counter, String hex, {String reason = 'undecodable_rec_v20'}) => + ArchiveRecord( + counter: counter, + hex: hex, + packetType: 0x2F, + capturedAt: 1750000000000 + counter * 1000, + reason: reason, + ); + +Future _path(String name) async => + p.join(await databaseFactory.getDatabasesPath(), name); + +Future _fresh(String name) async { + await LocalDb.close(); + LocalDb.dbName = name; + await databaseFactory.deleteDatabase(await _path(name)); +} + +Future>> _viewRows() async => + (await LocalDb.instance).rawQuery( + 'SELECT device_id, hex, counter, packet_type, rec_ts, captured_at, ' + 'reason FROM raw_archive ORDER BY device_id, hex', + ); + +Future _bytes(Database db, List names) async { + final marks = List.filled(names.length, '?').join(','); + final r = await db.rawQuery( + 'SELECT SUM(pgsize) AS b FROM dbstat WHERE name IN ($marks)', + names, + ); + return (r.first['b'] as num?)?.toInt() ?? 0; +} + +/// A database as the previous build left it: user_version 58 and the +/// hex-keyed table, with [rows] in it. +Future _legacyDb(String name, List> rows) async { + await _fresh(name); + final db = await databaseFactory.openDatabase(await _path(name), + options: OpenDatabaseOptions( + version: LocalDb.schemaVersion, + onCreate: (db, _) async { + await db.execute(''' + CREATE TABLE raw_archive ( + device_id TEXT NOT NULL DEFAULT '', + hex TEXT NOT NULL, + counter INTEGER, + packet_type INTEGER NOT NULL, + rec_ts INTEGER, + captured_at INTEGER NOT NULL, + reason TEXT NOT NULL, + PRIMARY KEY (device_id, hex) + ) + '''); + await db.execute('CREATE INDEX idx_raw_archive_captured ' + 'ON raw_archive(captured_at DESC)'); + }, + )); + final batch = db.batch(); + for (final r in rows) { + batch.insert('raw_archive', r); + } + await batch.commit(noResult: true); + await db.close(); +} + +Map _legacyRow(String device, String hex, int counter) => { + 'device_id': device, + 'hex': hex, + 'counter': counter, + 'packet_type': 0x2F, + 'rec_ts': null, + 'captured_at': 1750000000000 + counter, + 'reason': 'undecodable_rec_v20', + }; + +void main() { + setUpAll(() { + TestWidgetsFlutterBinding.ensureInitialized(); + SharedPreferences.setMockInitialValues({}); + sqfliteFfiInit(); + databaseFactory = databaseFactoryFfi; + }); + + tearDown(() async => LocalDb.close()); + + test('a frame is stored once, as bytes, and reads back as its hex', () async { + await _fresh('raw_frames_store.db'); + final a = _frameHex(1), b = _frameHex(2); + await LocalDb.commitSyncBatch(const [], const [], + archives: [_rec(1, a), _rec(2, b), _rec(3, a)]); + await LocalDb.archiveRawRecord(_rec(4, b)); // a re-flood, other path + final rows = await _viewRows(); + expect([for (final r in rows) r['hex']], unorderedEquals([a, b])); + final db = await LocalDb.instance; + final stored = await db.query(LocalDb.rawArchiveTable); + expect(stored, hasLength(2)); + for (final r in stored) { + expect(r['hex'], isNull); + expect((r['frame'] as List).length, _frameBytes); + expect((r['hash'] as List).length, 20); + } + }); + + test('text that is not canonical hex is kept verbatim', () async { + await _fresh('raw_frames_text.db'); + const odd = ['ABCDEF', 'abc', 'not hex at all', 'Deadbeef']; + await LocalDb.commitSyncBatch(const [], const [], + archives: [for (var i = 0; i < odd.length; i++) _rec(i, odd[i])]); + expect([for (final r in await _viewRows()) r['hex']], + unorderedEquals(odd)); + }); + + test('two devices keep their own copy of a byte-identical frame', () async { + await _fresh('raw_frames_devices.db'); + final hex = _frameHex(9); + await LocalDb.archiveRawRecord(_rec(1, hex)); + await LocalDb.archiveRawRecord(_rec(1, hex), deviceId: 'ring-1'); + expect(await _viewRows(), hasLength(2)); + }); + + test('plain inserts and deletes on the old name still work', () async { + await _fresh('raw_frames_compat.db'); + final db = await LocalDb.instance; + await db.insert('raw_archive', _legacyRow('', 'beef', 7)); + await db.insert('raw_archive', _legacyRow('', 'beef', 7), + conflictAlgorithm: ConflictAlgorithm.ignore); + expect(await _viewRows(), hasLength(1)); + await db.delete('raw_archive', where: 'hex = ?', whereArgs: ['beef']); + expect(await _viewRows(), isEmpty); + }); + + test('an existing archive moves across in pages, losing nothing', () async { + final rows = [ + for (var i = 0; i < 1203; i++) + _legacyRow(i % 7 == 0 ? 'ring-1' : '', _frameHex(100 + i), i), + _legacyRow('', 'Not-Hex', 5000), + ]; + await _legacyDb('raw_frames_migrate.db', rows); + final expected = [ + for (final r in rows) + '${r['device_id']}|${r['hex']}|${r['counter']}|${r['captured_at']}', + ]..sort(); + Future> seen() async => [ + for (final r in await _viewRows()) + '${r['device_id']}|${r['hex']}|${r['counter']}|${r['captured_at']}', + ]..sort(); + + // Opening renames the table aside; the view already shows every row. + expect(await seen(), expected); + final db = await LocalDb.instance; + // The table and its key's autoindex, whatever the rename called it. + final legacyIndex = [ + for (final r in await db.rawQuery("SELECT name FROM sqlite_master " + "WHERE type = 'index' AND tbl_name = 'raw_archive_legacy'")) + r['name'] as String, + ]; + expect(legacyIndex, isNotEmpty); + final legacyBytes = + await _bytes(db, ['raw_archive_legacy', ...legacyIndex]); + + // A frame arriving mid-move, already in the legacy table: one copy after. + await LocalDb.archiveRawRecord(_rec(1, rows[1]['hex'] as String)); + + // One page per call, a frame lands between calls, and the pass resumes. + var calls = 0; + while (await LocalDb.migrateLegacyRawArchive(budget: Duration.zero) > 0) { + calls++; + if (calls == 1) { + await LocalDb.archiveRawRecord(_rec(9999, _frameHex(9999))); + expected.add('|${_frameHex(9999)}|9999|${1750000000000 + 9999000}'); + expected.sort(); + } + } + expect(calls, greaterThanOrEqualTo(3)); + expect( + await db.rawQuery("SELECT 1 FROM sqlite_master WHERE name = 'raw_archive_legacy'"), + isEmpty, + ); + final view = await db.rawQuery( + "SELECT sql FROM sqlite_master WHERE name = 'raw_archive'"); + expect('${view.single['sql']}', isNot(contains('legacy'))); + final after = await seen(); + // The mid-move duplicate collapsed; counter differs, so compare keys. + expect(after.map((s) => s.split('|').take(2).join('|')).toList(), + expected.map((s) => s.split('|').take(2).join('|')).toList()); + expect(after.length, expected.length); + + final newBytes = await _bytes(db, [ + LocalDb.rawArchiveTable, + for (final r in await db.rawQuery("SELECT name FROM sqlite_master " + "WHERE type = 'index' AND tbl_name = ?", [LocalDb.rawArchiveTable])) + r['name'] as String, + ]); + // ignore: avoid_print + print('raw_archive: ${rows.length} frames, legacy ${legacyBytes >> 10} KiB' + ' -> ${newBytes >> 10} KiB ' + '(${(legacyBytes / rows.length).round()} -> ' + '${(newBytes / after.length).round()} bytes/frame)'); + // A 2.2 KB row cannot share a 4 KB page with another, so a frame costs a + // page; the hex row and its key copy cost two and change. + expect(newBytes, lessThan(legacyBytes * 0.55)); + + // Reopening a converted database changes nothing. + await LocalDb.close(); + expect(await seen(), after); + }); + + test('a backup of the new layout restores every frame', () async { + await _fresh('raw_frames_backup_src.db'); + final frames = [_frameHex(1), 'Not-Hex', _frameHex(2)]; + await LocalDb.commitSyncBatch(const [], const [], + archives: [for (var i = 0; i < frames.length; i++) _rec(i, frames[i])]); + final before = await _viewRows(); + final copy = await LocalDb.exportCopy( + into: Directory.systemTemp.createTempSync('raw_frames')); + await _fresh('raw_frames_backup_dst.db'); + await LocalDb.importFromDbFile(copy); + expect(await _viewRows(), before); + }); + + test('thinning keeps the same 1-in-60 sample in both tables', () async { + final rows = [ + for (var i = 0; i < 240; i++) _legacyRow('', _frameHex(i), i), + ]; + await _legacyDb('raw_frames_thin.db', rows); + await LocalDb.instance; + // Half moved, half still aside, when the retention pass thins. + await LocalDb.migrateLegacyRawArchive(budget: Duration.zero); + final deleted = await LocalDb.thinRawArchiveBefore(1750000000000 + 1000); + expect(deleted, 240 - 4); // counters 0, 60, 120, 180 stay + expect([for (final r in await _viewRows()) r['counter']]..sort(), + [0, 60, 120, 180]); + }); +} From 89a982372593d25d6a9c31730ed48928ddc614e0 Mon Sep 17 00:00:00 2001 From: abdulsaheel Date: Wed, 28 Oct 2026 23:12:06 +0530 Subject: [PATCH 05/33] freeze golden compared the new day_result.imported flag as output It's an index over the payload's own marker, which the golden already pins by hash. (cherry picked from commit 32ada653bf3e2d933a3e0df52dcd67fa1abaa3dd) --- test/whoop_freeze_golden_test.dart | 3 +++ 1 file changed, 3 insertions(+) diff --git a/test/whoop_freeze_golden_test.dart b/test/whoop_freeze_golden_test.dart index cfc95b335..3dc238007 100644 --- a/test/whoop_freeze_golden_test.dart +++ b/test/whoop_freeze_golden_test.dart @@ -198,6 +198,9 @@ Future> _snapshot(Profile profile) async { String sha(Object? stored) => sha256.convert(utf8.encode(stored as String)).toString(); for (final r in dayResults) { + // An index over the payload's own "imported" marker (pinned below by the + // payload hash), kept by trigger — storage, not output. + r.remove('imported'); r['payload_sha256'] = sha(r['payload_json']); r['payload_json'] = SeriesCodec.decodePayloadJson(r['payload_json']); r['window_json'] = jsonDecode(r['window_json'] as String); From 742c42f65907eea07108d4d32b1bfe84f974d0a5 Mon Sep 17 00:00:00 2001 From: abdulsaheel Date: Wed, 28 Oct 2026 23:12:06 +0530 Subject: [PATCH 06/33] every heavy pass re-derived each pending day even with no new data A heavy pass re-ran the full derive for every unfinalized day, 8-14 s each, so a pass with nothing new took close to a minute. I fingerprint what a derive of a day reads (its substrate window, the user's corrections and review state, sessions, its own and its neighbours' rows, the baseline history before it, the profile, the data edge and the algorithm version) and leave a day alone when the fingerprint matches one stored after a derive that changed nothing it reads. Today, forced passes and edited days always derive, and so does a day with a session still running. A rescan triggered only by the quiet level now re-derives just the days priced on a level that moved, instead of every day in the 21-day window, finalized ones included. (cherry picked from commit fd0d53f86076244aa4a91bb119e911c2b0a19a64) --- lib/compute/derivation_engine.dart | 246 ++++++++++++++++++++++- lib/data/db.dart | 268 ++++++++++++++++++++++++++ lib/data/recent_slice.dart | 1 + test/derive_unchanged_skip_test.dart | 181 +++++++++++++++++ test/rescan_quiet_narrowing_test.dart | 44 +++++ 5 files changed, 733 insertions(+), 7 deletions(-) create mode 100644 test/derive_unchanged_skip_test.dart create mode 100644 test/rescan_quiet_narrowing_test.dart diff --git a/lib/compute/derivation_engine.dart b/lib/compute/derivation_engine.dart index 97b4b8cbf..6c3c1fd15 100644 --- a/lib/compute/derivation_engine.dart +++ b/lib/compute/derivation_engine.dart @@ -2803,6 +2803,47 @@ class _BaselineHistoryCache { return [for (var i = from; i < samples.length; i++) samples[i].value]; } + /// The dated quiet levels [_quietFingerprint] covers, newest included, as + /// `[date, value]` pairs: what [rescanRecent] compares against the list it + /// stored last time to find the days whose strain window moved. + List> quietWindowEntries() { + final series = _series['quiet_hrr'] ?? const <_DatedValue>[]; + const span = _rescanWindowDays + _baselineWindowDays + 1; + final from = series.length <= span ? 0 : series.length - span; + return [ + for (var i = from; i < series.length; i++) + [series[i].date, series[i].value], + ]; + } + + /// Every history value a derive of [date] can read, hashed: each series' + /// entries strictly before [date] with their method family (the windows + /// `valuesBefore`/`maxBefore` serve), plus every `dyn_p90` entry, whose + /// dates the frozen movement floor reads whole. Days after [date] are left + /// out, so today's values moving through the day do not unsettle the days + /// before it. + String inputsBefore(String date) { + final b = StringBuffer(); + for (final key in _series.keys.toList()..sort()) { + final series = _series[key]!; + final fams = _families[key]; + final end = + key == 'dyn_p90' ? series.length : _beforeIndex(series, date); + b.write('|$key:'); + for (var i = 0; i < end; i++) { + final v = series[i]; + b + ..write(v.date) + ..write('=') + ..write(v.value) + ..write('@') + ..write(fams?[v.date] ?? '') + ..write(','); + } + } + return sha1.convert(utf8.encode(b.toString())).toString(); + } + /// Strain's quiet-waking level is a baseline too, and [rescanRecent] has to /// re-price the recent days when ANY of their windows moves — a count and a /// median of the whole series do not see that: each day is priced on the @@ -3176,6 +3217,80 @@ class DerivationEngine { /// Any later pass on this shared engine (a drain's light pass) resets them /// too, so a caller that must attribute the outcome to its own run passes /// [run]'s `onOutcome` instead of reading these after other awaits. + /// + /// Days [run] left alone in its last pass because nothing they read had + /// moved (see [_dayInputFingerprint]). + int lastRunUnchangedDays = 0; + + /// Test seam: derive every targeted day, as before the unchanged-day skip. + @visibleForTesting + static bool debugDeriveEveryDay = false; + + /// Test seam: the version [_dayInputFingerprint] keys on, to stand in for a + /// [kAlgoVersion] bump. + @visibleForTesting + static int? debugFingerprintAlgoVersion; + + /// Everything a derive of [dayId] reads, hashed, or null when it must + /// derive regardless (today: its result follows the wall clock). + /// + /// [run] re-derives every unfinalized day on every pass — on a soak with + /// no new data, four to six days at 8-14 s each, the whole pass for + /// nothing. A day is skipped when this matches the value stored by a + /// derive of it that was a FIXED POINT: the fingerprint read just before + /// that derive equalled the one read just after, so the derive rewrote + /// nothing it reads (its own row, its neighbours', the review state, the + /// sessions it scores). Deterministic compute over unchanged inputs then + /// gives the same result again, so not running it is the same as running + /// it. A day still converging (tonight's sleep landing on the next pass, a + /// review row it just wrote) derives again until it is one. + /// + /// Mixed in: the algorithm version, the profile, the data edge (stored in + /// the payload, so new data anywhere is a new result), the timezone the + /// day's window is cut in, the session-sensor globals, the baseline history + /// before the day, and [LocalDb.dayInputSignature] — the rows themselves. + Future _dayInputFingerprint( + String dayId, + Profile profile, + int dataNowSec, + _BaselineHistoryCache history, + ) async { + if (dayId == todayLabel()) return null; + final dayStart = _localDayLabelToSec(dayId); + final dayEnd = _localNextDayLabelToSec(dayId); + final wearable = await activeWearable(); + final rows = await LocalDb.dayInputSignature( + dayId: dayId, + prevDayId: localDateLabel(dayStart - 43200), + nextDayId: localDateLabel(dayEnd + 43200), + fromSec: dayStart - kNocturnalSearchLookbackSec - 86400, + toSec: dayEnd + napBoundaryBufferSec + 86400, + wearableDeviceId: wearable?.$1, + ); + if (rows == null) return null; + final now = DateTime.now(); + final finalized = (await LocalDb.finalizedDayIds(kAlgoVersion)).toList() + ..sort(); + final head = [ + 'v${debugFingerprintAlgoVersion ?? kAlgoVersion}', + dataNowSec, + // The edges a day is aged against (band and wearable; see + // [_agingEdgeFor]), from the pass's own memo. + await _agingEdgeFor(null, dataNowSec, history), + await (history._wearableEdge ??= wearableLastTs()), + now.timeZoneName, + now.timeZoneOffset.inMinutes, + dayStart, + dayEnd, + jsonEncode(profile.toMap()), + (LocalDb.sessionSensorSources.toList()..sort()).join(','), + LocalDb.sessionWearableSource, + '$wearable', + history.inputsBefore(dayId), + finalized.join(','), + ].join('|'); + return sha1.convert(utf8.encode('$head#$rows')).toString(); + } String? lastRunError; int lastRunFailedDays = 0; @@ -3220,6 +3335,7 @@ class DerivationEngine { ..['last_error'] = null; lastRunError = null; lastRunFailedDays = 0; + lastRunUnchangedDays = 0; FirebaseTraceHandle? runTrace; try { @@ -3368,6 +3484,7 @@ class DerivationEngine { var done = 0; var completed = 0; var failures = 0; + var unchanged = 0; final activeDays = {}; _diag['stage'] = 'per_day'; _diag['active_days'] = const []; @@ -3382,10 +3499,38 @@ class DerivationEngine { // which Dart's cooperative single-threaded scheduler makes atomic // relative to the other concurrent workers even though the actual // isolate CPU work they await genuinely runs in parallel across cores. + // A forced pass and a day the user just edited always derive. + Future fingerprint(String dayId) async { + if (force || editedDays.contains(dayId) || debugDeriveEveryDay) { + return null; + } + try { + return await _dayInputFingerprint( + dayId, profile, dataNowSec, history); + } catch (e) { + _log('derive day $dayId: fingerprint failed, deriving: $e'); + return null; + } + } + Future processDay(String dayId) async { activeDays.add(dayId); _diag['active_days'] = activeDays.toList(); try { + final before = await fingerprint(dayId); + if (before != null && + before == await LocalDb.deriveFingerprint(dayId)) { + unchanged++; + _diag['unchanged_days'] = unchanged; + activeDays.remove(dayId); + _diag['active_days'] = activeDays.toList(); + completed++; + onDayDone?.call(dayId, completed, orderedDays.length); + return; + } + // Cleared before the derive so an interrupted one cannot leave a + // fingerprint vouching for the row it was rewriting. + await LocalDb.putDeriveFingerprint(dayId, null); final prepared = await _prepareTargetDay(dayId); // Override day whose raw has been pruned (≥14 d): re-deriving would // produce an empty/absent result and clobber the user's manual sleep. @@ -3399,6 +3544,11 @@ class DerivationEngine { await _derivePreparedDay(prepared, profile, dataNowSec, history); done++; _diag['done_days'] = done; + // Stored only when the derive was a fixed point; see + // [_dayInputFingerprint]. + if (before != null && before == await fingerprint(dayId)) { + await LocalDb.putDeriveFingerprint(dayId, before); + } } else { _log('derive day $dayId skipped: no bounded window payload'); await _markDaySkipped( @@ -3436,8 +3586,15 @@ class DerivationEngine { _diag['stage'] = 'quiet_repass'; done += await _quietLevelRepass(profile, dataNowSec, scope.rawDays); - // 4. Cross-day rollup + notifications (best-effort). - if (done > 0) { + lastRunUnchangedDays = unchanged; + if (unchanged > 0) { + _log('derive: $unchanged day(s) unchanged since their last derive'); + } + + // 4. Cross-day rollup + notifications (best-effort). A day left alone + // because nothing it reads moved still counts: the rollup and the + // notifications ran on every such pass before the skip existed. + if (done > 0 || unchanged > 0) { _diag['stage'] = 'baselines'; await _refreshBaselines(); _diag['stage'] = 'cross_day'; @@ -5072,6 +5229,7 @@ class DerivationEngine { if (todoDays.isEmpty) { _log('rescan: no recent decoded-backed days'); await LocalDb.setCursor('baseline_sig', sig); + await _storeQuietEntries(); return 0; } _log( @@ -5080,11 +5238,31 @@ class DerivationEngine { ); final history = await _BaselineHistoryCache.load(); + // Only the quiet level moved: strain is priced on the quiet levels of + // the days BEFORE each day, so only a day after a changed level can + // score differently. The rest — finalized days included — keep their + // rows, which a re-derive would rewrite unchanged. + final prevQuiet = await LocalDb.getCursor(_baselineQuietCursor); + var rescanDays = todoDays; + if (prev != null && + prevQuiet != null && + _withoutQuietSignature(prev) == _withoutQuietSignature(sig)) { + final changed = changedQuietDates( + _decodeQuietEntries(prevQuiet), history.quietWindowEntries()); + if (changed != null) { + rescanDays = [ + for (final d in todoDays) + if (changed.any((c) => c.compareTo(d) < 0)) d, + ]; + _log('rescan: quiet level moved on $changed — ' + '${rescanDays.length} of ${todoDays.length} day(s) priced on it'); + } + } // Same bounded worker-pool pattern as run()/runDays — up to // _rescanWindowDays (21) days is exactly the kind of sweep that used // to run fully sequentially for no reason (independent day_id-keyed // writes + one frozen baseline snapshot shared read-only here). - final orderedDays = todoDays.reversed.toList(); + final orderedDays = rescanDays.reversed.toList(); var done = 0; var completed = 0; @@ -5106,12 +5284,15 @@ class DerivationEngine { await runWithConcurrency(orderedDays, _deriveConcurrency, processDay); - await _refreshBaselines(); - // Cross-day rollup + notifications reflect the refreshed scalars. - await _runCrossDay(profile); - await _runNotifications(); + if (orderedDays.isNotEmpty) { + await _refreshBaselines(); + // Cross-day rollup + notifications reflect the refreshed scalars. + await _runCrossDay(profile); + await _runNotifications(); + } // Store the new signature so the next tick is a cheap no-op until it moves. await LocalDb.setCursor('baseline_sig', await _baselineSignature()); + await _storeQuietEntries(); await LocalDb.refreshComputeFreshness(); return done; } catch (e, st) { @@ -5123,6 +5304,57 @@ class DerivationEngine { } } + /// Cursor holding [_BaselineHistoryCache.quietWindowEntries] as of the + /// last `baseline_sig`, so a quiet-only change can be narrowed to the days + /// it reaches. + static const String _baselineQuietCursor = 'baseline_quiet'; + + Future _storeQuietEntries() async => LocalDb.setCursor( + _baselineQuietCursor, + jsonEncode((await _BaselineHistoryCache.load()).quietWindowEntries()), + ); + + /// The baseline signature without its quiet-level part (`|q…`, last). + static String _withoutQuietSignature(String sig) { + final i = sig.lastIndexOf('|q'); + return i < 0 ? sig : sig.substring(0, i); + } + + static List>? _decodeQuietEntries(String json) { + try { + return [ + for (final e in jsonDecode(json) as List) + [(e as List)[0] as String, (e[1] as num).toDouble()], + ]; + } catch (_) { + return null; + } + } + + /// The dates whose quiet level differs between [before] and [after] (both + /// `[date, value]`, ascending), over the dates both lists cover — an entry + /// that merely slid out of the older end of one is not a change. Null when + /// either is unreadable: the caller then rescans everything, as before. + @visibleForTesting + static Set? changedQuietDates( + List>? before, + List> after, + ) { + if (before == null) return null; + final a = {for (final e in before) e[0] as String: e[1]}; + final b = {for (final e in after) e[0] as String: e[1]}; + String? first(Map m) => + m.isEmpty ? null : (m.keys.toList()..sort()).first; + final fa = first(a), fb = first(b); + final cut = fa == null || fb == null + ? '' + : (fa.compareTo(fb) > 0 ? fa : fb); + return { + for (final d in {...a.keys, ...b.keys}) + if (d.compareTo(cut) >= 0 && a[d] != b[d]) d, + }; + } + /// The days [rescanRecent] re-derives: every day with substrate inside the /// rescan window, minus any whose derive window ([_targetDayWindow], from the /// previous noon) reaches below [prunedBeforeSec]. The prune cuts on local diff --git a/lib/data/db.dart b/lib/data/db.dart index 36c5d35b4..f72ae2e70 100644 --- a/lib/data/db.dart +++ b/lib/data/db.dart @@ -1254,6 +1254,7 @@ class LocalDb { await _createDerived(db); await _createDayResult(db); await _ensureDayResultImportedColumn(db); + await _createDeriveFingerprint(db); await _createUserTables(db); await _createSyncState(db); await _createSyncCursor(db); @@ -9616,6 +9617,273 @@ class LocalDb { return {for (final r in rows) r['day_id'] as String}; } + /// Every stored input a derive of [dayId] reads, as one string, or null + /// when the derive can depend on the wall clock (a session with no end, + /// which activity review treats as running until now). The derivation + /// engine hashes this with the profile, the algorithm version, the data + /// edge and the baseline history, and skips a day whose hash has not moved + /// since a derive that changed nothing it reads (see + /// `DerivationEngine._dayInputFingerprint`). + /// + /// CONSERVATIVE BY CONSTRUCTION: windows are wider than any one the derive + /// loads ([fromSec]..[toSec] is the night search through the nap buffer + /// with a day of slack either side), whole small tables are read where the + /// derive reads part of them, and the only columns left out are write + /// stamps (`computed_at`, `updated_at`, `updated_ts`) and the device row's + /// battery/last-seen readings, none of which a derive reads. Over-reading + /// costs a derive that could have been skipped; under-reading would serve + /// a stale day, so anything in doubt is in. + static Future dayInputSignature({ + required String dayId, + required String prevDayId, + required String nextDayId, + required int fromSec, + required int toSec, + required String? wearableDeviceId, + }) async { + final db = await instance; + final out = StringBuffer(); + String rows(List> rs, {Set drop = const {}}) { + final keys = rs.isEmpty + ? const [] + : (rs.first.keys.where((k) => !drop.contains(k)).toList()..sort()); + return jsonEncode([ + for (final r in rs) [for (final k in keys) r[k]], + ]); + } + + Future add(String label, String sql, + [List args = const [], Set drop = const {}]) async { + out + ..write(label) + ..write('=') + ..write(rows(await db.rawQuery(sql, args), drop: drop)) + ..write(';'); + } + + Future has(String table) async => + (await _objectType(db, table)) != null; + + final days = [prevDayId, dayId, nextDayId]; + const stamps = {'computed_at', 'updated_at', 'updated_ts'}; + final openSession = await db.rawQuery( + 'SELECT 1 FROM sessions WHERE end_ts IS NULL AND start_ts <= ? LIMIT 1', + [toSec + 86400], + ); + if (openSession.isNotEmpty) return null; + + final oneHzCols = { + for (final c in await db.rawQuery('PRAGMA table_info(decoded_onehz)')) + c['name'] as String, + }..removeAll(const {'device_id', 'device_family', 'source'}); + await add( + 'onehz', + 'SELECT COUNT(*), TOTAL(rowid), MAX(rowid), ' + '${[for (final c in oneHzCols.toList()..sort()) 'TOTAL($c)'].join(', ')}, ' + 'TOTAL(LENGTH(device_id)), TOTAL(LENGTH(device_family)), ' + 'TOTAL(LENGTH(source)) ' + 'FROM decoded_onehz WHERE rec_ts BETWEEN ? AND ?', + [fromSec, toSec], + ); + await add( + 'rr', + 'SELECT COUNT(*), TOTAL(rowid), MAX(rowid), TOTAL(rr_ms), ' + 'TOTAL(rr_ts_ms), TOTAL(LENGTH(device_id)) ' + 'FROM decoded_rr WHERE rec_ts BETWEEN ? AND ?', + [fromSec, toSec], + ); + // The wear and charge transitions are read back past the window (the + // state already open at its start), so all of them count. + await add( + 'band_events', + 'SELECT COUNT(*), TOTAL(rowid), MAX(rowid) FROM band_events ' + 'WHERE (ts BETWEEN ? AND ?) OR event_id IN (?, ?, ?, ?)', + [ + fromSec, + toSec, + proto.EventId.chargingOn, + proto.EventId.chargingOff, + proto.EventId.wristOn, + proto.EventId.wristOff, + ], + ); + await add('live_coverage', + 'SELECT * FROM live_coverage WHERE day = ? ORDER BY rowid', [dayId]); + await add( + 'sessions', + 'SELECT * FROM sessions WHERE start_ts BETWEEN ? AND ? ORDER BY id', + [fromSec - 2 * 86400, toSec + 86400], + ); + for (final t in const [ + 'session_sensor', + 'session_score_mask', + 'signal_priority', + 'step_calibration', + 'activity_review_meta', + 'activity_review_days', + ]) { + if (await has(t)) { + await add(t, 'SELECT * FROM $t ORDER BY rowid', const [], stamps); + } + } + final marks = List.filled(days.length, '?').join(','); + await add('sleep_override', + 'SELECT * FROM sleep_override WHERE day_id IN ($marks) ORDER BY day_id', + days, stamps); + if (await has('sleep_nap')) { + await add( + 'sleep_nap', + 'SELECT * FROM sleep_nap WHERE day_id IN ($marks) ' + 'OR (start_ts < ? AND end_ts > ?) ORDER BY rowid', + [...days, toSec, fromSec], + stamps, + ); + } + if (await has('activity_suggestions')) { + await add( + 'activity_suggestions', + 'SELECT * FROM activity_suggestions WHERE day_id IN ($marks) ' + 'OR (detection_start < ? AND detection_end > ?) ORDER BY id', + [...days, toSec + 86400, fromSec - 86400], + stamps, + ); + } + await add( + 'device', + 'SELECT * FROM device ORDER BY id', + const [], + {...stamps, 'last_seen', 'battery_pct', 'battery_ts', 'name'}, + ); + await add( + 'cursor', + "SELECT name, value FROM sync_cursor WHERE name IN (?, ?) " + "OR name LIKE 'wearable_enabled:%' ORDER BY name", + ['active_wearable', kDecodedPrunedBeforeCursor], + ); + await add( + 'device_coverage', + 'SELECT * FROM device_coverage WHERE end_ts > ? AND start_ts < ? ' + 'ORDER BY rowid', + [fromSec, toSec], + ); + if (await has('vendor_sleep_epoch')) { + await add( + 'vendor_sleep', + 'SELECT COUNT(*), TOTAL(rowid), MAX(rowid) FROM vendor_sleep_epoch ' + 'WHERE night_onset_ts BETWEEN ? AND ?', + [fromSec - 86400, toSec + 86400], + ); + } + if (wearableDeviceId != null) { + // A wearable's substrate is read without a window (its whole HR + // history sets its edge) and from the archive (rings bank replies + // there), so all of that device's rows count. + await add( + 'wearable_rows', + 'SELECT COUNT(*), TOTAL(rowid), MAX(rowid), TOTAL(hr) ' + 'FROM decoded_onehz WHERE device_id = ?', + [wearableDeviceId], + ); + await add( + 'wearable_archive', + 'SELECT COUNT(*), MAX(captured_at), TOTAL(captured_at), ' + 'TOTAL(LENGTH(hex)) FROM raw_archive WHERE device_id = ?', + [wearableDeviceId], + ); + } + await add( + 'baselines', + "SELECT key, payload_json FROM baselines " + "WHERE key IN ('sleep_user_profile', 'movement_floor') ORDER BY key", + ); + await add( + 'freshness', + 'SELECT key, payload_json FROM compute_freshness WHERE key = ?', + ['quiet_repass:$dayId'], + ); + // The day's own rows and its neighbours' — read back by the carry-forward + // guards, tonight's sleep and activity review. Hashed, not inlined. + final results = await db.rawQuery( + 'SELECT day_id, algo_version, finalized, partial, skipped, rhr, rmssd, ' + 'readiness, window_json, payload_json FROM day_result ' + 'WHERE day_id IN ($marks) ORDER BY day_id, algo_version', + days, + ); + out.write('day_result='); + for (final r in results) { + final payload = r['payload_json']; + out + ..write([ + for (final k in const [ + 'day_id', 'algo_version', 'finalized', 'partial', 'skipped', + 'rhr', 'rmssd', 'readiness', 'window_json', + ]) + r[k], + ].join(',')) + ..write(',') + ..write(payload is String + ? sha1.convert(utf8.encode(payload)).toString() + : 'null') + ..write('|'); + } + out.write(';'); + await add( + 'candidates', + 'SELECT day_id, algo_version, payload_json FROM sleep_session_candidates ' + 'WHERE day_id = ? ORDER BY algo_version', + [dayId], + ); + out + ..write('windows=') + ..write(rows(await sleepWindowRows(60))) + ..write(';'); + return out.toString(); + } + + /// The input fingerprint the last derive of [dayId] stored, when that + /// derive changed nothing it reads — see `DerivationEngine`'s skip. + static Future deriveFingerprint(String dayId) async { + final db = await instance; + final r = await db.query(_deriveFingerprintTable, + columns: ['fp'], where: 'day_id = ?', whereArgs: [dayId], limit: 1); + return r.isEmpty ? null : r.first['fp'] as String?; + } + + /// Store (or, with null, clear) [dayId]'s fingerprint. Rows two months + /// older than [dayId] go with it: only days still re-deriving are ever + /// looked up. Measured from the day, not the wall clock, so a backfill of + /// old days does not delete its own rows. + static Future putDeriveFingerprint(String dayId, String? fp) async { + final db = await instance; + await db.transaction((txn) async { + if (fp == null) { + await txn.delete(_deriveFingerprintTable, + where: 'day_id = ?', whereArgs: [dayId]); + return; + } + await txn.insert(_deriveFingerprintTable, {'day_id': dayId, 'fp': fp}, + conflictAlgorithm: ConflictAlgorithm.replace); + final t = DateTime.tryParse(dayId); + if (t != null) { + await txn.delete(_deriveFingerprintTable, + where: 'day_id < ?', + whereArgs: [dayLabelOf(DateTime(t.year, t.month - 2, t.day))]); + } + }); + } + + static const String _deriveFingerprintTable = 'derive_fingerprint'; + + /// One row per day: the fingerprint above. Recomputable bookkeeping — + /// losing it costs one derive per day — so additive on the open pass with + /// no version bump, like the other repair-path tables. + static Future _createDeriveFingerprint(Database db) async { + await db.execute( + 'CREATE TABLE IF NOT EXISTS $_deriveFingerprintTable ' + '(day_id TEXT PRIMARY KEY, fp TEXT NOT NULL)', + ); + } + /// The set of day_id labels that are FINALIZED at [algoVersion] (locked). A /// finalized day is never recomputed even on a version bump. /// diff --git a/lib/data/recent_slice.dart b/lib/data/recent_slice.dart index 64ed6d41f..966b590fa 100644 --- a/lib/data/recent_slice.dart +++ b/lib/data/recent_slice.dart @@ -81,6 +81,7 @@ const Map kSliceTables = { 'metric_method': (SliceKeep.all, null, 'how each daily metric was measured'), 'baselines': (SliceKeep.all, null, 'personal baselines'), 'compute_freshness': (SliceKeep.all, null, 'compute bookkeeping'), + 'derive_fingerprint': (SliceKeep.all, null, 'which days changed since they were last scored'), 'sleep_nap': (SliceKeep.all, null, 'naps'), 'sleep_override': (SliceKeep.all, null, 'sleep times the user edited'), // Workouts and activity: one row each. diff --git a/test/derive_unchanged_skip_test.dart b/test/derive_unchanged_skip_test.dart new file mode 100644 index 000000000..647b20922 --- /dev/null +++ b/test/derive_unchanged_skip_test.dart @@ -0,0 +1,181 @@ +// A heavy pass leaves a pending day alone when nothing it reads has moved +// since a derive of it that changed nothing it reads. That is only allowed if +// it is invisible: the same passes with every day derived must store exactly +// the same results, and an override, new data, a profile change or an +// algorithm bump must still re-derive. + +import 'dart:convert'; + +import 'package:flutter_test/flutter_test.dart'; +import 'package:openstrap_edge/compute/derivation_engine.dart'; +import 'package:openstrap_edge/compute/profile.dart'; +import 'package:openstrap_edge/data/db.dart'; +import 'package:openstrap_edge/data/models.dart' show RawRecord, Sample; +import 'package:path/path.dart' as p; +import 'package:shared_preferences/shared_preferences.dart'; +import 'package:sqflite_common_ffi/sqflite_ffi.dart'; + +import 'support/synthetic_day.dart'; + +const _profile = Profile(ageYears: 32, weightKg: 72, heightCm: 176, sex: 'm'); + +// Mid-July, no DST edge. The newest days stay pending: the data edge is less +// than 48 h past them. +final _days = [ + for (var i = 0; i < 3; i++) SyntheticDay(DateTime(2026, 7, 13 + i)), +]; + +Future _freshDb(String name) async { + SharedPreferences.setMockInitialValues({}); + await LocalDb.close(); + LocalDb.dbName = name; + await databaseFactory + .deleteDatabase(p.join(await databaseFactory.getDatabasesPath(), name)); + var committed = 0; + for (final d in _days) { + for (final (raws, samples) in d.primaryBatches()) { + final keep = [ + for (var j = 0; j < raws.length; j++) + if (raws[j].recTs! > committed) j, + ]; + if (keep.isEmpty) continue; + await LocalDb.commitSyncBatch( + [for (final j in keep) raws[j]], + [for (final j in keep) samples[j]], + deviceFamily: 'gen4', + ); + committed = raws[keep.last].recTs!; + } + } +} + +Future<({int derived, int unchanged})> _pass({Profile profile = _profile}) async { + final e = DerivationEngine(); + final n = await e.run(profile, heavy: true); + return (derived: n, unchanged: e.lastRunUnchangedDays); +} + +/// Every stored output of a derive, write stamps dropped (the columns, and +/// the wall-clock stamps inside the baseline artifacts). +Future _snapshot() async { + final db = await LocalDb.instance; + Future>> rows(String sql) async => [ + for (final r in await db.rawQuery(sql)) + { + for (final e in r.entries) + if (!e.key.endsWith('_at') && e.key != 'updated_ts') + e.key: e.value is String + ? (e.value as String).replaceAll( + RegExp(r'"(updated|built)[A-Za-z_]*":\s*\d+'), '') + : e.value, + }, + ]; + return jsonEncode({ + 'day_result': await rows( + 'SELECT * FROM day_result ORDER BY day_id, algo_version'), + 'metric_series': + await rows('SELECT * FROM metric_series ORDER BY date, key'), + 'metric_series_version': + await rows('SELECT * FROM metric_series_version ORDER BY date'), + 'wake_day_features': await rows( + 'SELECT * FROM wake_day_features ORDER BY day_id, algo_version'), + 'sleep_session_candidates': await rows('SELECT * FROM ' + 'sleep_session_candidates ORDER BY day_id, algo_version'), + 'activity_suggestions': + await rows('SELECT * FROM activity_suggestions ORDER BY id'), + 'sleep_nap': await rows('SELECT * FROM sleep_nap ORDER BY start_ts'), + 'baselines': await rows('SELECT * FROM baselines ORDER BY key'), + }); +} + +void main() { + setUpAll(() { + sqfliteFfiInit(); + databaseFactory = databaseFactoryFfi; + }); + + tearDown(() async { + DerivationEngine.debugDeriveEveryDay = false; + DerivationEngine.debugFingerprintAlgoVersion = null; + await LocalDb.close(); + }); + + test('skipping unchanged days stores exactly what deriving them does', + () async { + const passes = 5; + await _freshDb('unchanged_skip_on.db'); + final log = <({int derived, int unchanged})>[]; + for (var i = 0; i < passes; i++) { + log.add(await _pass()); + } + final withSkip = await _snapshot(); + + await _freshDb('unchanged_skip_off.db'); + DerivationEngine.debugDeriveEveryDay = true; + for (var i = 0; i < passes; i++) { + final r = await _pass(); + expect(r.unchanged, 0); + } + final everyDay = await _snapshot(); + + // The skip actually happened: by the last pass nothing was derived. + expect(log.last.derived, 0, reason: '$log'); + expect(log.last.unchanged, greaterThan(0), reason: '$log'); + expect(withSkip, everyDay); + }, timeout: const Timeout(Duration(minutes: 10))); + + test('an override, new data, a profile change and a version bump re-derive', + () async { + await _freshDb('unchanged_skip_triggers.db'); + Future settle() async { + for (var i = 0; i < 6; i++) { + if ((await _pass()).derived == 0) return; + } + fail('never settled'); + } + + await settle(); + final settled = await _pass(); + expect(settled.derived, 0); + final pending = settled.unchanged; + expect(pending, greaterThan(0)); + + // The user's own sleep window on a pending day. + final d = _days[1]; + await LocalDb.putSleepOverride( + dayId: '2026-07-14', + onsetTs: SyntheticDay.sec(d.sleepOnset) + 600, + offsetTs: SyntheticDay.sec(d.sleepOffset), + source: 'manual', + ); + expect((await _pass()).derived, greaterThanOrEqualTo(1)); + await settle(); + + // A second re-delivered inside a pending day's window (counter-reset + // recovery replaces the row in place, the data edge does not move). + final t = SyntheticDay.sec(DateTime(2026, 7, 14, 12)); + await LocalDb.commitSyncBatch( + [ + RawRecord( + counter: 1, packetType: 0x2F, hex: 'aa', capturedAt: t * 1000, + recTs: t), + ], + [Sample(tsEpoch: t, counter: 1, hr: 140, ax: 0, ay: 0, az: 1)], + deviceFamily: 'gen4', + ); + expect((await _pass()).derived, greaterThanOrEqualTo(1)); + await settle(); + + expect( + (await _pass( + profile: const Profile( + ageYears: 33, weightKg: 72, heightCm: 176, sex: 'm'))) + .derived, + pending, + ); + await settle(); + + DerivationEngine.debugFingerprintAlgoVersion = kAlgoVersion + 1; + expect((await _pass()).derived, pending); + }, timeout: const Timeout(Duration(minutes: 15))); +} diff --git a/test/rescan_quiet_narrowing_test.dart b/test/rescan_quiet_narrowing_test.dart new file mode 100644 index 000000000..cd338b09c --- /dev/null +++ b/test/rescan_quiet_narrowing_test.dart @@ -0,0 +1,44 @@ +// A rescan triggered only by the quiet level re-prices just the days whose +// strain window holds a level that moved. + +import 'package:flutter_test/flutter_test.dart'; +import 'package:openstrap_edge/compute/derivation_engine.dart'; + +void main() { + List> entries(Map m) => [ + for (final e in (m.entries.toList() + ..sort((a, b) => a.key.compareTo(b.key)))) + [e.key, e.value], + ]; + + test('a value that moved is a change, and only that one', () { + final before = entries({ + '2026-07-01': .10, + '2026-07-02': .15, + '2026-07-03': .20, + }); + final after = entries({ + '2026-07-01': .10, + '2026-07-02': .25, + '2026-07-03': .20, + }); + expect(DerivationEngine.changedQuietDates(before, after), {'2026-07-02'}); + }); + + test('a new level and a removed one are changes', () { + final before = entries({'2026-07-01': .10, '2026-07-02': .15}); + final after = entries({'2026-07-01': .10, '2026-07-03': .30}); + expect(DerivationEngine.changedQuietDates(before, after), + {'2026-07-02', '2026-07-03'}); + }); + + test('an entry that slid off the older end is not a change', () { + final before = entries({'2026-07-01': .10, '2026-07-02': .15}); + final after = entries({'2026-07-02': .15, '2026-07-03': .30}); + expect(DerivationEngine.changedQuietDates(before, after), {'2026-07-03'}); + }); + + test('nothing stored yet means rescan everything', () { + expect(DerivationEngine.changedQuietDates(null, const []), isNull); + }); +} From ac52c7f96f3370b320b5dbd984ed0e01f1752583 Mon Sep 17 00:00:00 2001 From: abdulsaheel Date: Wed, 28 Oct 2026 23:28:05 +0530 Subject: [PATCH 07/33] compact pruned against its own day list instead of the derive scope The scope is what a pass prunes against, wearable days and all; reading it the same way keeps the two from drifting. (cherry picked from commit d2ec6cfe8370a67b3026ff0832a0333e6f2c00ca) --- lib/compute/derivation_engine.dart | 17 ++++++----------- 1 file changed, 6 insertions(+), 11 deletions(-) diff --git a/lib/compute/derivation_engine.dart b/lib/compute/derivation_engine.dart index 6c3c1fd15..a48b7429d 100644 --- a/lib/compute/derivation_engine.dart +++ b/lib/compute/derivation_engine.dart @@ -3143,19 +3143,14 @@ class DerivationEngine { final before = await LocalDb.databaseBytesOnDisk(); onStep?.call(CompactStep.prune); try { - // Every day with substrate, the wearable's included, as the scope - // hands a pass's prune: a pending day missing here is not held. - final rawByDay = await LocalDb.decodedRecTsMaxByDay(); - final wearable = await activeWearable(); - if (wearable != null) { - for (final day in (await wearableRecTsMaxByDay(wearable)).keys) { - rawByDay.putIfAbsent(day, () => 0); - } - } - final rawDays = rawByDay.keys.toList()..sort(); + // The scope a pass would prune against: every day with substrate, + // the wearable's included, so a pending one is held. + final scope = await _deriveScope(heavy: true, force: false); final dataNowSec = await _withWearableEdge(await LocalDb.lastDecodedRecTs()) ?? 0; - if (dataNowSec > 0) await _pruneOldDecoded(rawDays, dataNowSec); + if (dataNowSec > 0) { + await _pruneOldDecoded(scope.rawDays, dataNowSec); + } } catch (e) { _log('compact: prune skipped: $e'); } From 042cc309fe588ff3da5431f98e4da3f6df65559d Mon Sep 17 00:00:00 2001 From: abdulsaheel Date: Wed, 28 Oct 2026 23:28:05 +0530 Subject: [PATCH 08/33] backup folder test looked for its result card without scrolling to it The storage group pushed the card below the fold. (cherry picked from commit c09893572cb6313701706c659427419fb67573ac) --- test/data_backup_folder_ui_test.dart | 3 +++ 1 file changed, 3 insertions(+) diff --git a/test/data_backup_folder_ui_test.dart b/test/data_backup_folder_ui_test.dart index 18f7e6f6e..1e90b3646 100644 --- a/test/data_backup_folder_ui_test.dart +++ b/test/data_backup_folder_ui_test.dart @@ -118,6 +118,9 @@ void main() { await t.tap(find.text('Backup folder')); await t.pumpAndSettle(); expect(app.backupFolder?.name, 'Backups'); + // The result card sits under the last group, past the fold. + await t.scrollUntilVisible(find.textContaining('No folder picker'), 200, + scrollable: find.byType(Scrollable).first); expect(find.textContaining('No folder picker'), findsOneWidget); }, variant: TargetPlatformVariant.only(TargetPlatform.android), From 0ac2e35761b0ff8be337b76946f3a02722743b4e Mon Sep 17 00:00:00 2001 From: abdulsaheel Date: Thu, 29 Oct 2026 00:38:17 +0530 Subject: [PATCH 09/33] store the quiet snapshot before the baseline signature, recheck for a sync before vacuum a kill between the two rescan cursor writes left a fresh signature over a stale quiet snapshot, so the next quiet-only rescan narrowed against the wrong levels. the compaction also only checked for a running sync before the prune; one that started meanwhile then waited out the whole vacuum. --- lib/compute/derivation_engine.dart | 33 +++++++++++++++++++++++++++--- lib/state/app_state.dart | 16 +++++++++++---- test/compact_database_test.dart | 15 ++++++++++++++ 3 files changed, 57 insertions(+), 7 deletions(-) diff --git a/lib/compute/derivation_engine.dart b/lib/compute/derivation_engine.dart index a48b7429d..44945a617 100644 --- a/lib/compute/derivation_engine.dart +++ b/lib/compute/derivation_engine.dart @@ -3042,15 +3042,22 @@ class CompactResult { required this.freeBytes, required this.neededBytes, required this.vacuumed, + this.deferred = false, }); final int beforeBytes, afterBytes, neededBytes; + /// True when the VACUUM was skipped because a writer that must not wait on + /// it (a sync, a live session) started while the prune ran. The prune + /// still ran. + final bool deferred; + /// Free space on the volume when it was checked; null when the platform /// could not say. final int? freeBytes; - /// False when the free-space gate refused the VACUUM (the prune still ran). + /// False when the free-space gate refused the VACUUM, or it was [deferred] + /// (the prune still ran). final bool vacuumed; int get freedBytes => beforeBytes > afterBytes ? beforeBytes - afterBytes : 0; @@ -3135,9 +3142,14 @@ class DerivationEngine { /// The VACUUM runs only when [freeDiskBytes] reports at least twice the /// database (plus its WAL) free: it writes a whole second copy before it /// drops the first. Unknown free space counts as not enough. + /// + /// [busy] is asked again right before the VACUUM: the caller's own check + /// ran before the prune, and a sync that started since would otherwise sit + /// on the write lock for the whole rewrite. Future compactStorage({ required Future Function() freeDiskBytes, void Function(CompactStep step)? onStep, + bool Function()? busy, }) => whileIdle(() async { final before = await LocalDb.databaseBytesOnDisk(); @@ -3166,6 +3178,16 @@ class DerivationEngine { vacuumed: false, ); } + if (busy?.call() ?? false) { + return CompactResult( + beforeBytes: before, + afterBytes: size, + freeBytes: free, + neededBytes: 2 * size, + vacuumed: false, + deferred: true, + ); + } onStep?.call(CompactStep.vacuum); await LocalDb.vacuumAndCheckpoint(); return CompactResult( @@ -5223,8 +5245,12 @@ class DerivationEngine { ); if (todoDays.isEmpty) { _log('rescan: no recent decoded-backed days'); - await LocalDb.setCursor('baseline_sig', sig); + // Quiet snapshot first: a kill between the two writes must leave the + // signature stale (a cheap full rescan next tick), never a fresh + // signature over an old quiet snapshot that narrows the next rescan + // against the wrong levels. await _storeQuietEntries(); + await LocalDb.setCursor('baseline_sig', sig); return 0; } _log( @@ -5286,8 +5312,9 @@ class DerivationEngine { await _runNotifications(); } // Store the new signature so the next tick is a cheap no-op until it moves. - await LocalDb.setCursor('baseline_sig', await _baselineSignature()); + // Quiet snapshot before the signature, for the reason given above. await _storeQuietEntries(); + await LocalDb.setCursor('baseline_sig', await _baselineSignature()); await LocalDb.refreshComputeFreshness(); return done; } catch (e, st) { diff --git a/lib/state/app_state.dart b/lib/state/app_state.dart index 9ba7217aa..978503203 100644 --- a/lib/state/app_state.dart +++ b/lib/state/app_state.dart @@ -2474,15 +2474,23 @@ class AppState extends ChangeNotifier { Future compactDatabase({ void Function(CompactStep step)? onStep, }) async { - if (busy || syncingNow || _liveSessionActive) return null; - if (_deriveScheduler.offloadActive) return null; + bool writerRunning() => + busy || + syncingNow || + _liveSessionActive || + _deriveScheduler.offloadActive; + if (writerRunning()) return null; final result = await _derive.compactStorage( freeDiskBytes: freeDiskBytes, onStep: onStep, + busy: writerRunning, ); _log('[db] compact: ${result.beforeBytes >> 20} MB -> ' - '${result.afterBytes >> 20} MB (vacuumed=${result.vacuumed})'); - return result; + '${result.afterBytes >> 20} MB (vacuumed=${result.vacuumed}, ' + 'deferred=${result.deferred})'); + // Same answer as refusing up front: the prune ran, the VACUUM waits for + // the sync to finish. + return result.deferred ? null : result; } /// User-initiated "Re-analyze data": force-derive EVERY day that has raw, diff --git a/test/compact_database_test.dart b/test/compact_database_test.dart index 38383c08a..edaae622a 100644 --- a/test/compact_database_test.dart +++ b/test/compact_database_test.dart @@ -75,6 +75,21 @@ void main() { expect(await LocalDb.freelistBytes(), greaterThan(0)); }); + test('skips the VACUUM when a sync started while the prune ran', () async { + await _useFreshDb('compact_busy.db'); + await _bloat(); + final steps = []; + final r = await DerivationEngine().compactStorage( + freeDiskBytes: () async => 1 << 40, + onStep: steps.add, + busy: () => true, + ); + expect(r.vacuumed, isFalse); + expect(r.deferred, isTrue); + expect(steps, [CompactStep.prune]); + expect(await LocalDb.freelistBytes(), greaterThan(0)); + }); + test('unknown free space counts as not enough', () async { await _useFreshDb('compact_unknown.db'); await _bloat(); From 5830ec70d16c2aef0815d820138c8f626251b9c3 Mon Sep 17 00:00:00 2001 From: abdulsaheel Date: Sat, 31 Oct 2026 13:25:27 +0530 Subject: [PATCH 10/33] keep the raw archive view and index steps atomic the rename and the index drop now commit together, and the view is rebuilt in the same transaction that drops the legacy table, so no reader ever finds raw_archive missing. --- lib/data/db.dart | 45 ++++++++++++++++++++++++++------------------- 1 file changed, 26 insertions(+), 19 deletions(-) diff --git a/lib/data/db.dart b/lib/data/db.dart index 7128fd8f4..0cf721d38 100644 --- a/lib/data/db.dart +++ b/lib/data/db.dart @@ -7481,7 +7481,7 @@ class LocalDb { /// old name working — an older build after a rollback, a test fixture — /// with the text itself as the key: correct, just without the size saving. /// Everything in this build writes through [rawArchiveRow] instead. - static Future _ensureRawArchiveView(Database db) async { + static Future _ensureRawArchiveView(DatabaseExecutor db) async { final legacy = await _objectType(db, _rawArchiveLegacy) == 'table'; await db.execute('DROP VIEW IF EXISTS raw_archive'); await db.execute( @@ -7526,21 +7526,27 @@ class LocalDb { /// both until then, so no reader ever sees a frame missing. static Future _ensureRawArchiveStore(Database db) async { if (await _objectType(db, 'raw_archive') == 'table') { - if (await _objectType(db, _rawArchiveLegacy) == 'table') { - // Only reachable if a file was hand-assembled with both; fold the - // newer table into the aside one rather than lose either. - await db.execute( - 'INSERT OR IGNORE INTO $_rawArchiveLegacy ' - '(device_id, hex, counter, packet_type, rec_ts, captured_at, reason) ' - 'SELECT device_id, hex, counter, packet_type, rec_ts, captured_at, ' - 'reason FROM raw_archive', - ); - await db.execute('DROP TABLE raw_archive'); - } else { - await db.execute('ALTER TABLE raw_archive RENAME TO $_rawArchiveLegacy'); - } - // Frees the name for [rawArchiveTable]'s index (see there). - await db.execute('DROP INDEX IF EXISTS idx_raw_archive_captured'); + // One transaction: a crash between the rename and the index drop would + // leave the frames table without its captured_at index until the + // legacy table is gone, because the index name is still taken. + await db.transaction((txn) async { + if (await _objectType(txn, _rawArchiveLegacy) == 'table') { + // Only reachable if a file was hand-assembled with both; fold the + // newer table into the aside one rather than lose either. + await txn.execute( + 'INSERT OR IGNORE INTO $_rawArchiveLegacy ' + '(device_id, hex, counter, packet_type, rec_ts, captured_at, ' + 'reason) SELECT device_id, hex, counter, packet_type, rec_ts, ' + 'captured_at, reason FROM raw_archive', + ); + await txn.execute('DROP TABLE raw_archive'); + } else { + await txn + .execute('ALTER TABLE raw_archive RENAME TO $_rawArchiveLegacy'); + } + // Frees the name for [rawArchiveTable]'s index (see there). + await txn.execute('DROP INDEX IF EXISTS idx_raw_archive_captured'); + }); } await _createRawArchiveFrames(db); if (await _objectType(db, 'raw_archive') != 'view') { @@ -7637,10 +7643,11 @@ class LocalDb { await txn.execute('DROP TRIGGER IF EXISTS raw_archive_delete'); await txn.execute('DROP VIEW IF EXISTS raw_archive'); await txn.execute('DROP TABLE $_rawArchiveLegacy'); + // Rebuilt in the same transaction: between a commit and a rebuild + // outside it there is no raw_archive at all, and a reader on + // another isolate fails with "no such table". + await _ensureRawArchiveView(txn); }); - if (await _objectType(db, 'raw_archive') != 'view') { - await _ensureRawArchiveView(db); - } return moved; } final hi = (page.last['_r'] as num).toInt(); From 09c4cd9faa6919fb6a36cb5a456f29d1d2b21643 Mon Sep 17 00:00:00 2001 From: abdulsaheel Date: Sat, 31 Oct 2026 13:26:56 +0530 Subject: [PATCH 11/33] keep a year of archive for installs that already had one the default window dropped to 90 days, and an existing archive with no stored setting would have lost everything past that on the next housekeeping pass, with no confirmation. on the first open it keeps the year it was getting; the setting still shortens it. --- lib/data/db.dart | 1 + lib/data/substrate_archive.dart | 30 +++++++++++++++++++ test/substrate_archive_window_test.dart | 39 +++++++++++++++++++++++++ 3 files changed, 70 insertions(+) diff --git a/lib/data/db.dart b/lib/data/db.dart index 0cf721d38..52441624b 100644 --- a/lib/data/db.dart +++ b/lib/data/db.dart @@ -1342,6 +1342,7 @@ class LocalDb { // / notifications all existing. DROP+CREATE so a shape change takes effect. await _ensureCoachViews(db); await _dropRawStore(db); + await SubstrateArchive.keepLegacyWindowOnce(db); } /// Older WHOOP CSV imports filed the export's absolute skin temp (°C) under diff --git a/lib/data/substrate_archive.dart b/lib/data/substrate_archive.dart index 4a3ac0d2b..c1fe8c3fc 100644 --- a/lib/data/substrate_archive.dart +++ b/lib/data/substrate_archive.dart @@ -185,6 +185,36 @@ abstract final class SubstrateArchive { return out; } + /// Once per install, on the first open of a build whose default is shorter + /// than a year: an archive that already has buckets but no stored setting + /// was being kept for 365 days, and the new default would delete everything + /// older than 90 on the next housekeeping pass, with no confirmation and no + /// way back. That archive keeps its year; the user can still shorten it from + /// data settings. An empty archive gets the new default. + static Future keepLegacyWindowOnce(Database db) async { + const marker = 'substrate_archive_policy_default_checked'; + final done = await db.query('compute_freshness', + columns: ['key'], where: 'key = ?', whereArgs: [marker], limit: 1); + if (done.isNotEmpty) return; + await db.transaction((txn) async { + final stored = await txn.query('compute_freshness', + columns: ['key'], where: 'key = ?', whereArgs: [policyKey], limit: 1); + final hasArchive = + (await txn.rawQuery('SELECT 1 FROM $table LIMIT 1')).isNotEmpty; + final now = DateTime.now().millisecondsSinceEpoch; + if (stored.isEmpty && hasArchive) { + await txn.insert('compute_freshness', { + 'key': policyKey, + 'payload_json': const SubstrateArchivePolicy(365).toJson(), + 'updated_at': now, + }); + } + await txn.insert('compute_freshness', + {'key': marker, 'payload_json': '{}', 'updated_at': now}, + conflictAlgorithm: ConflictAlgorithm.replace); + }); + } + /// The policy as stored, on [ex] — inside a transaction when the caller /// needs it to agree with what that transaction writes. static Future readPolicy(DatabaseExecutor ex) async { diff --git a/test/substrate_archive_window_test.dart b/test/substrate_archive_window_test.dart index 4cb7906c6..a425d3768 100644 --- a/test/substrate_archive_window_test.dart +++ b/test/substrate_archive_window_test.dart @@ -2,6 +2,7 @@ import 'package:flutter_test/flutter_test.dart'; import 'package:openstrap_edge/data/substrate_archive.dart'; +import 'package:sqflite_common_ffi/sqflite_ffi.dart'; void main() { test('defaults to 90 days', () { @@ -35,4 +36,42 @@ void main() { expect(month.keepsLongerThan(forever), isFalse); expect(year.keepsLongerThan(year), isFalse); }); + + group('an archive from before the 90-day default', () { + late Database db; + setUpAll(sqfliteFfiInit); + setUp(() async { + db = await databaseFactoryFfi.openDatabase(inMemoryDatabasePath); + await db.execute('CREATE TABLE compute_freshness (key TEXT PRIMARY KEY, ' + 'payload_json TEXT NOT NULL, updated_at INTEGER NOT NULL)'); + await db.execute('CREATE TABLE ${SubstrateArchive.table} (day TEXT)'); + }); + tearDown(() => db.close()); + + test('keeps its year when nothing was ever chosen', () async { + await db.insert(SubstrateArchive.table, {'day': '2025-11-01'}); + await SubstrateArchive.keepLegacyWindowOnce(db); + expect((await SubstrateArchive.readPolicy(db)).keepDays, 365); + }); + + test('a new install gets the default, and the check runs once', () async { + await SubstrateArchive.keepLegacyWindowOnce(db); + expect((await SubstrateArchive.readPolicy(db)).keepDays, 90); + // Archiving starts later on this build: still the default. + await db.insert(SubstrateArchive.table, {'day': '2026-10-01'}); + await SubstrateArchive.keepLegacyWindowOnce(db); + expect((await SubstrateArchive.readPolicy(db)).keepDays, 90); + }); + + test('a window the user picked is left alone', () async { + await db.insert(SubstrateArchive.table, {'day': '2025-11-01'}); + await db.insert('compute_freshness', { + 'key': SubstrateArchive.policyKey, + 'payload_json': const SubstrateArchivePolicy(30).toJson(), + 'updated_at': 0, + }); + await SubstrateArchive.keepLegacyWindowOnce(db); + expect((await SubstrateArchive.readPolicy(db)).keepDays, 30); + }); + }); } From 8fdafd806d8e4a39d055b5c3e6b450d4196abd34 Mon Sep 17 00:00:00 2001 From: abdulsaheel Date: Sat, 31 Oct 2026 13:27:58 +0530 Subject: [PATCH 12/33] a fingerprint read or write that fails no longer fails the day the fingerprint only decides whether a day can be skipped, so a busy database there now costs the skip instead of marking the day skipped. a fingerprint that could not be cleared before the derive is not overwritten after it. the unchanged-days count also resets each run. --- lib/compute/derivation_engine.dart | 37 ++++++++++++++++++++++++++---- 1 file changed, 32 insertions(+), 5 deletions(-) diff --git a/lib/compute/derivation_engine.dart b/lib/compute/derivation_engine.dart index 03b6a9ff7..eed675f13 100644 --- a/lib/compute/derivation_engine.dart +++ b/lib/compute/derivation_engine.dart @@ -3241,6 +3241,7 @@ class DerivationEngine { 'todo_days': 0, 'done_days': 0, 'skipped_days': 0, + 'unchanged_days': 0, // List, not a single day — several days can be in flight concurrently // (see run()'s bounded worker pool). 'active_days': [], @@ -3371,6 +3372,7 @@ class DerivationEngine { ..['todo_days'] = 0 ..['done_days'] = 0 ..['skipped_days'] = 0 + ..['unchanged_days'] = 0 ..['active_days'] = [] ..['concurrency'] = _deriveConcurrency ..['last_error'] = null; @@ -3554,13 +3556,34 @@ class DerivationEngine { } } + // The stored fingerprint is only an optimization: a read or write that + // fails (SQLITE_BUSY with two isolates deriving) must cost the skip, + // never the day. + Future storedFingerprint(String dayId) async { + try { + return await LocalDb.deriveFingerprint(dayId); + } catch (e) { + _log('derive day $dayId: fingerprint read failed, deriving: $e'); + return null; + } + } + + Future storeFingerprint(String dayId, String? value) async { + try { + await LocalDb.putDeriveFingerprint(dayId, value); + return true; + } catch (e) { + _log('derive day $dayId: fingerprint write failed: $e'); + return false; + } + } + Future processDay(String dayId) async { activeDays.add(dayId); _diag['active_days'] = activeDays.toList(); try { final before = await fingerprint(dayId); - if (before != null && - before == await LocalDb.deriveFingerprint(dayId)) { + if (before != null && before == await storedFingerprint(dayId)) { unchanged++; _diag['unchanged_days'] = unchanged; activeDays.remove(dayId); @@ -3571,7 +3594,7 @@ class DerivationEngine { } // Cleared before the derive so an interrupted one cannot leave a // fingerprint vouching for the row it was rewriting. - await LocalDb.putDeriveFingerprint(dayId, null); + final cleared = await storeFingerprint(dayId, null); final prepared = await _prepareTargetDay(dayId); // Override day whose raw has been pruned (≥14 d): re-deriving would // produce an empty/absent result and clobber the user's manual sleep. @@ -3587,8 +3610,11 @@ class DerivationEngine { _diag['done_days'] = done; // Stored only when the derive was a fixed point; see // [_dayInputFingerprint]. - if (before != null && before == await fingerprint(dayId)) { - await LocalDb.putDeriveFingerprint(dayId, before); + // Never over a fingerprint that could not be cleared first. + if (cleared && + before != null && + before == await fingerprint(dayId)) { + await storeFingerprint(dayId, before); } } else { _log('derive day $dayId skipped: no bounded window payload'); @@ -3803,6 +3829,7 @@ class DerivationEngine { ..['todo_days'] = 0 ..['done_days'] = 0 ..['skipped_days'] = 0 + ..['unchanged_days'] = 0 ..['active_days'] = [] ..['concurrency'] = _deriveConcurrency ..['last_error'] = null; From bda8f16ecf4920b61e61388b23784cba40750a08 Mon Sep 17 00:00:00 2001 From: abdulsaheel Date: Sat, 31 Oct 2026 13:28:56 +0530 Subject: [PATCH 13/33] stop moving the old archive when a band offload starts housekeeping could keep committing archive pages for up to 15 seconds after an offload began, so the offload's commits queued behind them. --- lib/compute/derivation_engine.dart | 1 + lib/data/db.dart | 10 ++++++++-- test/raw_archive_frames_test.dart | 9 +++++++++ 3 files changed, 18 insertions(+), 2 deletions(-) diff --git a/lib/compute/derivation_engine.dart b/lib/compute/derivation_engine.dart index eed675f13..17e126987 100644 --- a/lib/compute/derivation_engine.dart +++ b/lib/compute/derivation_engine.dart @@ -7771,6 +7771,7 @@ class DerivationEngine { // WHOOP MG's is ~500 MB of hex, so a background pass takes little. final moved = await LocalDb.migrateLegacyRawArchive( budget: Duration(seconds: background ? 3 : 15), + shouldStop: offloadActive, ); if (moved > 0) _log('moved $moved archived frame(s) to the frame store'); } catch (e) { diff --git a/lib/data/db.dart b/lib/data/db.dart index 52441624b..48cb0029a 100644 --- a/lib/data/db.dart +++ b/lib/data/db.dart @@ -7622,8 +7622,13 @@ class LocalDb { /// the same commit, so a kill between pages loses nothing and repeats /// nothing. Once it is empty the legacy table is dropped and the view /// rebuilt over the new table alone. Returns rows moved; stops after the - /// first page past [budget] and resumes on the next call. - static Future migrateLegacyRawArchive({Duration? budget}) async { + /// first page past [budget], or when [shouldStop] says a band offload has + /// started (its commits would queue behind these pages), and resumes on + /// the next call. + static Future migrateLegacyRawArchive({ + Duration? budget, + bool Function()? shouldStop, + }) async { final db = await instance; if (await _objectType(db, _rawArchiveLegacy) != 'table') return 0; final clock = Stopwatch()..start(); @@ -7677,6 +7682,7 @@ class LocalDb { }); moved += page.length; if (budget != null && clock.elapsed > budget) return moved; + if (shouldStop?.call() ?? false) return moved; } } diff --git a/test/raw_archive_frames_test.dart b/test/raw_archive_frames_test.dart index 350c918b0..a7343ec16 100644 --- a/test/raw_archive_frames_test.dart +++ b/test/raw_archive_frames_test.dart @@ -233,6 +233,15 @@ void main() { expect(await seen(), after); }); + test('the move stops after a page once an offload starts', () async { + await _legacyDb('raw_frames_offload.db', [ + for (var i = 0; i < 1200; i++) _legacyRow('', _frameHex(100 + i), i), + ]); + // No time budget: only the offload signal can stop it early. + expect(await LocalDb.migrateLegacyRawArchive(shouldStop: () => true), 500); + expect(await LocalDb.migrateLegacyRawArchive(), 700); + }); + test('a backup of the new layout restores every frame', () async { await _fresh('raw_frames_backup_src.db'); final frames = [_frameHex(1), 'Not-Hex', _frameHex(2)]; From 359b85a02e9edc31a6b85870615432813eea1d83 Mon Sep 17 00:00:00 2001 From: abdulsaheel Date: Sat, 31 Oct 2026 13:34:07 +0530 Subject: [PATCH 14/33] measure the archive window from the archive when there are no band rows a restored database can hold archived readings and no decoded rows, so a shorter window was saved and nothing was ever deleted. choosing a window now always runs the eviction, so one that failed after the setting was saved is retried by choosing it again. --- lib/compute/derivation_engine.dart | 2 +- lib/data/db.dart | 11 +++++++++++ lib/state/app_state.dart | 8 +++++--- test/substrate_archive_db_test.dart | 12 ++++++++++++ 4 files changed, 29 insertions(+), 4 deletions(-) diff --git a/lib/compute/derivation_engine.dart b/lib/compute/derivation_engine.dart index 17e126987..0a11bebb9 100644 --- a/lib/compute/derivation_engine.dart +++ b/lib/compute/derivation_engine.dart @@ -7784,7 +7784,7 @@ class DerivationEngine { } // Substrate archive retention, measured against the data edge like the // live prune. Off the commit path; vacuumIfBloated reclaims the pages. - final edge = await LocalDb.lastDecodedRecTs(); + final edge = await LocalDb.substrateArchiveEdge(); if (edge != null) { final evicted = await LocalDb.evictSubstrateArchive( await LocalDb.substrateArchivePolicy(), diff --git a/lib/data/db.dart b/lib/data/db.dart index 48cb0029a..08439b7fb 100644 --- a/lib/data/db.dart +++ b/lib/data/db.dart @@ -14425,6 +14425,17 @@ class LocalDb { int dataNowSec, ) async => SubstrateArchive.evict(await instance, policy, dataNowSec); + /// The data edge the archive window is measured from: the newest band + /// reading, or, on a restored database with an archive but no decoded rows + /// yet, the archive's own newest reading — without it a shorter window + /// would be saved and nothing ever deleted. Null only with neither. + static Future substrateArchiveEdge() async { + final band = await lastDecodedRecTs(); + if (band != null) return band; + final newest = (await substrateArchiveStats())['newest_to_ts']; + return newest is num ? newest.toInt() : null; + } + /// `{buckets, bytes, raw_bytes, oldest_from_ts, newest_to_ts}` of the /// substrate archive. static Future> substrateArchiveStats() async => diff --git a/lib/state/app_state.dart b/lib/state/app_state.dart index 34b7800de..9a13af6b0 100644 --- a/lib/state/app_state.dart +++ b/lib/state/app_state.dart @@ -2577,11 +2577,13 @@ class AppState extends ChangeNotifier { /// Set how long pruned raw readings stay in `substrate_archive`. A shorter /// window evicts the days it no longer covers right away, behind the derive /// latch so a prune that is archiving a bucket finishes first. + /// + /// The eviction runs on every set, not only on a shortening: it deletes + /// nothing past what a longer window keeps, and an eviction that failed + /// after the setting was saved is retried by choosing the same window again. Future setSubstrateArchiveWindow(SubstrateArchivePolicy next) async { - final current = await LocalDb.substrateArchivePolicy(); await LocalDb.setSubstrateArchivePolicy(next); - if (!current.keepsLongerThan(next)) return; - final edge = await LocalDb.lastDecodedRecTs(); + final edge = await LocalDb.substrateArchiveEdge(); if (edge == null) return; final evicted = await DerivationEngine.whileIdle( () => LocalDb.evictSubstrateArchive(next, edge)); diff --git a/test/substrate_archive_db_test.dart b/test/substrate_archive_db_test.dart index d007a39bc..6502b59ba 100644 --- a/test/substrate_archive_db_test.dart +++ b/test/substrate_archive_db_test.dart @@ -437,6 +437,18 @@ void main() { expect(await (await _db()).query('substrate_archive'), isEmpty); }); + test('with no decoded rows left, the window is measured from the archive', + () async { + await _seed(); + await LocalDb.pruneDecodedBeforeRecTs(cutoff, archive: policy); + final db = await _db(); + await db.delete('decoded_onehz'); + expect(await LocalDb.lastDecodedRecTs(), isNull); + final newest = + (await LocalDb.substrateArchiveStats())['newest_to_ts'] as int; + expect(await LocalDb.substrateArchiveEdge(), newest); + }); + test('evictSubstrateArchive drops buckets older than keepDays; -1 keeps ' 'all; setting the policy to 0 purges', () async { await _seed(); From 36241f9d53f37323961779b538d84e9da360cab2 Mon Sep 17 00:00:00 2001 From: abdulsaheel Date: Sat, 31 Oct 2026 15:02:55 +0530 Subject: [PATCH 15/33] never cache a partial derive as unchanged failures in a row can store the same partial row, and its fingerprint then marked the day settled, so it was skipped even after the failure cleared. only a derive that ran to completion stores one now. --- lib/compute/derivation_engine.dart | 23 ++++++++++++++++------- test/derive_unchanged_skip_test.dart | 18 ++++++++++++++++++ 2 files changed, 34 insertions(+), 7 deletions(-) diff --git a/lib/compute/derivation_engine.dart b/lib/compute/derivation_engine.dart index 0a11bebb9..940b61808 100644 --- a/lib/compute/derivation_engine.dart +++ b/lib/compute/derivation_engine.dart @@ -3605,13 +3605,17 @@ class DerivationEngine { _log('derive day $dayId skipped: override day, raw pruned — kept'); } else if (prepared != null) { _diag['prepared_days'] = (_diag['prepared_days'] as int) + 1; - await _derivePreparedDay(prepared, profile, dataNowSec, history); + final complete = await _derivePreparedDay( + prepared, profile, dataNowSec, history); done++; _diag['done_days'] = done; // Stored only when the derive was a fixed point; see - // [_dayInputFingerprint]. - // Never over a fingerprint that could not be cleared first. - if (cleared && + // [_dayInputFingerprint]. Never for a partial or declined result + // (two failures in a row can store the same row, and the day + // would then never be retried), and never over a fingerprint + // that could not be cleared first. + if (complete && + cleared && before != null && before == await fingerprint(dayId)) { await storeFingerprint(dayId, before); @@ -5753,7 +5757,11 @@ class DerivationEngine { ); } - Future _derivePreparedDay( + /// True only when the whole derive ran and its result was stored: a + /// decline that kept the existing row, or a second half that failed (a + /// partial row, however much of it was carried forward), is false, so the + /// caller never records that day as settled. + Future _derivePreparedDay( PreparedDerivationDay day, Profile profile, int dataNowSec, @@ -5952,7 +5960,7 @@ class DerivationEngine { if (_isRealDayResult(existing)) { _log('derive ${day.date}: no substrate (raw pruned) — kept the ' 'existing result rather than blanking it'); - return; + return false; } } @@ -6027,7 +6035,7 @@ class DerivationEngine { "under a day that already had real night scalars — kept the " 'existing result rather than overwriting the readiness baseline ' '(edge#305)'); - return; + return false; } } @@ -6629,6 +6637,7 @@ class DerivationEngine { ], ); } + return secondHalfOk; } /// Bank one day's (phone steps, counter ticks) pair for the (family, diff --git a/test/derive_unchanged_skip_test.dart b/test/derive_unchanged_skip_test.dart index 647b20922..e494c3780 100644 --- a/test/derive_unchanged_skip_test.dart +++ b/test/derive_unchanged_skip_test.dart @@ -97,6 +97,7 @@ void main() { tearDown(() async { DerivationEngine.debugDeriveEveryDay = false; DerivationEngine.debugFingerprintAlgoVersion = null; + DerivationEngine.debugSecondHalfHook = null; await LocalDb.close(); }); @@ -178,4 +179,21 @@ void main() { DerivationEngine.debugFingerprintAlgoVersion = kAlgoVersion + 1; expect((await _pass()).derived, pending); }, timeout: const Timeout(Duration(minutes: 15))); + + test('a day whose second half keeps failing is never cached as unchanged', + () async { + await _freshDb('unchanged_skip_partial.db'); + DerivationEngine.debugSecondHalfHook = + (_) async => throw StateError('second half down'); + // Failures in a row settle on the same partial row; none may vouch for + // it. + for (var i = 0; i < 4; i++) { + expect((await _pass()).unchanged, 0, reason: 'failing pass $i'); + } + DerivationEngine.debugSecondHalfHook = null; + // The failure cleared: the days derive again instead of being skipped. + final healed = await _pass(); + expect(healed.unchanged, 0); + expect(healed.derived, greaterThan(0)); + }, timeout: const Timeout(Duration(minutes: 10))); } From 0397aff54ffba546d61b6aff33a55c7f61d09f5b Mon Sep 17 00:00:00 2001 From: abdulsaheel Date: Sat, 31 Oct 2026 15:07:25 +0530 Subject: [PATCH 16/33] run the archive window check only on this phone's database, and never fatally it ran inside the schema repair that also opens export copies, where it wrote a setting into the copy, and a failure there could get the database quarantined. it now runs only when the app opens its own database, and a failure is logged and retried on the next open. --- lib/data/db.dart | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/lib/data/db.dart b/lib/data/db.dart index 08439b7fb..f50188633 100644 --- a/lib/data/db.dart +++ b/lib/data/db.dart @@ -18,7 +18,7 @@ import 'dart:isolate'; import 'dart:typed_data'; import 'package:crypto/crypto.dart' show sha1; -import 'package:flutter/foundation.dart' show visibleForTesting; +import 'package:flutter/foundation.dart' show debugPrint, visibleForTesting; import 'package:openstrap_protocol/openstrap_protocol.dart' as proto; import 'package:path/path.dart' as p; import 'package:path_provider/path_provider.dart'; @@ -1234,6 +1234,15 @@ class LocalDb { }, onOpen: (db) async { await _repairOpenSchema(db); + // This phone's own database only: [_repairOpenSchema] also opens + // export copies, where it would write a setting the user never chose. + // Best effort — a failure must not get the database quarantined; the + // next open retries it. + try { + await SubstrateArchive.keepLegacyWindowOnce(db); + } catch (e) { + debugPrint('[db] archive window check skipped: $e'); + } }, version: schemaVersion, ); @@ -1342,7 +1351,6 @@ class LocalDb { // / notifications all existing. DROP+CREATE so a shape change takes effect. await _ensureCoachViews(db); await _dropRawStore(db); - await SubstrateArchive.keepLegacyWindowOnce(db); } /// Older WHOOP CSV imports filed the export's absolute skin temp (°C) under From 233552995025db680ece14c545d9b3b4257d6542 Mon Sep 17 00:00:00 2001 From: abdulsaheel Date: Sat, 31 Oct 2026 15:07:25 +0530 Subject: [PATCH 17/33] keep a year for any database or backup from before the 90-day default one helper decides it: no stored setting, not yet opened by this build, and some history to lose, whether archived buckets or band readings a later prune would archive. the first open uses it, and so does a restore, so an old backup with no setting comes back with its year instead of losing the older half of what it brings. --- lib/data/substrate_archive.dart | 68 ++++++++++++++++++------- test/substrate_archive_db_test.dart | 6 ++- test/substrate_archive_window_test.dart | 42 ++++++++++++++- 3 files changed, 96 insertions(+), 20 deletions(-) diff --git a/lib/data/substrate_archive.dart b/lib/data/substrate_archive.dart index c1fe8c3fc..b01d2c1e8 100644 --- a/lib/data/substrate_archive.dart +++ b/lib/data/substrate_archive.dart @@ -185,27 +185,52 @@ abstract final class SubstrateArchive { return out; } - /// Once per install, on the first open of a build whose default is shorter - /// than a year: an archive that already has buckets but no stored setting - /// was being kept for 365 days, and the new default would delete everything - /// older than 90 on the next housekeeping pass, with no confirmation and no - /// way back. That archive keeps its year; the user can still shorten it from - /// data settings. An empty archive gets the new default. + /// The window a database that predates the 90-day default was getting: a + /// year. Such a database has no stored setting and already holds history — + /// archived buckets, or band readings a later prune would archive — that + /// the new default would delete with no confirmation and no way back. Null + /// when it does not predate it (a fresh install, or a setting was chosen). + static Future legacyWindow( + DatabaseExecutor ex) async { + Future any(String sql) async { + try { + return (await ex.rawQuery(sql)).isNotEmpty; + } on DatabaseException catch (e) { + if (e.isNoSuchTableError()) return false; + rethrow; + } + } + + // A stored setting, or a database that has already been opened by a + // build with the new default (it carries the marker), is not legacy: a + // backup of one with no setting was getting the 90 days. + if (await any('SELECT 1 FROM compute_freshness ' + "WHERE key IN ('$policyKey', '$_defaultCheckedKey') LIMIT 1")) { + return null; + } + final hasHistory = await any('SELECT 1 FROM $table LIMIT 1') || + await any('SELECT 1 FROM decoded_onehz LIMIT 1'); + return hasHistory ? const SubstrateArchivePolicy(365) : null; + } + + /// Once per install, on the first open of a build with the 90-day default: + /// a database that predates it keeps its year ([legacyWindow]); the user can + /// still shorten it from data settings. A fresh one gets the new default. + static const String _defaultCheckedKey = + 'substrate_archive_policy_default_checked'; + static Future keepLegacyWindowOnce(Database db) async { - const marker = 'substrate_archive_policy_default_checked'; + const marker = _defaultCheckedKey; final done = await db.query('compute_freshness', columns: ['key'], where: 'key = ?', whereArgs: [marker], limit: 1); if (done.isNotEmpty) return; await db.transaction((txn) async { - final stored = await txn.query('compute_freshness', - columns: ['key'], where: 'key = ?', whereArgs: [policyKey], limit: 1); - final hasArchive = - (await txn.rawQuery('SELECT 1 FROM $table LIMIT 1')).isNotEmpty; + final legacy = await legacyWindow(txn); final now = DateTime.now().millisecondsSinceEpoch; - if (stored.isEmpty && hasArchive) { + if (legacy != null) { await txn.insert('compute_freshness', { 'key': policyKey, - 'payload_json': const SubstrateArchivePolicy(365).toJson(), + 'payload_json': legacy.toJson(), 'updated_at': now, }); } @@ -266,10 +291,19 @@ abstract final class SubstrateArchive { if (e.isNoSuchTableError()) return; rethrow; } - if (theirs.isEmpty) return; - final adopted = SubstrateArchivePolicy.fromJson( - theirs.first['payload_json'] as String?, - ); + // A backup from before the setting existed has no row; its archive was + // being kept for a year, and restoring it under the 90-day default would + // drop the older part of what it brings. + final SubstrateArchivePolicy adopted; + if (theirs.isNotEmpty) { + adopted = SubstrateArchivePolicy.fromJson( + theirs.first['payload_json'] as String?, + ); + } else { + final legacy = await legacyWindow(src); + if (legacy == null) return; + adopted = legacy; + } await db.transaction((txn) async { final ours = await txn.query( 'compute_freshness', diff --git a/test/substrate_archive_db_test.dart b/test/substrate_archive_db_test.dart index 6502b59ba..32a3b89ee 100644 --- a/test/substrate_archive_db_test.dart +++ b/test/substrate_archive_db_test.dart @@ -89,10 +89,14 @@ Future _clear() async { 'decoded_onehz', 'decoded_rr', 'substrate_archive', - 'compute_freshness', ]) { await db.delete(t); } + // Everything but the marker every opened phone database carries: without it + // an export here would read as a backup from before the 90-day default. + await db.delete('compute_freshness', + where: 'key != ?', + whereArgs: ['substrate_archive_policy_default_checked']); SubstrateArchive.debugCorruptEncode = false; SubstrateArchive.debugBeforeBucket = null; SubstrateArchive.debugThrowInBucket = null; diff --git a/test/substrate_archive_window_test.dart b/test/substrate_archive_window_test.dart index a425d3768..381a6ee80 100644 --- a/test/substrate_archive_window_test.dart +++ b/test/substrate_archive_window_test.dart @@ -41,10 +41,11 @@ void main() { late Database db; setUpAll(sqfliteFfiInit); setUp(() async { - db = await databaseFactoryFfi.openDatabase(inMemoryDatabasePath); + db = await databaseFactoryFfi.openDatabase(inMemoryDatabasePath, + options: OpenDatabaseOptions(singleInstance: false)); await db.execute('CREATE TABLE compute_freshness (key TEXT PRIMARY KEY, ' 'payload_json TEXT NOT NULL, updated_at INTEGER NOT NULL)'); - await db.execute('CREATE TABLE ${SubstrateArchive.table} (day TEXT)'); + await db.execute('CREATE TABLE ${SubstrateArchive.table} (utc_day INTEGER, day TEXT)'); }); tearDown(() => db.close()); @@ -63,6 +64,43 @@ void main() { expect((await SubstrateArchive.readPolicy(db)).keepDays, 90); }); + test('an empty archive with band readings already stored keeps a year', + () async { + await db.execute('CREATE TABLE decoded_onehz (rec_ts INTEGER)'); + await db.insert('decoded_onehz', {'rec_ts': 1750000000}); + await SubstrateArchive.keepLegacyWindowOnce(db); + expect((await SubstrateArchive.readPolicy(db)).keepDays, 365); + }); + + test('a backup from before the setting restores with its year', () async { + final backup = await databaseFactoryFfi.openDatabase( + inMemoryDatabasePath, + options: OpenDatabaseOptions(singleInstance: false)); + addTearDown(backup.close); + await backup.execute('CREATE TABLE compute_freshness (key TEXT PRIMARY ' + 'KEY, payload_json TEXT NOT NULL, updated_at INTEGER NOT NULL)'); + await backup.execute('CREATE TABLE ${SubstrateArchive.table} (utc_day INTEGER, day TEXT)'); + // Fresh install: the check already ran and found nothing. + await SubstrateArchive.keepLegacyWindowOnce(db); + await SubstrateArchive.adoptPolicy(backup, db); + expect((await SubstrateArchive.readPolicy(db)).keepDays, 90, + reason: 'a backup with no history brings no window'); + await backup.insert(SubstrateArchive.table, {'day': '2025-11-01'}); + // A backup of a phone already on this build, with no setting chosen, + // was getting the default: nothing to adopt. + await backup.insert('compute_freshness', { + 'key': 'substrate_archive_policy_default_checked', + 'payload_json': '{}', + 'updated_at': 0, + }); + await SubstrateArchive.adoptPolicy(backup, db); + expect((await SubstrateArchive.readPolicy(db)).keepDays, 90); + // One from before this build keeps its year. + await backup.delete('compute_freshness'); + await SubstrateArchive.adoptPolicy(backup, db); + expect((await SubstrateArchive.readPolicy(db)).keepDays, 365); + }); + test('a window the user picked is left alone', () async { await db.insert(SubstrateArchive.table, {'day': '2025-11-01'}); await db.insert('compute_freshness', { From 897bb8a5d3d2089885781a7fb721cbbac2bccf4f Mon Sep 17 00:00:00 2001 From: abdulsaheel Date: Sat, 31 Oct 2026 18:41:45 +0530 Subject: [PATCH 18/33] count beat-only records as history when picking the archive window --- lib/data/substrate_archive.dart | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/lib/data/substrate_archive.dart b/lib/data/substrate_archive.dart index b01d2c1e8..628a43796 100644 --- a/lib/data/substrate_archive.dart +++ b/lib/data/substrate_archive.dart @@ -209,7 +209,9 @@ abstract final class SubstrateArchive { return null; } final hasHistory = await any('SELECT 1 FROM $table LIMIT 1') || - await any('SELECT 1 FROM decoded_onehz LIMIT 1'); + await any('SELECT 1 FROM decoded_onehz LIMIT 1') || + // Beat-only records (no 1 Hz row) are history too. + await any('SELECT 1 FROM decoded_rr LIMIT 1'); return hasHistory ? const SubstrateArchivePolicy(365) : null; } From 0796dbfccb0cfb4990d725cee79d4489dc0f6d75 Mon Sep 17 00:00:00 2001 From: abdulsaheel Date: Sat, 31 Oct 2026 20:50:32 +0530 Subject: [PATCH 19/33] name the archive view so an older build can still open the database the build before this layout runs CREATE INDEX IF NOT EXISTS ... ON raw_archive on every open, and sqlite refuses to index a view before it looks at the index name, so with the view under that name a rolled-back build failed to open, quarantined the file and could not salvage it. the view is raw_archive_view now: the old build makes itself an empty raw_archive table, its index line is a no-op, and the next open of this build folds whatever it archived back in. --- lib/coach/coach_db.dart | 1 + lib/compute/inputs/colmi_inputs.dart | 4 +- lib/compute/inputs/ultrahuman_inputs.dart | 2 +- lib/data/db.dart | 48 ++++++++++++------ test/colmi_link_test.dart | 2 +- test/db_migration_ladder_test.dart | 12 ++--- test/db_p0_fixes_test.dart | 2 +- .../db_v42_retention_and_provenance_test.dart | 16 +++--- test/miband_link_test.dart | 2 +- test/mt12_dyn_accel_test.dart | 3 +- test/oura_link_test.dart | 2 +- test/pebble_link_test.dart | 2 +- test/raw_archive_frames_test.dart | 49 +++++++++++++++++-- test/raw_archive_test.dart | 2 +- test/session_devices_test.dart | 2 +- test/startup_and_rebuild_recovery_test.dart | 11 +++-- test/synthetic_colmi_day_test.dart | 4 +- test/ultrahuman_link_test.dart | 4 +- 18 files changed, 117 insertions(+), 51 deletions(-) diff --git a/lib/coach/coach_db.dart b/lib/coach/coach_db.dart index ae0a235a9..a3bab2501 100644 --- a/lib/coach/coach_db.dart +++ b/lib/coach/coach_db.dart @@ -98,6 +98,7 @@ class CoachDb { 'session_score_mask', 'session_sensor', // raw / decoded substrate 'raw_records', 'raw_archive', 'raw_archive_frames', 'raw_archive_legacy', + 'raw_archive_view', 'decoded_onehz', 'decoded_rr', 'samples', 'events', 'band_events', 'band_battery', 'substrate_archive', 'device_coverage', 'signal_priority', diff --git a/lib/compute/inputs/colmi_inputs.dart b/lib/compute/inputs/colmi_inputs.dart index c95f559ff..b9823df78 100644 --- a/lib/compute/inputs/colmi_inputs.dart +++ b/lib/compute/inputs/colmi_inputs.dart @@ -56,7 +56,7 @@ Future> colmiSkinTemps( // ring's history (kColmiHistoryDays) later: scan only those captures. for (final r in await db.rawQuery( 'SELECT hex, COALESCE(rec_ts * 1000, captured_at) AS anchor_ms ' - 'FROM raw_archive WHERE device_id = ? ' + 'FROM ${LocalDb.rawArchiveView} WHERE device_id = ? ' "AND reason = 'colmi_big_0x${kColmiBigTemperature.toRadixString(16)}' " 'AND captured_at >= ? AND captured_at <= ?', [ @@ -143,7 +143,7 @@ const Set kColmiNightRows = { Future colmiSleepUndecoded(String deviceId) async { final db = await LocalDb.instance; final rows = await db.rawQuery( - 'SELECT hex FROM raw_archive WHERE device_id = ? AND reason = ? ' + 'SELECT hex FROM ${LocalDb.rawArchiveView} WHERE device_id = ? AND reason = ? ' 'ORDER BY captured_at DESC LIMIT 1', [deviceId, 'colmi_cmd_0x${kColmiCmdSetTime.toRadixString(16).padLeft(2, '0')}'], ); diff --git a/lib/compute/inputs/ultrahuman_inputs.dart b/lib/compute/inputs/ultrahuman_inputs.dart index 720c38fea..dd467784d 100644 --- a/lib/compute/inputs/ultrahuman_inputs.dart +++ b/lib/compute/inputs/ultrahuman_inputs.dart @@ -49,7 +49,7 @@ Future> ultrahumanRecords( final db = await LocalDb.instance; final out = []; for (final r in await db.rawQuery( - "SELECT hex FROM raw_archive WHERE device_id = ? " + "SELECT hex FROM ${LocalDb.rawArchiveView} WHERE device_id = ? " "AND reason = 'ultrahuman_record' AND rec_ts >= ? AND rec_ts <= ? " 'ORDER BY rec_ts', [deviceId, from, to], diff --git a/lib/data/db.dart b/lib/data/db.dart index f50188633..6d376a804 100644 --- a/lib/data/db.dart +++ b/lib/data/db.dart @@ -7421,6 +7421,17 @@ class LocalDb { /// into [rawArchiveTable] in pages by [migrateLegacyRawArchive]. static const String _rawArchiveLegacy = 'raw_archive_legacy'; + /// The view every reader goes through ([_ensureRawArchiveView]). NOT named + /// `raw_archive`: an older build opening this file after a rollback runs + /// `CREATE TABLE IF NOT EXISTS raw_archive` and then `CREATE INDEX IF NOT + /// EXISTS idx_raw_archive_captured ON raw_archive`, and SQLite refuses to + /// index a view before it looks at the name, so a view under the old name + /// failed that open and got the database quarantined. With the name free, + /// the old build makes itself an empty table, the index line is a no-op + /// (the name is taken by [rawArchiveTable]'s), and the next open of this + /// build folds whatever it wrote back in ([_ensureRawArchiveStore]). + static const String rawArchiveView = 'raw_archive_view'; + /// The frame as the `raw_archive` view's `hex` column: the stored text for a /// frame kept verbatim, else the bytes in lowercase hex — exactly the /// string [rawArchiveRow] was handed. @@ -7492,9 +7503,15 @@ class LocalDb { /// Everything in this build writes through [rawArchiveRow] instead. static Future _ensureRawArchiveView(DatabaseExecutor db) async { final legacy = await _objectType(db, _rawArchiveLegacy) == 'table'; - await db.execute('DROP VIEW IF EXISTS raw_archive'); + await db.execute('DROP TRIGGER IF EXISTS raw_archive_insert'); + await db.execute('DROP TRIGGER IF EXISTS raw_archive_delete'); + // The view's old name, from a build of this layout before the rename. + if (await _objectType(db, 'raw_archive') == 'view') { + await db.execute('DROP VIEW raw_archive'); + } + await db.execute('DROP VIEW IF EXISTS $rawArchiveView'); await db.execute( - 'CREATE VIEW raw_archive AS ' + 'CREATE VIEW $rawArchiveView AS ' 'SELECT device_id, $_rawArchiveHexSql AS hex, counter, packet_type, ' 'rec_ts, captured_at, reason FROM $rawArchiveTable' '${legacy ? ' UNION ALL SELECT device_id, hex, counter, packet_type, ' @@ -7502,7 +7519,7 @@ class LocalDb { ); await db.execute( 'CREATE TRIGGER IF NOT EXISTS raw_archive_insert ' - 'INSTEAD OF INSERT ON raw_archive BEGIN ' + 'INSTEAD OF INSERT ON $rawArchiveView BEGIN ' 'INSERT OR IGNORE INTO $rawArchiveTable ' '(device_id, hash, counter, packet_type, rec_ts, captured_at, reason, hex) ' "VALUES (COALESCE(NEW.device_id, '$kPrimaryDeviceId'), " @@ -7511,7 +7528,7 @@ class LocalDb { ); await db.execute( 'CREATE TRIGGER IF NOT EXISTS raw_archive_delete ' - 'INSTEAD OF DELETE ON raw_archive BEGIN ' + 'INSTEAD OF DELETE ON $rawArchiveView BEGIN ' 'DELETE FROM $rawArchiveTable WHERE device_id = OLD.device_id ' 'AND $_rawArchiveHexSql = OLD.hex; ' '${legacy ? 'DELETE FROM $_rawArchiveLegacy WHERE device_id = ' @@ -7558,12 +7575,13 @@ class LocalDb { }); } await _createRawArchiveFrames(db); - if (await _objectType(db, 'raw_archive') != 'view') { + if (await _objectType(db, rawArchiveView) != 'view' || + await _objectType(db, 'raw_archive') == 'view') { await _ensureRawArchiveView(db); } else if ((await _objectType(db, _rawArchiveLegacy) == 'table') != (await db.rawQuery( "SELECT 1 FROM sqlite_master WHERE type = 'view' " - "AND name = 'raw_archive' AND sql LIKE '%$_rawArchiveLegacy%'", + "AND name = '$rawArchiveView' AND sql LIKE '%$_rawArchiveLegacy%'", )) .isNotEmpty) { // The view and the legacy table disagree (a restore, a crash between @@ -7655,7 +7673,7 @@ class LocalDb { } await txn.execute('DROP TRIGGER IF EXISTS raw_archive_insert'); await txn.execute('DROP TRIGGER IF EXISTS raw_archive_delete'); - await txn.execute('DROP VIEW IF EXISTS raw_archive'); + await txn.execute('DROP VIEW IF EXISTS $rawArchiveView'); await txn.execute('DROP TABLE $_rawArchiveLegacy'); // Rebuilt in the same transaction: between a commit and a rebuild // outside it there is no raw_archive at all, and a reader on @@ -8211,8 +8229,11 @@ class LocalDb { // Both are cheap and both are required: a DB whose ladder has not created // the archive yet (or is mid-ladder) must no-op rather than throw — a throw // in here rolls the WHOLE upgrade back and quarantines the database. - for (final t in const ['raw_archive', 'decoded_onehz']) { - // `raw_archive` is a view over [rawArchiveTable] once converted. + // The archive is the `raw_archive` table on the ladder's older layouts, + // and [rawArchiveView] over [rawArchiveTable] once converted. + final framesLayout = await _objectType(db, rawArchiveView) == 'view'; + for (final t in [framesLayout ? rawArchiveView : 'raw_archive', + 'decoded_onehz']) { final present = await db.rawQuery( "SELECT 1 FROM sqlite_master WHERE type IN ('table', 'view') " 'AND name=?', @@ -8225,7 +8246,6 @@ class LocalDb { // the same (String, String) slot, marked by [_redriveRowidCursor]; a // position from the old hex-ordered pass restarts at the top, which the // replay's skip-what-is-decoded rule makes safe. - final framesLayout = await _objectType(db, 'raw_archive') == 'view'; var afterRowid = framesLayout && startDevice == _redriveRowidCursor ? int.tryParse(startHex) ?? 0 : 0; @@ -8240,7 +8260,7 @@ class LocalDb { // rekey (v51): this function runs from the oldV<44 rung too, i.e. BEFORE // the v51 rekey, so at that point `raw_archive` is still hex-keyed and the // row-value comparison below must fall back to comparing `hex` alone. - final preArchiveDeviceKey = + final preArchiveDeviceKey = !framesLayout && !(await _columnsOf(db, 'raw_archive')).contains('device_id'); final marks = List.filled(reasons.length, '?').join(','); @@ -8491,11 +8511,11 @@ class LocalDb { final db = await instance; final count = Sqflite.firstIntValue( - await db.rawQuery('SELECT COUNT(*) FROM raw_archive'), + await db.rawQuery('SELECT COUNT(*) FROM $rawArchiveView'), ) ?? 0; final byReason = await db.rawQuery( - 'SELECT reason, COUNT(*) AS n FROM raw_archive ' + 'SELECT reason, COUNT(*) AS n FROM $rawArchiveView ' 'GROUP BY reason ORDER BY n DESC, reason ASC', ); return { @@ -9874,7 +9894,7 @@ class LocalDb { await add( 'wearable_archive', 'SELECT COUNT(*), MAX(captured_at), TOTAL(captured_at), ' - 'TOTAL(LENGTH(hex)) FROM raw_archive WHERE device_id = ?', + 'TOTAL(LENGTH(hex)) FROM $rawArchiveView WHERE device_id = ?', [wearableDeviceId], ); } diff --git a/test/colmi_link_test.dart b/test/colmi_link_test.dart index cfe1df23e..a4b86e388 100644 --- a/test/colmi_link_test.dart +++ b/test/colmi_link_test.dart @@ -123,7 +123,7 @@ void main() { test('every reply is banked verbatim, never re-drivable', () async { await sync(); final db = await LocalDb.instance; - final archive = await db.query('raw_archive'); + final archive = await db.query(LocalDb.rawArchiveView); final reasons = archive.map((a) => a['reason']).toSet(); expect(reasons, containsAll(['colmi_cmd_0x03', 'colmi_cmd_0x15', 'colmi_big_0x27'])); diff --git a/test/db_migration_ladder_test.dart b/test/db_migration_ladder_test.dart index 886d662b7..5d6f3019f 100644 --- a/test/db_migration_ladder_test.dart +++ b/test/db_migration_ladder_test.dart @@ -1414,7 +1414,7 @@ void main() { final db = await LocalDb.instance; for (final t in const [ - 'raw_archive', 'band_events', 'events', 'band_battery', + LocalDb.rawArchiveView, 'band_events', 'events', 'band_battery', ]) { final rows = await db.query(t); expect(rows.length, 1, reason: '$t row count must be unchanged'); @@ -1446,7 +1446,7 @@ void main() { final db2 = await LocalDb.instance; expect(LocalDb.lastRebuild, isNull); for (final t in const [ - 'raw_archive', 'band_events', 'events', 'band_battery', + LocalDb.rawArchiveView, 'band_events', 'events', 'band_battery', ]) { expect((await db2.query(t)).length, 1); } @@ -1506,7 +1506,7 @@ void main() { expect(await _openThroughLocalDb(name), LocalDb.schemaVersion); final db = await LocalDb.instance; - Future archiveAs(String deviceId) => db.insert('raw_archive', { + Future archiveAs(String deviceId) => db.insert(LocalDb.rawArchiveView, { 'device_id': deviceId, 'hex': 'deadbeef', 'counter': 1, @@ -1518,10 +1518,10 @@ void main() { await archiveAs(''); await archiveAs('device-b'); - expect(await db.query('raw_archive'), hasLength(2)); + expect(await db.query(LocalDb.rawArchiveView), hasLength(2)); await archiveAs(''); // re-flood - expect(await db.query('raw_archive'), hasLength(2)); + expect(await db.query(LocalDb.rawArchiveView), hasLength(2)); }, ); @@ -1541,7 +1541,7 @@ void main() { '01570c500b010c020c0100000000000000000000000000000000000000000000' '010053748080000000fcaf98c0000000'; for (final deviceId in const ['', 'device-b']) { - await db.insert('raw_archive', { + await db.insert(LocalDb.rawArchiveView, { 'device_id': deviceId, 'hex': hex, 'counter': 21510400, diff --git a/test/db_p0_fixes_test.dart b/test/db_p0_fixes_test.dart index c809d937d..c0efc5f06 100644 --- a/test/db_p0_fixes_test.dart +++ b/test/db_p0_fixes_test.dart @@ -465,7 +465,7 @@ void main() { expect(b3.first['rr_ts_ms'], t3 * 1000); // Both archived frames survived the restore. - final archived = await db.query('raw_archive', orderBy: 'hex ASC'); + final archived = await db.query(LocalDb.rawArchiveView, orderBy: 'hex ASC'); expect([for (final a in archived) a['hex']], ['deadbeef01', 'deadbeef02'], reason: 'raw_archive was missing from the import merge list, so a ' diff --git a/test/db_v42_retention_and_provenance_test.dart b/test/db_v42_retention_and_provenance_test.dart index 20d87ca14..f41ebb5ef 100644 --- a/test/db_v42_retention_and_provenance_test.dart +++ b/test/db_v42_retention_and_provenance_test.dart @@ -376,7 +376,7 @@ void main() { // 600 rows at 1-in-60 ⇒ counters 0, 60, … 540. expect( await _count( - "SELECT COUNT(*) FROM raw_archive WHERE reason = 'undecodable_rec_v20' " + "SELECT COUNT(*) FROM ${LocalDb.rawArchiveView} WHERE reason = 'undecodable_rec_v20' " 'AND captured_at < ?', [cutoffSec * 1000], ), @@ -384,13 +384,13 @@ void main() { ); expect( await _count( - "SELECT COUNT(*) FROM raw_archive WHERE reason = 'undecodable_rec_v25'", + "SELECT COUNT(*) FROM ${LocalDb.rawArchiveView} WHERE reason = 'undecodable_rec_v25'", ), 1, ); expect( await _count( - "SELECT COUNT(*) FROM raw_archive WHERE reason = 'undecodable_rec_v20' " + "SELECT COUNT(*) FROM ${LocalDb.rawArchiveView} WHERE reason = 'undecodable_rec_v20' " 'AND captured_at >= ?', [cutoffSec * 1000], ), @@ -398,7 +398,7 @@ void main() { ); // Idempotent: running it again takes nothing more. await LocalDb.pruneDecodedBeforeRecTs(cutoffSec); - expect(await _count('SELECT COUNT(*) FROM raw_archive'), 12); + expect(await _count('SELECT COUNT(*) FROM ${LocalDb.rawArchiveView}'), 12); }); test('prune cursor only moves forward', () async { @@ -557,15 +557,15 @@ void main() { expect(await _count('SELECT COUNT(*) FROM metric_series_version'), derived); - final before = await _count('SELECT COUNT(*) FROM raw_archive'); + final before = await _count('SELECT COUNT(*) FROM ${LocalDb.rawArchiveView}'); final byReason = await db.rawQuery( - 'SELECT reason, COUNT(*) AS n FROM raw_archive GROUP BY reason', + 'SELECT reason, COUNT(*) AS n FROM ${LocalDb.rawArchiveView} GROUP BY reason', ); // Thin at "now", so the whole export is behind the retention edge. final thinned = await LocalDb.thinRawArchiveBefore( DateTime.now().millisecondsSinceEpoch, ); - final after = await _count('SELECT COUNT(*) FROM raw_archive'); + final after = await _count('SELECT COUNT(*) FROM ${LocalDb.rawArchiveView}'); // ignore: avoid_print print('[v42] ${p.basename(src)} archive $before → $after ' '(-$thinned) by_reason=$byReason'); @@ -575,7 +575,7 @@ void main() { if (r['reason'] == 'undecodable_rec_v20') continue; expect( await _count( - 'SELECT COUNT(*) FROM raw_archive WHERE reason = ?', + 'SELECT COUNT(*) FROM ${LocalDb.rawArchiveView} WHERE reason = ?', [r['reason']], ), r['n'], diff --git a/test/miband_link_test.dart b/test/miband_link_test.dart index b8bb12233..6fe190c35 100644 --- a/test/miband_link_test.dart +++ b/test/miband_link_test.dart @@ -76,7 +76,7 @@ void main() { nowSeconds: () => _nowSec, ); final db = await LocalDb.instance; - final archive = await db.query('raw_archive', orderBy: 'hex'); + final archive = await db.query(LocalDb.rawArchiveView, orderBy: 'hex'); expect(archive, hasLength(3)); expect( archive.map((a) => a['reason']).toSet(), diff --git a/test/mt12_dyn_accel_test.dart b/test/mt12_dyn_accel_test.dart index 99a65481c..c7390f3f2 100644 --- a/test/mt12_dyn_accel_test.dart +++ b/test/mt12_dyn_accel_test.dart @@ -130,7 +130,8 @@ void main() { for (final e in before.entries) { final after = - (await db.rawQuery('SELECT COUNT(*) AS n FROM ${e.key}')) + (await db.rawQuery('SELECT COUNT(*) AS n FROM ' + '${e.key == 'raw_archive' ? LocalDb.rawArchiveView : e.key}')) .first['n'] as int; // GREATER-OR-EQUAL, not equal: the v44 rung replays archived records // into decoded_onehz, so that table may legitimately GAIN rows. No diff --git a/test/oura_link_test.dart b/test/oura_link_test.dart index cdde013f7..ed0e2fa61 100644 --- a/test/oura_link_test.dart +++ b/test/oura_link_test.dart @@ -98,7 +98,7 @@ Future _run(List>> batches) async { return ReplayBandLinkResult( writes: [for (final w in link.writes) w.$2], onehz: await db.query('decoded_onehz', orderBy: 'ts_ms'), - archive: await db.query('raw_archive', orderBy: 'captured_at, hex'), + archive: await db.query(LocalDb.rawArchiveView, orderBy: 'captured_at, hex'), ); } diff --git a/test/pebble_link_test.dart b/test/pebble_link_test.dart index 7c4eb6d2b..c9b121645 100644 --- a/test/pebble_link_test.dart +++ b/test/pebble_link_test.dart @@ -51,7 +51,7 @@ void main() { nowSeconds: () => ts + 3600, ); final db = await LocalDb.instance; - expect((await db.query('raw_archive')).length, greaterThanOrEqualTo(1)); + expect((await db.query(LocalDb.rawArchiveView)).length, greaterThanOrEqualTo(1)); final hr = await db.query('decoded_onehz'); expect(hr.single['hr'], 61); expect(hr.single['source'], 'pebble'); diff --git a/test/raw_archive_frames_test.dart b/test/raw_archive_frames_test.dart index a7343ec16..6e81c61c9 100644 --- a/test/raw_archive_frames_test.dart +++ b/test/raw_archive_frames_test.dart @@ -47,7 +47,7 @@ Future _fresh(String name) async { Future>> _viewRows() async => (await LocalDb.instance).rawQuery( 'SELECT device_id, hex, counter, packet_type, rec_ts, captured_at, ' - 'reason FROM raw_archive ORDER BY device_id, hex', + 'reason FROM ${LocalDb.rawArchiveView} ORDER BY device_id, hex', ); Future _bytes(Database db, List names) async { @@ -149,11 +149,11 @@ void main() { test('plain inserts and deletes on the old name still work', () async { await _fresh('raw_frames_compat.db'); final db = await LocalDb.instance; - await db.insert('raw_archive', _legacyRow('', 'beef', 7)); - await db.insert('raw_archive', _legacyRow('', 'beef', 7), + await db.insert(LocalDb.rawArchiveView, _legacyRow('', 'beef', 7)); + await db.insert(LocalDb.rawArchiveView, _legacyRow('', 'beef', 7), conflictAlgorithm: ConflictAlgorithm.ignore); expect(await _viewRows(), hasLength(1)); - await db.delete('raw_archive', where: 'hex = ?', whereArgs: ['beef']); + await db.delete(LocalDb.rawArchiveView, where: 'hex = ?', whereArgs: ['beef']); expect(await _viewRows(), isEmpty); }); @@ -205,7 +205,7 @@ void main() { isEmpty, ); final view = await db.rawQuery( - "SELECT sql FROM sqlite_master WHERE name = 'raw_archive'"); + "SELECT sql FROM sqlite_master WHERE name = '${LocalDb.rawArchiveView}'"); expect('${view.single['sql']}', isNot(contains('legacy'))); final after = await seen(); // The mid-move duplicate collapsed; counter differs, so compare keys. @@ -242,6 +242,45 @@ void main() { expect(await LocalDb.migrateLegacyRawArchive(), 700); }); + test('an older build can still open the file, and its frames come back', + () async { + await _fresh('raw_frames_rollback.db'); + await LocalDb.close(); + LocalDb.dbName = 'raw_frames_rollback.db'; + await LocalDb.archiveRawRecord(_rec(1, _frameHex(1))); + await LocalDb.close(); + // What the build before this layout runs on every open, verbatim. With + // a view named raw_archive the index line failed ("views may not be + // indexed") and the open with it. + final old = await databaseFactory.openDatabase( + await _path('raw_frames_rollback.db')); + await old.execute(''' + CREATE TABLE IF NOT EXISTS raw_archive ( + device_id TEXT NOT NULL DEFAULT '', + hex TEXT NOT NULL, + counter INTEGER, + packet_type INTEGER NOT NULL, + rec_ts INTEGER, + captured_at INTEGER NOT NULL, + reason TEXT NOT NULL, + PRIMARY KEY (device_id, hex) + ) + '''); + await old.execute('CREATE INDEX IF NOT EXISTS idx_raw_archive_captured ' + 'ON raw_archive(captured_at DESC)'); + // The old build archives a frame of its own. + await old.insert('raw_archive', _legacyRow('', _frameHex(2), 2)); + await old.close(); + + // Back on this build: both frames are there. + final keys = [ + for (final r in await (await LocalDb.instance).rawQuery( + 'SELECT hex FROM ${LocalDb.rawArchiveView} ORDER BY hex')) + r['hex'], + ]; + expect(keys, [_frameHex(1), _frameHex(2)]..sort()); + }); + test('a backup of the new layout restores every frame', () async { await _fresh('raw_frames_backup_src.db'); final frames = [_frameHex(1), 'Not-Hex', _frameHex(2)]; diff --git a/test/raw_archive_test.dart b/test/raw_archive_test.dart index 72daaf58f..389f95712 100644 --- a/test/raw_archive_test.dart +++ b/test/raw_archive_test.dart @@ -360,7 +360,7 @@ void main() { // Opening runs the whole ladder, v44 included. final db = await LocalDb.instance; final archived = (await db.rawQuery( - 'SELECT COUNT(*) AS n FROM raw_archive WHERE reason IN ' + 'SELECT COUNT(*) AS n FROM ${LocalDb.rawArchiveView} WHERE reason IN ' '(${List.filled(LocalDb.redrivableArchiveReasons.length, '?').join(',')})', LocalDb.redrivableArchiveReasons, )).first['n']; diff --git a/test/session_devices_test.dart b/test/session_devices_test.dart index c2f1528f1..a01219cb7 100644 --- a/test/session_devices_test.dart +++ b/test/session_devices_test.dart @@ -72,7 +72,7 @@ void main() { [('body_temp', 36.5), ('body_temp', 36.52)].reversed.toList()); expect(rows.first['ts_ms'], morning.millisecondsSinceEpoch); expect(rows.every((r) => r['attribution'] == kThermometer.label), isTrue); - final archive = await (await LocalDb.instance).query('raw_archive'); + final archive = await (await LocalDb.instance).query(LocalDb.rawArchiveView); expect(archive.length, 3, reason: 'every indication is banked'); }); diff --git a/test/startup_and_rebuild_recovery_test.dart b/test/startup_and_rebuild_recovery_test.dart index ecfa4d8dc..df34ef3b3 100644 --- a/test/startup_and_rebuild_recovery_test.dart +++ b/test/startup_and_rebuild_recovery_test.dart @@ -300,7 +300,7 @@ void main() { 'payload_json': '{"mean":51}', 'updated_at': 1, }); - await db.insert('raw_archive', { + await db.insert(LocalDb.rawArchiveView, { 'hex': 'deadbeef', 'counter': 7, 'packet_type': 0x2f, @@ -375,7 +375,10 @@ void main() { reason: 'the old file is parked, never deleted'); for (final t in salvaged) { - expect(await db.query(t), hasLength(1), reason: '$t did not come back'); + // The archive reads back through its view on this layout. + final live = t == 'raw_archive' ? LocalDb.rawArchiveView : t; + expect(await db.query(live), hasLength(1), + reason: '$t did not come back'); expect(r.salvaged[t], 1, reason: '$t was not reported as recovered'); } // Values, not just counts — a merge that dropped columns would pass a @@ -404,7 +407,9 @@ void main() { final again = await LocalDb.instance; expect(LocalDb.lastRebuild, isNull); for (final t in salvaged) { - expect(await again.query(t), hasLength(1), reason: '$t lost on reopen'); + final live = t == 'raw_archive' ? LocalDb.rawArchiveView : t; + expect(await again.query(live), hasLength(1), + reason: '$t lost on reopen'); } await LocalDb.close(); }); diff --git a/test/synthetic_colmi_day_test.dart b/test/synthetic_colmi_day_test.dart index b975a5be3..bab2f3ee0 100644 --- a/test/synthetic_colmi_day_test.dart +++ b/test/synthetic_colmi_day_test.dart @@ -254,7 +254,7 @@ void main() { // counts from. final db = await LocalDb.instance; final stamps = await db.rawQuery( - "SELECT DISTINCT rec_ts FROM raw_archive WHERE reason LIKE 'colmi_big_%' " + "SELECT DISTINCT rec_ts FROM ${LocalDb.rawArchiveView} WHERE reason LIKE 'colmi_big_%' " "AND device_id = 'colmi-synthetic'"); expect(stamps.map((r) => r['rec_ts']), [SyntheticDay.sec(DateTime(2026, 10, 4, 22, 30))]); @@ -267,7 +267,7 @@ void main() { final reply = colmiBigDataRequest( kColmiBigTemperature, [0, 30, for (var k = 0; k < 48; k++) 150]); final db = await LocalDb.instance; - await db.insert('raw_archive', { + await db.insert(LocalDb.rawArchiveView, { 'device_id': 'colmi-midnight', 'hex': [for (final b in reply) b.toRadixString(16).padLeft(2, '0')] .join(), diff --git a/test/ultrahuman_link_test.dart b/test/ultrahuman_link_test.dart index 17e1cd267..452fa7631 100644 --- a/test/ultrahuman_link_test.dart +++ b/test/ultrahuman_link_test.dart @@ -152,7 +152,7 @@ void main() { final db = await LocalDb.instance; expect((await db.query('decoded_onehz')).length, 4, reason: 're-reading today re-banks the same seconds, not duplicates'); - final archive = await db.query('raw_archive'); + final archive = await db.query(LocalDb.rawArchiveView); expect(archive.length, 4, reason: 'raw_archive is keyed on the bytes'); }); @@ -161,7 +161,7 @@ void main() { await UltrahumanLink.instance.ingestForTest(_deviceId, _ring(first), nowSeconds: () => _sec(_today.add(const Duration(hours: 12)))); final db = await LocalDb.instance; - final archive = await db.query('raw_archive', orderBy: 'counter'); + final archive = await db.query(LocalDb.rawArchiveView, orderBy: 'counter'); expect(archive.map((r) => r['counter']), [1, 2, 3]); }); From 872ff981629ff3cef2c02cd3dd81d44d327afdd1 Mon Sep 17 00:00:00 2001 From: abdulsaheel Date: Sat, 31 Oct 2026 20:52:20 +0530 Subject: [PATCH 20/33] never adopt a backup's archive setting over live readings a restore only checked for a local setting and a local archive, so a phone with decoded readings and no archive yet could take a backup's Off, and the next prune deleted those readings without archiving them. --- lib/data/substrate_archive.dart | 11 +++++++- test/substrate_archive_db_test.dart | 34 +++++++++++++++++++++++-- test/substrate_archive_window_test.dart | 3 ++- 3 files changed, 44 insertions(+), 4 deletions(-) diff --git a/lib/data/substrate_archive.dart b/lib/data/substrate_archive.dart index 628a43796..a5a50e564 100644 --- a/lib/data/substrate_archive.dart +++ b/lib/data/substrate_archive.dart @@ -275,7 +275,8 @@ abstract final class SubstrateArchive { /// keep. /// /// Never adopted over local history. A local policy row always wins, and so - /// does a local archive under the implicit default: that history was kept + /// does a local archive or any live decoded reading under the implicit + /// default: that history was kept /// on this phone, and importing some other backup's "Off" must not be what /// deletes it (a restore only ever ADDS history). Checked and written in one /// transaction, so an archive cannot appear between the two. @@ -317,6 +318,14 @@ abstract final class SubstrateArchive { if (ours.isNotEmpty) return; final anyLocal = await txn.query(table, columns: ['utc_day'], limit: 1); if (anyLocal.isNotEmpty) return; + // Live readings are local history too, the part the next prune moves + // into the archive: under an adopted "Off" that prune deletes them + // without archiving, and a shorter window drops the older ones. + for (final live in const ['decoded_onehz', 'decoded_rr']) { + if ((await txn.rawQuery('SELECT 1 FROM $live LIMIT 1')).isNotEmpty) { + return; + } + } await txn.insert('compute_freshness', { 'key': policyKey, 'payload_json': adopted.toJson(), diff --git a/test/substrate_archive_db_test.dart b/test/substrate_archive_db_test.dart index 32a3b89ee..aeddd85ce 100644 --- a/test/substrate_archive_db_test.dart +++ b/test/substrate_archive_db_test.dart @@ -1016,13 +1016,15 @@ void main() { await LocalDb.pruneDecodedBeforeRecTs(cutoff); final snapshot = await LocalDb.exportCopy(); final db = await _db(); - // A fresh phone: no policy row and no archive of its own. + // A fresh phone: no policy row, no archive and no readings of its own. await db.delete( 'compute_freshness', where: 'key = ?', whereArgs: [SubstrateArchive.policyKey], ); await db.delete('substrate_archive'); + await db.delete('decoded_onehz'); + await db.delete('decoded_rr'); await LocalDb.importFromDbFile(snapshot); expect( await LocalDb.substrateArchivePolicy(), @@ -1039,6 +1041,31 @@ void main() { }); }); + test('a backup\'s Off policy is not adopted over live readings with no ' + 'archive yet', () async { + await _seed(); + await LocalDb.setSubstrateArchivePolicy(const SubstrateArchivePolicy(0)); + final offBackup = await LocalDb.exportCopy(); + final db = await _db(); + // This phone: readings, no archive yet, no stored setting. + await db.delete('compute_freshness', + where: 'key = ?', whereArgs: [SubstrateArchive.policyKey]); + await db.delete('substrate_archive'); + final live = (await db.query('decoded_onehz')).length; + expect(live, greaterThan(0)); + + await LocalDb.importFromDbFile(offBackup); + expect( + await db.query('compute_freshness', + where: 'key = ?', whereArgs: [SubstrateArchive.policyKey]), + isEmpty, + ); + // The next prune archives them instead of deleting them outright. + await LocalDb.pruneDecodedBeforeRecTs(cutoff); + expect(await db.query('substrate_archive'), isNotEmpty); + await File(offBackup).delete(); + }); + test('a backup\'s Off policy never deletes this phone\'s archive; on a ' 'fresh phone it is adopted', () async { await _seed(); @@ -1069,8 +1096,11 @@ void main() { isEmpty, ); - // A fresh phone adopts it — and then takes no archive rows in. + // A fresh phone (no archive, no readings) adopts it — and then takes no + // archive rows in. await db.delete('substrate_archive'); + await db.delete('decoded_onehz'); + await db.delete('decoded_rr'); await LocalDb.importFromDbFile(offBackup); expect( await LocalDb.substrateArchivePolicy(), diff --git a/test/substrate_archive_window_test.dart b/test/substrate_archive_window_test.dart index 381a6ee80..4211f64de 100644 --- a/test/substrate_archive_window_test.dart +++ b/test/substrate_archive_window_test.dart @@ -46,6 +46,8 @@ void main() { await db.execute('CREATE TABLE compute_freshness (key TEXT PRIMARY KEY, ' 'payload_json TEXT NOT NULL, updated_at INTEGER NOT NULL)'); await db.execute('CREATE TABLE ${SubstrateArchive.table} (utc_day INTEGER, day TEXT)'); + await db.execute('CREATE TABLE decoded_onehz (rec_ts INTEGER)'); + await db.execute('CREATE TABLE decoded_rr (rec_ts INTEGER)'); }); tearDown(() => db.close()); @@ -66,7 +68,6 @@ void main() { test('an empty archive with band readings already stored keeps a year', () async { - await db.execute('CREATE TABLE decoded_onehz (rec_ts INTEGER)'); await db.insert('decoded_onehz', {'rec_ts': 1750000000}); await SubstrateArchive.keepLegacyWindowOnce(db); expect((await SubstrateArchive.readPolicy(db)).keepDays, 365); From 2648925a4b974b6af5bfb281c11c007fd8da9a1b Mon Sep 17 00:00:00 2001 From: abdulsaheel Date: Sat, 31 Oct 2026 21:46:01 +0530 Subject: [PATCH 21/33] read no setting as a year until the window check has run if the one-time check failed before it finished, housekeeping read the 90-day default and could evict the older archive before the check was retried. the default applies only once the check has run. --- lib/data/substrate_archive.dart | 23 ++++++++++++++++------- test/substrate_archive_window_test.dart | 8 ++++++++ 2 files changed, 24 insertions(+), 7 deletions(-) diff --git a/lib/data/substrate_archive.dart b/lib/data/substrate_archive.dart index a5a50e564..e8920bb03 100644 --- a/lib/data/substrate_archive.dart +++ b/lib/data/substrate_archive.dart @@ -244,17 +244,26 @@ abstract final class SubstrateArchive { /// The policy as stored, on [ex] — inside a transaction when the caller /// needs it to agree with what that transaction writes. + /// + /// With no stored setting, a year until [keepLegacyWindowOnce] has run on + /// this database: if that check failed before it finished, housekeeping + /// reading the 90-day default in the meantime would evict exactly what the + /// check is there to keep. The default applies once the check has run. static Future readPolicy(DatabaseExecutor ex) async { final rows = await ex.query( 'compute_freshness', - columns: ['payload_json'], - where: 'key = ?', - whereArgs: [policyKey], - limit: 1, - ); - return SubstrateArchivePolicy.fromJson( - rows.isEmpty ? null : rows.first['payload_json'] as String?, + columns: ['key', 'payload_json'], + where: 'key IN (?, ?)', + whereArgs: [policyKey, _defaultCheckedKey], ); + for (final r in rows) { + if (r['key'] == policyKey) { + return SubstrateArchivePolicy.fromJson(r['payload_json'] as String?); + } + } + return rows.isEmpty + ? const SubstrateArchivePolicy(365) + : SubstrateArchivePolicy.defaults; } /// Store [policy]; switching to Off purges the archive IN THE SAME diff --git a/test/substrate_archive_window_test.dart b/test/substrate_archive_window_test.dart index 4211f64de..51df94524 100644 --- a/test/substrate_archive_window_test.dart +++ b/test/substrate_archive_window_test.dart @@ -102,6 +102,14 @@ void main() { expect((await SubstrateArchive.readPolicy(db)).keepDays, 365); }); + test('until the one-time check has run, no setting reads as a year', + () async { + // The check failed (or has not run): nothing may be evicted past 90. + expect((await SubstrateArchive.readPolicy(db)).keepDays, 365); + await SubstrateArchive.keepLegacyWindowOnce(db); // empty: the default + expect((await SubstrateArchive.readPolicy(db)).keepDays, 90); + }); + test('a window the user picked is left alone', () async { await db.insert(SubstrateArchive.table, {'day': '2025-11-01'}); await db.insert('compute_freshness', { From 24e1208ec7dd34726a89602398c844c0225623bb Mon Sep 17 00:00:00 2001 From: abdulsaheel Date: Sat, 31 Oct 2026 21:47:12 +0530 Subject: [PATCH 22/33] give an old archive table its device column before moving it a rollback to a build from before the v51 rekey makes raw_archive without device_id, and the view and the move both read that column, so its frames were stranded. the column is added the way the v51 rekey filled it before the table is set aside. --- lib/data/db.dart | 12 ++++++++++- test/raw_archive_frames_test.dart | 34 +++++++++++++++++++++++++++++++ 2 files changed, 45 insertions(+), 1 deletion(-) diff --git a/lib/data/db.dart b/lib/data/db.dart index 6d376a804..b82b5b2f3 100644 --- a/lib/data/db.dart +++ b/lib/data/db.dart @@ -1440,7 +1440,8 @@ class LocalDb { } /// The column names [table] currently has (empty if the table is absent). - static Future> _columnsOf(Database db, String table) async { + static Future> _columnsOf( + DatabaseExecutor db, String table) async { final info = await db.rawQuery('PRAGMA table_info($table)'); return { for (final c in info) @@ -7556,6 +7557,15 @@ class LocalDb { // leave the frames table without its captured_at index until the // legacy table is gone, because the index name is still taken. await db.transaction((txn) async { + // A build from before the v51 rekey, rolled back to, makes this table + // without device_id; the view and the move both read it. Filled the + // way the v51 rekey filled it: the primary device. + if (!(await _columnsOf(txn, 'raw_archive')).contains('device_id')) { + await txn.execute( + 'ALTER TABLE raw_archive ADD COLUMN device_id TEXT NOT NULL ' + "DEFAULT '$kPrimaryDeviceId'", + ); + } if (await _objectType(txn, _rawArchiveLegacy) == 'table') { // Only reachable if a file was hand-assembled with both; fold the // newer table into the aside one rather than lose either. diff --git a/test/raw_archive_frames_test.dart b/test/raw_archive_frames_test.dart index 6e81c61c9..e20d8f025 100644 --- a/test/raw_archive_frames_test.dart +++ b/test/raw_archive_frames_test.dart @@ -281,6 +281,40 @@ void main() { expect(keys, [_frameHex(1), _frameHex(2)]..sort()); }); + test('a rollback to a build from before device keys still migrates', + () async { + await _fresh('raw_frames_rollback_v50.db'); + await LocalDb.instance; + await LocalDb.close(); + // A schema-50 build: raw_archive keyed by hex alone, no device_id. + final old = await databaseFactory.openDatabase( + await _path('raw_frames_rollback_v50.db')); + await old.execute(''' + CREATE TABLE IF NOT EXISTS raw_archive ( + hex TEXT PRIMARY KEY, + counter INTEGER, + packet_type INTEGER NOT NULL, + rec_ts INTEGER, + captured_at INTEGER NOT NULL, + reason TEXT NOT NULL + ) + '''); + await old.insert('raw_archive', { + 'hex': _frameHex(50), + 'counter': 50, + 'packet_type': 0x2F, + 'captured_at': 1750000050000, + 'reason': 'undecodable_rec_v20', + }); + await old.close(); + + await LocalDb.instance; + while (await LocalDb.migrateLegacyRawArchive() > 0) {} + final rows = await _viewRows(); + expect(rows.map((r) => '${r['device_id']}|${r['hex']}'), + ['|${_frameHex(50)}']); + }); + test('a backup of the new layout restores every frame', () async { await _fresh('raw_frames_backup_src.db'); final frames = [_frameHex(1), 'Not-Hex', _frameHex(2)]; From 8bb081470d28f449cb365a4f6eba6b1cf348efe3 Mon Sep 17 00:00:00 2001 From: abdulsaheel Date: Sat, 31 Oct 2026 23:31:52 +0530 Subject: [PATCH 23/33] check the partial-derive fingerprint in its own lighter test the check ran five extra heavy passes over three days inside a file whose equivalence test already runs ten. it now runs on one day in its own file and reads the stored fingerprint after each failing pass, which needs three passes instead of five; the equivalence file is back as it was. --- test/derive_partial_not_cached_test.dart | 66 ++++++++++++++++++++++++ test/derive_unchanged_skip_test.dart | 18 ------- 2 files changed, 66 insertions(+), 18 deletions(-) create mode 100644 test/derive_partial_not_cached_test.dart diff --git a/test/derive_partial_not_cached_test.dart b/test/derive_partial_not_cached_test.dart new file mode 100644 index 000000000..9384a06b0 --- /dev/null +++ b/test/derive_partial_not_cached_test.dart @@ -0,0 +1,66 @@ +// A derive whose second half failed stores a partial row. Two failures in a +// row can store the same partial row, and if that pass then recorded the +// day's input fingerprint, every later pass would skip the day as unchanged +// and the missing detail would never be filled in. Only a complete derive +// may record one. Its own file: the derive passes are heavy, and the +// equivalence test next door already runs ten of them. + +import 'package:flutter_test/flutter_test.dart'; +import 'package:openstrap_edge/compute/derivation_engine.dart'; +import 'package:openstrap_edge/compute/profile.dart'; +import 'package:openstrap_edge/data/db.dart'; +import 'package:path/path.dart' as p; +import 'package:shared_preferences/shared_preferences.dart'; +import 'package:sqflite_common_ffi/sqflite_ffi.dart'; + +import 'support/synthetic_day.dart'; + +const _profile = Profile(ageYears: 32, weightKg: 72, heightCm: 176, sex: 'm'); + +// One day, mid-July, still pending (the data edge is inside it). +final _day = SyntheticDay(DateTime(2026, 7, 13)); + +void main() { + setUpAll(() { + sqfliteFfiInit(); + databaseFactory = databaseFactoryFfi; + }); + + tearDown(() async { + DerivationEngine.debugSecondHalfHook = null; + await LocalDb.close(); + }); + + test('a partial derive never records its fingerprint', () async { + const name = 'derive_partial_not_cached.db'; + SharedPreferences.setMockInitialValues({}); + await LocalDb.close(); + LocalDb.dbName = name; + await databaseFactory + .deleteDatabase(p.join(await databaseFactory.getDatabasesPath(), name)); + for (final (raws, samples) in _day.primaryBatches()) { + await LocalDb.commitSyncBatch(raws, samples, deviceFamily: 'gen4'); + } + + DerivationEngine.debugSecondHalfHook = + (_) async => throw StateError('second half down'); + // The passes that reach the fixed point: the second stores the same + // partial row as the first, the third derives from it unchanged. + for (var i = 0; i < 3; i++) { + final e = DerivationEngine(); + await e.run(_profile, heavy: true); + final derived = await _derivedDays(); + expect(derived, isNotEmpty); + for (final d in derived) { + expect(await LocalDb.deriveFingerprint(d), isNull, + reason: 'pass $i stored a fingerprint for partial day $d'); + } + } + }, timeout: const Timeout(Duration(minutes: 5))); +} + +Future> _derivedDays() async => [ + for (final r in await (await LocalDb.instance) + .query('day_result', columns: ['day_id'], distinct: true)) + r['day_id'] as String, + ]; diff --git a/test/derive_unchanged_skip_test.dart b/test/derive_unchanged_skip_test.dart index e494c3780..647b20922 100644 --- a/test/derive_unchanged_skip_test.dart +++ b/test/derive_unchanged_skip_test.dart @@ -97,7 +97,6 @@ void main() { tearDown(() async { DerivationEngine.debugDeriveEveryDay = false; DerivationEngine.debugFingerprintAlgoVersion = null; - DerivationEngine.debugSecondHalfHook = null; await LocalDb.close(); }); @@ -179,21 +178,4 @@ void main() { DerivationEngine.debugFingerprintAlgoVersion = kAlgoVersion + 1; expect((await _pass()).derived, pending); }, timeout: const Timeout(Duration(minutes: 15))); - - test('a day whose second half keeps failing is never cached as unchanged', - () async { - await _freshDb('unchanged_skip_partial.db'); - DerivationEngine.debugSecondHalfHook = - (_) async => throw StateError('second half down'); - // Failures in a row settle on the same partial row; none may vouch for - // it. - for (var i = 0; i < 4; i++) { - expect((await _pass()).unchanged, 0, reason: 'failing pass $i'); - } - DerivationEngine.debugSecondHalfHook = null; - // The failure cleared: the days derive again instead of being skipped. - final healed = await _pass(); - expect(healed.unchanged, 0); - expect(healed.derived, greaterThan(0)); - }, timeout: const Timeout(Duration(minutes: 10))); } From 3f762a570087bb2c86fb3da3d4e3a20562f8c8f4 Mon Sep 17 00:00:00 2001 From: abdulsaheel Date: Sun, 1 Nov 2026 01:06:42 +0530 Subject: [PATCH 24/33] give the archive move fifteen seconds only on heavy passes every light foreground pass ran it with the fifteen-second budget meant for a heavy pass, so a sync's quick derive could sit in housekeeping for that long. light and selected-day passes get three seconds now; heavy, forced and compaction keep fifteen. --- lib/compute/derivation_engine.dart | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/lib/compute/derivation_engine.dart b/lib/compute/derivation_engine.dart index 91b07fcfc..270ff8b53 100644 --- a/lib/compute/derivation_engine.dart +++ b/lib/compute/derivation_engine.dart @@ -3199,7 +3199,7 @@ class DerivationEngine { } catch (e) { _log('compact: prune skipped: $e'); } - await _runStorageHousekeeping(); + await _runStorageHousekeeping(long: true); final size = await LocalDb.databaseBytesOnDisk(); final free = await freeDiskBytes(); if (free == null || free < 2 * size) { @@ -3740,7 +3740,7 @@ class DerivationEngine { // `_running`. See _runStorageHousekeeping — this is the only place every // entry path and every early return actually reaches. _diag['stage'] = 'housekeeping'; - await _runStorageHousekeeping(); + await _runStorageHousekeeping(long: heavy || force); // ONE-SHOT: rescale stored strain onto the recalibrated scale. Days // inside the raw window re-derive from substrate above; everything older // has none, so its headline is rebuilt from the stored TRIMP + wake @@ -7893,16 +7893,19 @@ class DerivationEngine { /// every day reported 0 back to `reanalyzeAll`. This work is a storage /// optimization; nothing it does may change what the derive returns. /// - /// Bounded and resumable, so running it on every pass costs one small batch. + /// Bounded and resumable: a light pass gives it about three seconds (a page + /// of the archive move can run past that, never by more than one page), a + /// heavy or forced pass and an explicit compaction about fifteen. /// Off the path to a durable commit, and never inside a migration: `onUpgrade` /// runs under iOS's CPU watchdog (invariant 11). - Future _runStorageHousekeeping() async { + Future _runStorageHousekeeping({bool long = false}) async { try { // The pre-conversion undecodable archive, moved into the frame store a // bounded slice at a time (see LocalDb.migrateLegacyRawArchive). A - // WHOOP MG's is ~500 MB of hex, so a background pass takes little. + // WHOOP MG's is ~500 MB of hex, so a background or light pass takes + // little. final moved = await LocalDb.migrateLegacyRawArchive( - budget: Duration(seconds: background ? 3 : 15), + budget: Duration(seconds: long && !background ? 15 : 3), shouldStop: offloadActive, ); if (moved > 0) _log('moved $moved archived frame(s) to the frame store'); From 1a8322172f69b7215799ac33feb759fb5204b125 Mon Sep 17 00:00:00 2001 From: abdulsaheel Date: Sun, 1 Nov 2026 01:08:24 +0530 Subject: [PATCH 25/33] read a wearable's archive straight off its tables for the day signature the view builds each frame's hex from its bytes, so every pending day rebuilt that text for every frame just to sum its length. the tables give the same number (a blob's hex is twice its bytes) under the same column names, so the signature value and every stored fingerprint stay as they were. --- lib/data/db.dart | 19 +++++++++++++++--- test/raw_archive_frames_test.dart | 32 +++++++++++++++++++++++++++++++ 2 files changed, 48 insertions(+), 3 deletions(-) diff --git a/lib/data/db.dart b/lib/data/db.dart index f8aa8de91..76f197b90 100644 --- a/lib/data/db.dart +++ b/lib/data/db.dart @@ -10191,11 +10191,24 @@ class LocalDb { 'FROM decoded_onehz WHERE device_id = ?', [wearableDeviceId], ); + // Read off the tables, not the view: the view's hex is built from each + // frame's bytes, which was text per frame per day for a length. A + // stored blob's hex is exactly twice its bytes, and the columns keep the + // view query's names (the signature orders values by them), so the + // value — and every stored fingerprint — is the same as before. + final legacyArchive = await has(_rawArchiveLegacy); await add( 'wearable_archive', - 'SELECT COUNT(*), MAX(captured_at), TOTAL(captured_at), ' - 'TOTAL(LENGTH(hex)) FROM $rawArchiveView WHERE device_id = ?', - [wearableDeviceId], + 'SELECT COUNT(*) AS "COUNT(*)", ' + 'MAX(captured_at) AS "MAX(captured_at)", ' + 'TOTAL(captured_at) AS "TOTAL(captured_at)", ' + 'TOTAL(n) AS "TOTAL(LENGTH(hex))" FROM (' + 'SELECT captured_at, CASE WHEN hex IS NOT NULL THEN LENGTH(hex) ' + 'ELSE LENGTH(frame) * 2 END AS n FROM $rawArchiveTable ' + 'WHERE device_id = ?' + '${legacyArchive ? ' UNION ALL SELECT captured_at, LENGTH(hex) ' + 'FROM $_rawArchiveLegacy WHERE device_id = ?' : ''})', + [wearableDeviceId, if (legacyArchive) wearableDeviceId], ); } await add( diff --git a/test/raw_archive_frames_test.dart b/test/raw_archive_frames_test.dart index e20d8f025..8d25ceabe 100644 --- a/test/raw_archive_frames_test.dart +++ b/test/raw_archive_frames_test.dart @@ -5,6 +5,7 @@ // canonical hex survives verbatim, and an existing archive moves across in // pages without losing or duplicating a frame. +import 'dart:convert'; import 'dart:io'; import 'dart:math' as math; @@ -315,6 +316,37 @@ void main() { ['|${_frameHex(50)}']); }); + test('the input signature reads the archive tables to the same value as ' + 'the view', () async { + // Mid-move: legacy rows still aside, and the frame store holding both a + // blob frame and a verbatim text one. + await _legacyDb('raw_frames_signature.db', [ + for (var i = 0; i < 3; i++) _legacyRow('ring-1', _frameHex(300 + i), i), + ]); + await LocalDb.migrateLegacyRawArchive(budget: Duration.zero); + await LocalDb.archiveRawRecord(_rec(7, _frameHex(77)), deviceId: 'ring-1'); + await LocalDb.archiveRawRecord(_rec(8, 'Not-Hex'), deviceId: 'ring-1'); + final db = await LocalDb.instance; + final viewRow = (await db.rawQuery( + 'SELECT COUNT(*), MAX(captured_at), TOTAL(captured_at), ' + 'TOTAL(LENGTH(hex)) FROM ${LocalDb.rawArchiveView} WHERE device_id = ?', + ['ring-1'], + )).single; + final keys = viewRow.keys.toList()..sort(); + final expected = jsonEncode([ + [for (final k in keys) viewRow[k]], + ]); + final sig = await LocalDb.dayInputSignature( + dayId: '2025-06-15', + prevDayId: '2025-06-14', + nextDayId: '2025-06-16', + fromSec: 1749900000, + toSec: 1750100000, + wearableDeviceId: 'ring-1', + ); + expect(sig, contains('wearable_archive=$expected;')); + }); + test('a backup of the new layout restores every frame', () async { await _fresh('raw_frames_backup_src.db'); final frames = [_frameHex(1), 'Not-Hex', _frameHex(2)]; From 94b43db9509e0c875704d465fdceec30527486e8 Mon Sep 17 00:00:00 2001 From: abdulsaheel Date: Sun, 1 Nov 2026 01:09:26 +0530 Subject: [PATCH 26/33] say the archive window counts back from the latest reading the shortening confirmation said readings older than the window are deleted, but the window is measured from the newest stored reading, not from today. --- lib/l10n/app_de.arb | 2 +- lib/l10n/app_en.arb | 2 +- lib/l10n/app_es.arb | 2 +- lib/l10n/app_fr.arb | 2 +- lib/l10n/app_hi.arb | 2 +- lib/l10n/app_ru.arb | 2 +- lib/l10n/app_zh.arb | 2 +- lib/ui2/profile/data.dart | 3 ++- 8 files changed, 9 insertions(+), 8 deletions(-) diff --git a/lib/l10n/app_de.arb b/lib/l10n/app_de.arb index 22511b498..320c7d1f8 100644 --- a/lib/l10n/app_de.arb +++ b/lib/l10n/app_de.arb @@ -2931,7 +2931,7 @@ } } }, - "dataKeepShorterBody": "Messwerte, die älter als {window} sind, werden jetzt gelöscht. Diese Tage behalten ihre Werte, können nach künftigen Updates aber nicht neu berechnet werden.", + "dataKeepShorterBody": "Messwerte, die mehr als {window} vor deinem letzten Messwert liegen, werden jetzt gelöscht. Diese Tage behalten ihre Werte, können nach künftigen Updates aber nicht neu berechnet werden.", "@dataKeepShorterBody": { "placeholders": { "window": { diff --git a/lib/l10n/app_en.arb b/lib/l10n/app_en.arb index 4d34faf34..25034d2e2 100644 --- a/lib/l10n/app_en.arb +++ b/lib/l10n/app_en.arb @@ -16242,7 +16242,7 @@ } } }, - "dataKeepShorterBody": "Readings older than {window} are deleted now. Those days keep their scores, but can't be re-scored after future updates.", + "dataKeepShorterBody": "Readings more than {window} before your latest one are deleted now. Those days keep their scores, but can't be re-scored after future updates.", "@dataKeepShorterBody": { "description": "Confirm body when the archive window is shortened.", "placeholders": { diff --git a/lib/l10n/app_es.arb b/lib/l10n/app_es.arb index d0db5102f..8fb08fd55 100644 --- a/lib/l10n/app_es.arb +++ b/lib/l10n/app_es.arb @@ -2843,7 +2843,7 @@ } } }, - "dataKeepShorterBody": "Las lecturas de hace más de {window} se borran ahora. Esos días mantienen sus puntuaciones, pero no se podrán recalcular tras futuras actualizaciones.", + "dataKeepShorterBody": "Las lecturas de más de {window} antes de la última se borran ahora. Esos días mantienen sus puntuaciones, pero no se podrán recalcular tras futuras actualizaciones.", "@dataKeepShorterBody": { "placeholders": { "window": { diff --git a/lib/l10n/app_fr.arb b/lib/l10n/app_fr.arb index 18315e1dc..4d59f1618 100644 --- a/lib/l10n/app_fr.arb +++ b/lib/l10n/app_fr.arb @@ -2814,7 +2814,7 @@ } } }, - "dataKeepShorterBody": "Les mesures de plus de {window} sont supprimées maintenant. Ces jours gardent leurs scores, mais ne pourront pas être recalculés après les prochaines mises à jour.", + "dataKeepShorterBody": "Les mesures de plus de {window} avant la dernière sont supprimées maintenant. Ces jours gardent leurs scores, mais ne pourront pas être recalculés après les prochaines mises à jour.", "@dataKeepShorterBody": { "placeholders": { "window": { diff --git a/lib/l10n/app_hi.arb b/lib/l10n/app_hi.arb index 0c0262440..5efb70669 100644 --- a/lib/l10n/app_hi.arb +++ b/lib/l10n/app_hi.arb @@ -2814,7 +2814,7 @@ } } }, - "dataKeepShorterBody": "{window} से पुरानी रीडिंग अभी हटा दी जाएंगी। उन दिनों के स्कोर बने रहेंगे, पर आगे के अपडेट के बाद उन्हें दोबारा नहीं आंका जा सकेगा।", + "dataKeepShorterBody": "आपकी आख़िरी रीडिंग से {window} से ज़्यादा पहले की रीडिंग अभी हटा दी जाएंगी। उन दिनों के स्कोर बने रहेंगे, पर आगे के अपडेट के बाद उन्हें दोबारा नहीं आंका जा सकेगा।", "@dataKeepShorterBody": { "placeholders": { "window": { diff --git a/lib/l10n/app_ru.arb b/lib/l10n/app_ru.arb index f58713172..e405b89c8 100644 --- a/lib/l10n/app_ru.arb +++ b/lib/l10n/app_ru.arb @@ -7855,7 +7855,7 @@ } } }, - "dataKeepShorterBody": "Показания старше {window} будут удалены сейчас. Эти дни сохранят свои оценки, но их нельзя будет пересчитать после будущих обновлений.", + "dataKeepShorterBody": "Показания, снятые более чем за {window} до последнего, будут удалены сейчас. Эти дни сохранят свои оценки, но их нельзя будет пересчитать после будущих обновлений.", "@dataKeepShorterBody": { "placeholders": { "window": { diff --git a/lib/l10n/app_zh.arb b/lib/l10n/app_zh.arb index 5ac987ed5..1ced74f9e 100644 --- a/lib/l10n/app_zh.arb +++ b/lib/l10n/app_zh.arb @@ -2814,7 +2814,7 @@ } } }, - "dataKeepShorterBody": "超过 {window} 的读数将立即删除。这些日子会保留评分，但在以后的更新后无法重新计算。", + "dataKeepShorterBody": "比最新读数早 {window} 以上的读数将立即删除。这些日子会保留评分，但在以后的更新后无法重新计算。", "@dataKeepShorterBody": { "placeholders": { "window": { diff --git a/lib/ui2/profile/data.dart b/lib/ui2/profile/data.dart index 6da4dcd00..c85c5023d 100644 --- a/lib/ui2/profile/data.dart +++ b/lib/ui2/profile/data.dart @@ -270,7 +270,8 @@ class _DataScreenState extends State { title: l?.dataKeepShorterTitle(window) ?? 'Keep $window of raw readings?', body: l?.dataKeepShorterBody(window) ?? - 'Readings older than $window are deleted now. Those days keep ' + 'Readings more than $window before your latest one are deleted ' + 'now. Those days keep ' "their scores, but can't be re-scored after future updates.", remove: l?.dataKeepShorterConfirm ?? 'Delete older readings', keep: l?.dataKeepShorterCancel ?? 'Keep them', From 8999e8c27d4daf73274fa9de742de1e10dcd8268 Mon Sep 17 00:00:00 2001 From: abdulsaheel Date: Sun, 1 Nov 2026 01:27:13 +0530 Subject: [PATCH 27/33] restart an unfinished archive replay after a VACUUM the replay resumes from a frame-store rowid, and with no INTEGER PRIMARY KEY on that table a VACUUM can renumber it, so a resumed pass could skip frames. both VACUUM paths drop the resume point first; the replay skips what is already decoded, so starting over only costs time. --- lib/data/db.dart | 19 +++++++++++++++++++ test/raw_archive_test.dart | 21 +++++++++++++++++++++ 2 files changed, 40 insertions(+) diff --git a/lib/data/db.dart b/lib/data/db.dart index 76f197b90..5eb5453dd 100644 --- a/lib/data/db.dart +++ b/lib/data/db.dart @@ -8447,6 +8447,23 @@ class LocalDb { return r.recovered; } + /// Drop an unfinished [redriveArchiveOnce]'s resume point before a VACUUM. + /// That position is a [rawArchiveTable] rowid, and the table has no + /// INTEGER PRIMARY KEY, so VACUUM is free to renumber it: resuming there + /// could skip frames. The replay skips seconds already decoded, so starting + /// over from the top only costs time. + static Future _resetRedriveResume() async { + final row = await computeFreshness(kArchiveRedriveKey); + if (row?['payload_json'] case final String j) { + final v = jsonDecode(j); + if (v is! Map || v['done'] == true || v['after_hex'] == null) return; + await putComputeFreshness( + kArchiveRedriveKey, + jsonEncode({'recovered': v['recovered'] ?? 0}), + ); + } + } + /// GATES 4b — replay the archive through the CURRENT decoders, into /// `decoded_onehz` / `decoded_rr`. Returns the seconds recovered. /// @@ -10537,6 +10554,7 @@ class LocalDb { /// returns a row, which `execute` refuses on iOS's sqflite. static Future vacuumAndCheckpoint() async { final db = await instance; + await _resetRedriveResume(); await db.execute('VACUUM'); // VACUUM renumbers every btree root page; see [vacuumIfBloated]. await CoachDb.close(); @@ -14864,6 +14882,7 @@ class LocalDb { final free = await freelistBytes(); if (free < minFreeBytes) return 0; final db = await instance; + await _resetRedriveResume(); await db.execute('VACUUM'); // VACUUM rewrites the file and renumbers every btree root page. CoachDb // caches the root pages of its allow-listed views to decide what a coach diff --git a/test/raw_archive_test.dart b/test/raw_archive_test.dart index 389f95712..7eec6f9ee 100644 --- a/test/raw_archive_test.dart +++ b/test/raw_archive_test.dart @@ -377,4 +377,25 @@ void main() { } }, timeout: const Timeout(Duration(minutes: 20))); } + + + test('a VACUUM drops an unfinished replay\'s rowid resume point', () async { + await LocalDb.putComputeFreshness( + LocalDb.kArchiveRedriveKey, + jsonEncode({'recovered': 5, 'after_hex': '42', 'after_device': 'x'}), + ); + await LocalDb.vacuumAndCheckpoint(); + final state = jsonDecode((await LocalDb.computeFreshness( + LocalDb.kArchiveRedriveKey))!['payload_json'] as String) as Map; + expect(state, {'recovered': 5}); + + // A finished replay stays finished. + await LocalDb.putComputeFreshness( + LocalDb.kArchiveRedriveKey, jsonEncode({'recovered': 5, 'done': true})); + await LocalDb.vacuumAndCheckpoint(); + final done = jsonDecode((await LocalDb.computeFreshness( + LocalDb.kArchiveRedriveKey))!['payload_json'] as String) as Map; + expect(done['done'], true); + await LocalDb.deleteComputeFreshness(LocalDb.kArchiveRedriveKey); + }); } From a37c8b3c1c55c64d5d894b887a2db04ca0f93ad2 Mon Sep 17 00:00:00 2001 From: abdulsaheel Date: Sun, 1 Nov 2026 01:28:08 +0530 Subject: [PATCH 28/33] read the WAL checkpoint's busy flag after a compaction a reader holding a snapshot makes the truncating checkpoint return busy instead of throwing, and it was taken as done. it is read now and tried once more, and a compaction that still could not truncate says so in the log. --- lib/compute/derivation_engine.dart | 6 +++++- lib/data/db.dart | 24 ++++++++++++++++++++++-- test/raw_archive_test.dart | 13 +++++++++++++ 3 files changed, 40 insertions(+), 3 deletions(-) diff --git a/lib/compute/derivation_engine.dart b/lib/compute/derivation_engine.dart index 270ff8b53..4b29e3256 100644 --- a/lib/compute/derivation_engine.dart +++ b/lib/compute/derivation_engine.dart @@ -3222,7 +3222,11 @@ class DerivationEngine { ); } onStep?.call(CompactStep.vacuum); - await LocalDb.vacuumAndCheckpoint(); + if (!await LocalDb.vacuumAndCheckpoint()) { + // The size below is measured on disk, WAL included, so the result + // stays honest; the WAL shrinks at the next checkpoint. + _log('compact: WAL not truncated, a reader held it'); + } return CompactResult( beforeBytes: before, afterBytes: await LocalDb.databaseBytesOnDisk(), diff --git a/lib/data/db.dart b/lib/data/db.dart index 5eb5453dd..51cf3d383 100644 --- a/lib/data/db.dart +++ b/lib/data/db.dart @@ -10552,15 +10552,35 @@ class LocalDb { /// nothing else in flight); see [vacuumIfBloated] for why this is a one-off /// VACUUM and not `auto_vacuum`. The checkpoint is a `rawQuery`: the PRAGMA /// returns a row, which `execute` refuses on iOS's sqflite. - static Future vacuumAndCheckpoint() async { + /// + /// True when the WAL was truncated too. A reader still holding a snapshot + /// does not make the checkpoint throw: it returns busy = 1 in its first + /// column and leaves the WAL as it was, so the row is read, and tried once + /// more after a moment. + static Future vacuumAndCheckpoint() async { final db = await instance; await _resetRedriveResume(); await db.execute('VACUUM'); // VACUUM renumbers every btree root page; see [vacuumIfBloated]. await CoachDb.close(); - await db.rawQuery('PRAGMA wal_checkpoint(TRUNCATE)'); + for (var attempt = 0; attempt < 2; attempt++) { + if (attempt > 0) { + await Future.delayed(const Duration(milliseconds: 500)); + } + if (checkpointCompleted( + await db.rawQuery('PRAGMA wal_checkpoint(TRUNCATE)'))) { + return true; + } + } + return false; } + /// Whether a `PRAGMA wal_checkpoint` result row says it finished: its first + /// column is 1 when the checkpoint was blocked. + @visibleForTesting + static bool checkpointCompleted(List> rows) => + rows.isEmpty || rows.first.values.first != 1; + static Future databaseFileBytes() async { final dir = await getDatabasesPath(); final path = p.join(dir, dbName); diff --git a/test/raw_archive_test.dart b/test/raw_archive_test.dart index 7eec6f9ee..dd6163f9a 100644 --- a/test/raw_archive_test.dart +++ b/test/raw_archive_test.dart @@ -398,4 +398,17 @@ void main() { expect(done['done'], true); await LocalDb.deleteComputeFreshness(LocalDb.kArchiveRedriveKey); }); + + test('a blocked WAL checkpoint is not reported as done', () { + expect( + LocalDb.checkpointCompleted([ + {'busy': 1, 'log': 40, 'checkpointed': 12}, + ]), + isFalse); + expect( + LocalDb.checkpointCompleted([ + {'busy': 0, 'log': 0, 'checkpointed': 0}, + ]), + isTrue); + }); } From 24640a5ec83728087c6b655ac8ed0d44f05d203a Mon Sep 17 00:00:00 2001 From: abdulsaheel Date: Sun, 1 Nov 2026 01:28:43 +0530 Subject: [PATCH 29/33] say about, since the archive window drops whole days eviction removes whole UTC-day buckets, so a reading just past the window can stay until its day falls out. --- lib/l10n/app_de.arb | 2 +- lib/l10n/app_en.arb | 2 +- lib/l10n/app_es.arb | 2 +- lib/l10n/app_fr.arb | 2 +- lib/l10n/app_hi.arb | 2 +- lib/l10n/app_ru.arb | 2 +- lib/l10n/app_zh.arb | 2 +- lib/ui2/profile/data.dart | 6 +++--- 8 files changed, 10 insertions(+), 10 deletions(-) diff --git a/lib/l10n/app_de.arb b/lib/l10n/app_de.arb index 320c7d1f8..8043da17e 100644 --- a/lib/l10n/app_de.arb +++ b/lib/l10n/app_de.arb @@ -2931,7 +2931,7 @@ } } }, - "dataKeepShorterBody": "Messwerte, die mehr als {window} vor deinem letzten Messwert liegen, werden jetzt gelöscht. Diese Tage behalten ihre Werte, können nach künftigen Updates aber nicht neu berechnet werden.", + "dataKeepShorterBody": "Messwerte, die mehr als etwa {window} vor deinem letzten Messwert liegen, werden jetzt gelöscht. Diese Tage behalten ihre Werte, können nach künftigen Updates aber nicht neu berechnet werden.", "@dataKeepShorterBody": { "placeholders": { "window": { diff --git a/lib/l10n/app_en.arb b/lib/l10n/app_en.arb index 25034d2e2..724f9d181 100644 --- a/lib/l10n/app_en.arb +++ b/lib/l10n/app_en.arb @@ -16242,7 +16242,7 @@ } } }, - "dataKeepShorterBody": "Readings more than {window} before your latest one are deleted now. Those days keep their scores, but can't be re-scored after future updates.", + "dataKeepShorterBody": "Readings more than about {window} before your latest one are deleted now. Those days keep their scores, but can't be re-scored after future updates.", "@dataKeepShorterBody": { "description": "Confirm body when the archive window is shortened.", "placeholders": { diff --git a/lib/l10n/app_es.arb b/lib/l10n/app_es.arb index 8fb08fd55..e99aba10e 100644 --- a/lib/l10n/app_es.arb +++ b/lib/l10n/app_es.arb @@ -2843,7 +2843,7 @@ } } }, - "dataKeepShorterBody": "Las lecturas de más de {window} antes de la última se borran ahora. Esos días mantienen sus puntuaciones, pero no se podrán recalcular tras futuras actualizaciones.", + "dataKeepShorterBody": "Las lecturas de más de unos {window} antes de la última se borran ahora. Esos días mantienen sus puntuaciones, pero no se podrán recalcular tras futuras actualizaciones.", "@dataKeepShorterBody": { "placeholders": { "window": { diff --git a/lib/l10n/app_fr.arb b/lib/l10n/app_fr.arb index 4d59f1618..58840a3d3 100644 --- a/lib/l10n/app_fr.arb +++ b/lib/l10n/app_fr.arb @@ -2814,7 +2814,7 @@ } } }, - "dataKeepShorterBody": "Les mesures de plus de {window} avant la dernière sont supprimées maintenant. Ces jours gardent leurs scores, mais ne pourront pas être recalculés après les prochaines mises à jour.", + "dataKeepShorterBody": "Les mesures de plus d'environ {window} avant la dernière sont supprimées maintenant. Ces jours gardent leurs scores, mais ne pourront pas être recalculés après les prochaines mises à jour.", "@dataKeepShorterBody": { "placeholders": { "window": { diff --git a/lib/l10n/app_hi.arb b/lib/l10n/app_hi.arb index 5efb70669..027cccad3 100644 --- a/lib/l10n/app_hi.arb +++ b/lib/l10n/app_hi.arb @@ -2814,7 +2814,7 @@ } } }, - "dataKeepShorterBody": "आपकी आख़िरी रीडिंग से {window} से ज़्यादा पहले की रीडिंग अभी हटा दी जाएंगी। उन दिनों के स्कोर बने रहेंगे, पर आगे के अपडेट के बाद उन्हें दोबारा नहीं आंका जा सकेगा।", + "dataKeepShorterBody": "आपकी आख़िरी रीडिंग से लगभग {window} से ज़्यादा पहले की रीडिंग अभी हटा दी जाएंगी। उन दिनों के स्कोर बने रहेंगे, पर आगे के अपडेट के बाद उन्हें दोबारा नहीं आंका जा सकेगा।", "@dataKeepShorterBody": { "placeholders": { "window": { diff --git a/lib/l10n/app_ru.arb b/lib/l10n/app_ru.arb index e405b89c8..c1d4b9fe6 100644 --- a/lib/l10n/app_ru.arb +++ b/lib/l10n/app_ru.arb @@ -7855,7 +7855,7 @@ } } }, - "dataKeepShorterBody": "Показания, снятые более чем за {window} до последнего, будут удалены сейчас. Эти дни сохранят свои оценки, но их нельзя будет пересчитать после будущих обновлений.", + "dataKeepShorterBody": "Показания, снятые более чем примерно за {window} до последнего, будут удалены сейчас. Эти дни сохранят свои оценки, но их нельзя будет пересчитать после будущих обновлений.", "@dataKeepShorterBody": { "placeholders": { "window": { diff --git a/lib/l10n/app_zh.arb b/lib/l10n/app_zh.arb index 1ced74f9e..786598d17 100644 --- a/lib/l10n/app_zh.arb +++ b/lib/l10n/app_zh.arb @@ -2814,7 +2814,7 @@ } } }, - "dataKeepShorterBody": "比最新读数早 {window} 以上的读数将立即删除。这些日子会保留评分，但在以后的更新后无法重新计算。", + "dataKeepShorterBody": "比最新读数早约 {window} 以上的读数将立即删除。这些日子会保留评分，但在以后的更新后无法重新计算。", "@dataKeepShorterBody": { "placeholders": { "window": { diff --git a/lib/ui2/profile/data.dart b/lib/ui2/profile/data.dart index c85c5023d..0cc36ac09 100644 --- a/lib/ui2/profile/data.dart +++ b/lib/ui2/profile/data.dart @@ -270,9 +270,9 @@ class _DataScreenState extends State { title: l?.dataKeepShorterTitle(window) ?? 'Keep $window of raw readings?', body: l?.dataKeepShorterBody(window) ?? - 'Readings more than $window before your latest one are deleted ' - 'now. Those days keep ' - "their scores, but can't be re-scored after future updates.", + 'Readings more than about $window before your latest one are ' + 'deleted now. Those days keep their scores, but ' + "can't be re-scored after future updates.", remove: l?.dataKeepShorterConfirm ?? 'Delete older readings', keep: l?.dataKeepShorterCancel ?? 'Keep them', ); From e5827440c8d1eb287f12bf2f4f779abfcbd85e45 Mon Sep 17 00:00:00 2001 From: abdulsaheel Date: Sun, 1 Nov 2026 01:43:00 +0530 Subject: [PATCH 30/33] run the automatic vacuum under the derive latch it started after a heavy derive without the latch, so a re-analysis or the archive replay inside it could read rowids a VACUUM was about to renumber. it now skips when a pass holds the latch and tries again after the next heavy derive. --- lib/compute/derivation_engine.dart | 8 ++++++++ lib/state/app_state.dart | 9 ++++++++- test/derive_if_idle_test.dart | 32 ++++++++++++++++++++++++++++++ 3 files changed, 48 insertions(+), 1 deletion(-) create mode 100644 test/derive_if_idle_test.dart diff --git a/lib/compute/derivation_engine.dart b/lib/compute/derivation_engine.dart index 4b29e3256..2ba99a93f 100644 --- a/lib/compute/derivation_engine.dart +++ b/lib/compute/derivation_engine.dart @@ -3143,6 +3143,14 @@ class DerivationEngine { } } + /// Run [body] under the derive latch if no pass is in flight, or return + /// [busy] without running it. For opportunistic storage work (the + /// automatic VACUUM) that must never overlap a derive or the archive + /// replay inside one, and is fine to skip. [body] must not call a locked + /// entry point. + static Future ifIdle(T busy, Future Function() body) => + _withRunLock(busy, body); + /// Run [body] once no pass is in flight, holding the latch so none starts /// until it returns. For user deletes: a pass that built its substrate /// before the delete keeps writing `day_result`, `metric_series` and the diff --git a/lib/state/app_state.dart b/lib/state/app_state.dart index ab9e69fe6..7398bec59 100644 --- a/lib/state/app_state.dart +++ b/lib/state/app_state.dart @@ -2212,7 +2212,14 @@ class AppState extends ChangeNotifier { if (_background || busy || _liveSessionActive) return; _vacuumedThisLaunch = true; try { - final freed = await LocalDb.vacuumIfBloated(); + // Under the derive latch, skipped if a pass holds it: a VACUUM can + // renumber the archive's rowids under a replay reading them. + final freed = + await DerivationEngine.ifIdle(-1, () => LocalDb.vacuumIfBloated()); + if (freed < 0) { + _vacuumedThisLaunch = false; // the next heavy derive tries again + return; + } if (freed > 0) { _log('[db] vacuum returned ${freed >> 20} MB to the filesystem'); } diff --git a/test/derive_if_idle_test.dart b/test/derive_if_idle_test.dart new file mode 100644 index 000000000..43648a847 --- /dev/null +++ b/test/derive_if_idle_test.dart @@ -0,0 +1,32 @@ +// The automatic VACUUM runs through DerivationEngine.ifIdle: never while a +// derive (and the archive replay inside it) holds the latch, and skipped +// rather than queued when one does. + +import 'package:flutter_test/flutter_test.dart'; +import 'package:openstrap_edge/compute/derivation_engine.dart'; + +void main() { + test('skips while a pass holds the latch, runs once it is free', () async { + expect(await DerivationEngine.acquireForReset(), isTrue); + var ran = false; + try { + final r = await DerivationEngine.ifIdle(-1, () async { + ran = true; + return 1; + }); + expect(r, -1); + expect(ran, isFalse); + } finally { + DerivationEngine.releaseForReset(); + } + expect( + await DerivationEngine.ifIdle(-1, () async { + ran = true; + // The latch is held while the body runs. + expect(await DerivationEngine.ifIdle(-2, () async => 0), -2); + return 1; + }), + 1); + expect(ran, isTrue); + }); +} From 00abfdaaf5e7c5716ce1ecf85734475860458bdb Mon Sep 17 00:00:00 2001 From: abdulsaheel Date: Sun, 1 Nov 2026 01:44:20 +0530 Subject: [PATCH 31/33] finish a blocked WAL truncation at the next derive when a reader blocked the compaction's truncating checkpoint twice, nothing tried again and the WAL kept its full size until some later checkpoint. it is remembered now and retried at the next derive's housekeeping, under the same latch. --- lib/compute/derivation_engine.dart | 13 +++++++++++-- lib/data/db.dart | 19 +++++++++++++++---- test/raw_archive_test.dart | 6 ++++++ 3 files changed, 32 insertions(+), 6 deletions(-) diff --git a/lib/compute/derivation_engine.dart b/lib/compute/derivation_engine.dart index 2ba99a93f..6cf629b72 100644 --- a/lib/compute/derivation_engine.dart +++ b/lib/compute/derivation_engine.dart @@ -3232,8 +3232,8 @@ class DerivationEngine { onStep?.call(CompactStep.vacuum); if (!await LocalDb.vacuumAndCheckpoint()) { // The size below is measured on disk, WAL included, so the result - // stays honest; the WAL shrinks at the next checkpoint. - _log('compact: WAL not truncated, a reader held it'); + // stays honest; the next derive's housekeeping truncates it. + _log('compact: WAL not truncated yet, a reader held it'); } return CompactResult( beforeBytes: before, @@ -7911,6 +7911,15 @@ class DerivationEngine { /// Off the path to a durable commit, and never inside a migration: `onUpgrade` /// runs under iOS's CPU watchdog (invariant 11). Future _runStorageHousekeeping({bool long = false}) async { + // A compaction whose WAL truncation a reader blocked: done now, under the + // same latch, instead of leaving the WAL at its full size. + if (LocalDb.walTruncatePending) { + try { + if (await LocalDb.truncateWal()) _log('WAL truncated after compaction'); + } catch (e) { + _log('WAL truncation skipped (will retry): $e'); + } + } try { // The pre-conversion undecodable archive, moved into the frame store a // bounded slice at a time (see LocalDb.migrateLegacyRawArchive). A diff --git a/lib/data/db.dart b/lib/data/db.dart index 51cf3d383..79394a644 100644 --- a/lib/data/db.dart +++ b/lib/data/db.dart @@ -10567,14 +10567,25 @@ class LocalDb { if (attempt > 0) { await Future.delayed(const Duration(milliseconds: 500)); } - if (checkpointCompleted( - await db.rawQuery('PRAGMA wal_checkpoint(TRUNCATE)'))) { - return true; - } + if (await truncateWal()) return true; } + walTruncatePending = true; return false; } + /// Set when a compaction's truncating checkpoint was blocked; the next + /// derive's housekeeping tries it again ([truncateWal]). + static bool walTruncatePending = false; + + /// One truncating WAL checkpoint. True when it finished (and clears + /// [walTruncatePending]); false when a reader blocked it. + static Future truncateWal() async { + final done = checkpointCompleted( + await (await instance).rawQuery('PRAGMA wal_checkpoint(TRUNCATE)')); + if (done) walTruncatePending = false; + return done; + } + /// Whether a `PRAGMA wal_checkpoint` result row says it finished: its first /// column is 1 when the checkpoint was blocked. @visibleForTesting diff --git a/test/raw_archive_test.dart b/test/raw_archive_test.dart index dd6163f9a..58b5f935f 100644 --- a/test/raw_archive_test.dart +++ b/test/raw_archive_test.dart @@ -411,4 +411,10 @@ void main() { ]), isTrue); }); + + test('a blocked truncation stays pending until one gets through', () async { + LocalDb.walTruncatePending = true; + expect(await LocalDb.truncateWal(), isTrue); + expect(LocalDb.walTruncatePending, isFalse); + }); } From 4f157ea7e697176f830d86d73c6a0c68b8281003 Mon Sep 17 00:00:00 2001 From: abdulsaheel Date: Sun, 1 Nov 2026 02:01:47 +0530 Subject: [PATCH 32/33] let a heavy pass wait for its disk reclaim before the next job the reclaim now holds the derive latch, and it was started without being awaited, so the scheduler's next job could start under it, find the latch taken and be marked done without deriving anything. --- lib/state/derive_coordinator.dart | 5 ++++- test/derive_coordinator_test.dart | 24 +++++++++++++++++++++++- 2 files changed, 27 insertions(+), 2 deletions(-) diff --git a/lib/state/derive_coordinator.dart b/lib/state/derive_coordinator.dart index 6fda9a642..234939ac9 100644 --- a/lib/state/derive_coordinator.dart +++ b/lib/state/derive_coordinator.dart @@ -180,7 +180,10 @@ class DeriveCoordinator { if (heavy && _healthShareConsent()) { unawaited(HealthUploader.instance.maybeUpload(consented: true)); } - if (heavy) unawaited(_maybeReclaimDiskSpace()); + // Awaited, so the scheduler's next job starts only once it is done: + // the vacuum holds the derive latch, and a job whose run() found it + // held would come back 0 and be marked complete without deriving. + if (heavy) await _maybeReclaimDiskSpace(); } catch (e, st) { _log('[derive] post-drain failed: $e'); // Was silently swallowed before — this is a real pipeline failure diff --git a/test/derive_coordinator_test.dart b/test/derive_coordinator_test.dart index 47344d1d7..29b98d6c4 100644 --- a/test/derive_coordinator_test.dart +++ b/test/derive_coordinator_test.dart @@ -31,6 +31,7 @@ void main() { bool Function()? disposed, bool healthSync = false, Future Function()? runHealthExport, + Future Function()? reclaim, }) => DeriveCoordinator( // Every test installs a debug hook for the engine calls, so any real @@ -52,7 +53,7 @@ void main() { healthSyncEnabled: () => healthSync, telemetryConsent: () => false, healthShareConsent: () => false, - maybeReclaimDiskSpace: () async => reclaims++, + maybeReclaimDiskSpace: reclaim ?? () async => reclaims++, ); test('constructing the coordinator and its scheduler never touches derive', @@ -116,6 +117,27 @@ void main() { expect(reclaims, 1); }); + test('a heavy pass ends only after the disk reclaim does', () async { + // The scheduler starts its next job when a pass returns. The reclaim + // holds the derive latch, so a job started under it would derive nothing. + final reclaim = Completer(); + final coordinator = make(reclaim: () => reclaim.future); + addTearDown(coordinator.dispose); + coordinator.debugDeriveRun = (profile, {heavy = false, onDayDone}) async => 0; + coordinator.debugRescanRecent = (_) async => 0; + + var finished = false; + final heavy = + coordinator.afterDrain(heavy: true).then((_) => finished = true); + for (var i = 0; i < 5; i++) { + await Future.delayed(Duration.zero); + } + expect(finished, isFalse); + reclaim.complete(); + await heavy; + expect(finished, isTrue); + }); + test('a rescan that throws is logged and the pass carries on', () async { final coordinator = make(healthSync: true); addTearDown(coordinator.dispose); From 45cb95ca3ace478b11c14b2bee40eae51a13adde Mon Sep 17 00:00:00 2001 From: abdulsaheel Date: Sun, 1 Nov 2026 02:01:53 +0530 Subject: [PATCH 33/33] leave a blocked WAL truncation to sqlite this reverts the pending-flag retry from 00abfdaa. auto-checkpoint keeps the WAL bounded and it is truncated on the next close and open, so a retry flag that only one isolate can see isn't worth carrying. the compaction still reads the checkpoint result, tries once more, and logs when it was blocked. --- lib/compute/derivation_engine.dart | 13 ++----------- lib/data/db.dart | 19 ++++--------------- test/raw_archive_test.dart | 6 ------ 3 files changed, 6 insertions(+), 32 deletions(-) diff --git a/lib/compute/derivation_engine.dart b/lib/compute/derivation_engine.dart index 6cf629b72..2ba99a93f 100644 --- a/lib/compute/derivation_engine.dart +++ b/lib/compute/derivation_engine.dart @@ -3232,8 +3232,8 @@ class DerivationEngine { onStep?.call(CompactStep.vacuum); if (!await LocalDb.vacuumAndCheckpoint()) { // The size below is measured on disk, WAL included, so the result - // stays honest; the next derive's housekeeping truncates it. - _log('compact: WAL not truncated yet, a reader held it'); + // stays honest; the WAL shrinks at the next checkpoint. + _log('compact: WAL not truncated, a reader held it'); } return CompactResult( beforeBytes: before, @@ -7911,15 +7911,6 @@ class DerivationEngine { /// Off the path to a durable commit, and never inside a migration: `onUpgrade` /// runs under iOS's CPU watchdog (invariant 11). Future _runStorageHousekeeping({bool long = false}) async { - // A compaction whose WAL truncation a reader blocked: done now, under the - // same latch, instead of leaving the WAL at its full size. - if (LocalDb.walTruncatePending) { - try { - if (await LocalDb.truncateWal()) _log('WAL truncated after compaction'); - } catch (e) { - _log('WAL truncation skipped (will retry): $e'); - } - } try { // The pre-conversion undecodable archive, moved into the frame store a // bounded slice at a time (see LocalDb.migrateLegacyRawArchive). A diff --git a/lib/data/db.dart b/lib/data/db.dart index 79394a644..51cf3d383 100644 --- a/lib/data/db.dart +++ b/lib/data/db.dart @@ -10567,25 +10567,14 @@ class LocalDb { if (attempt > 0) { await Future.delayed(const Duration(milliseconds: 500)); } - if (await truncateWal()) return true; + if (checkpointCompleted( + await db.rawQuery('PRAGMA wal_checkpoint(TRUNCATE)'))) { + return true; + } } - walTruncatePending = true; return false; } - /// Set when a compaction's truncating checkpoint was blocked; the next - /// derive's housekeeping tries it again ([truncateWal]). - static bool walTruncatePending = false; - - /// One truncating WAL checkpoint. True when it finished (and clears - /// [walTruncatePending]); false when a reader blocked it. - static Future truncateWal() async { - final done = checkpointCompleted( - await (await instance).rawQuery('PRAGMA wal_checkpoint(TRUNCATE)')); - if (done) walTruncatePending = false; - return done; - } - /// Whether a `PRAGMA wal_checkpoint` result row says it finished: its first /// column is 1 when the checkpoint was blocked. @visibleForTesting diff --git a/test/raw_archive_test.dart b/test/raw_archive_test.dart index 58b5f935f..dd6163f9a 100644 --- a/test/raw_archive_test.dart +++ b/test/raw_archive_test.dart @@ -411,10 +411,4 @@ void main() { ]), isTrue); }); - - test('a blocked truncation stays pending until one gets through', () async { - LocalDb.walTruncatePending = true; - expect(await LocalDb.truncateWal(), isTrue); - expect(LocalDb.walTruncatePending, isFalse); - }); }