diff --git a/android/app/src/main/kotlin/wtf/openstrap/openstrap_edge/NativeChannels.kt b/android/app/src/main/kotlin/wtf/openstrap/openstrap_edge/NativeChannels.kt index fdd6cb8e2..c8cd72b84 100644 --- a/android/app/src/main/kotlin/wtf/openstrap/openstrap_edge/NativeChannels.kt +++ b/android/app/src/main/kotlin/wtf/openstrap/openstrap_edge/NativeChannels.kt @@ -37,6 +37,7 @@ object NativeChannels { private const val DEVICE_ACTIONS_CHANNEL = "openstrap/device_actions" private const val ANDROID_BG_CHANNEL = "openstrap/android_background" private const val BLE_NATIVE_CHANNEL = "openstrap/ble_native" + private const val STORAGE_CHANNEL = "openstrap/storage" const val TASKER_CHANNEL = "openstrap/tasker" const val ACTION_DOUBLE_TAP = "wtf.openstrap.openstrap_edge.DOUBLE_TAP" private const val TASKER_TOKEN_KEY = "tasker_auth_token" @@ -183,6 +184,24 @@ object NativeChannels { } } + // Free space on the volume holding the database, for "Compact database". + MethodChannel(engine.dartExecutor.binaryMessenger, STORAGE_CHANNEL) + .setMethodCallHandler { call, result -> + when (call.method) { + "freeBytes" -> { + val path = call.argument("path") ?: app.filesDir.path + result.success( + try { + android.os.StatFs(path).availableBytes + } catch (e: IllegalArgumentException) { + null + } + ) + } + else -> result.notImplemented() + } + } + // Band-gesture actions. All no-risk OS APIs: media-key dispatch (works for any // player, no permission), system media volume, ringtone + vibrate, torch. MethodChannel(engine.dartExecutor.binaryMessenger, DEVICE_ACTIONS_CHANNEL) diff --git a/ios/Runner/AppDelegate.swift b/ios/Runner/AppDelegate.swift index 0d02f683f..384db0ee9 100644 --- a/ios/Runner/AppDelegate.swift +++ b/ios/Runner/AppDelegate.swift @@ -95,6 +95,10 @@ import CoreMotion if let registrar = registry.registrar(forPlugin: "ConfigBridge") { ConfigBridge.register(messenger: registrar.messenger()) } + // Free space on the volume holding the database, for "Compact database". + if let registrar = registry.registrar(forPlugin: "StorageBridge") { + StorageBridge.register(messenger: registrar.messenger()) + } // The phone's OWN step count (CMPedometer), not HealthKit's multi-writer // aggregate. See lib/health/phone_pedometer.dart. if let registrar = registry.registrar(forPlugin: "PedometerBridge") { @@ -123,6 +127,38 @@ import CoreMotion } } +enum StorageBridge { + private static let channelName = "openstrap/storage" + + static func register(messenger: FlutterBinaryMessenger) { + let channel = FlutterMethodChannel(name: channelName, binaryMessenger: messenger) + channel.setMethodCallHandler { call, result in + switch call.method { + case "freeBytes": + // Bytes a write can actually use. The "important usage" figure counts + // purgeable space iOS frees on demand, which is what a VACUUM sees. + let path = (call.arguments as? [String: Any])?["path"] as? String + ?? NSHomeDirectory() + let url = URL(fileURLWithPath: path) + do { + let values = try url.resourceValues( + forKeys: [.volumeAvailableCapacityForImportantUsageKey]) + if let bytes = values.volumeAvailableCapacityForImportantUsage { + result(NSNumber(value: bytes)) + return + } + let attrs = try FileManager.default.attributesOfFileSystem(forPath: path) + result(attrs[.systemFreeSize] as? NSNumber) + } catch { + result(nil) + } + default: + result(FlutterMethodNotImplemented) + } + } + } +} + enum ConfigBridge { private static let channelName = "openstrap/ios_config" diff --git a/lib/coach/coach_db.dart b/lib/coach/coach_db.dart index 7d968dac0..a3bab2501 100644 --- a/lib/coach/coach_db.dart +++ b/lib/coach/coach_db.dart @@ -97,7 +97,9 @@ class CoachDb { 'workout_suggestions', 'workout_route', 'live_coverage', 'session_score_mask', 'session_sensor', // raw / decoded substrate - 'raw_records', 'raw_archive', 'decoded_onehz', 'decoded_rr', 'samples', + 'raw_records', 'raw_archive', 'raw_archive_frames', 'raw_archive_legacy', + 'raw_archive_view', + 'decoded_onehz', 'decoded_rr', 'samples', 'events', 'band_events', 'band_battery', 'substrate_archive', 'device_coverage', 'signal_priority', // WHOOP MG ECG. `ecg_reading` is a base table of an allowed view, so the diff --git a/lib/compute/derivation_engine.dart b/lib/compute/derivation_engine.dart index cf2622c45..2ba99a93f 100644 --- a/lib/compute/derivation_engine.dart +++ b/lib/compute/derivation_engine.dart @@ -2813,6 +2813,47 @@ class _BaselineHistoryCache { return [for (var i = from; i < samples.length; i++) samples[i].value]; } + /// The dated quiet levels [_quietFingerprint] covers, newest included, as + /// `[date, value]` pairs: what [rescanRecent] compares against the list it + /// stored last time to find the days whose strain window moved. + List> quietWindowEntries() { + final series = _series['quiet_hrr'] ?? const <_DatedValue>[]; + const span = _rescanWindowDays + _baselineWindowDays + 1; + final from = series.length <= span ? 0 : series.length - span; + return [ + for (var i = from; i < series.length; i++) + [series[i].date, series[i].value], + ]; + } + + /// Every history value a derive of [date] can read, hashed: each series' + /// entries strictly before [date] with their method family (the windows + /// `valuesBefore`/`maxBefore` serve), plus every `dyn_p90` entry, whose + /// dates the frozen movement floor reads whole. Days after [date] are left + /// out, so today's values moving through the day do not unsettle the days + /// before it. + String inputsBefore(String date) { + final b = StringBuffer(); + for (final key in _series.keys.toList()..sort()) { + final series = _series[key]!; + final fams = _families[key]; + final end = + key == 'dyn_p90' ? series.length : _beforeIndex(series, date); + b.write('|$key:'); + for (var i = 0; i < end; i++) { + final v = series[i]; + b + ..write(v.date) + ..write('=') + ..write(v.value) + ..write('@') + ..write(fams?[v.date] ?? '') + ..write(','); + } + } + return sha1.convert(utf8.encode(b.toString())).toString(); + } + /// Strain's quiet-waking level is a baseline too, and [rescanRecent] has to /// re-price the recent days when ANY of their windows moves — a count and a /// median of the whole series do not see that: each day is priced on the @@ -2999,6 +3040,39 @@ Future _agingEdgeFor(String? family, int dataNowSec, Future debugAgingEdge(String? family, int dataNowSec) => _agingEdgeFor(family, dataNowSec); +/// Where a [DerivationEngine.compactStorage] has got to. +enum CompactStep { prune, vacuum } + +/// What a [DerivationEngine.compactStorage] did. Sizes are the database file +/// plus its WAL, in bytes. +class CompactResult { + const CompactResult({ + required this.beforeBytes, + required this.afterBytes, + required this.freeBytes, + required this.neededBytes, + required this.vacuumed, + this.deferred = false, + }); + + final int beforeBytes, afterBytes, neededBytes; + + /// True when the VACUUM was skipped because a writer that must not wait on + /// it (a sync, a live session) started while the prune ran. The prune + /// still ran. + final bool deferred; + + /// Free space on the volume when it was checked; null when the platform + /// could not say. + final int? freeBytes; + + /// False when the free-space gate refused the VACUUM, or it was [deferred] + /// (the prune still ran). + final bool vacuumed; + + int get freedBytes => beforeBytes > afterBytes ? beforeBytes - afterBytes : 0; +} + class DerivationEngine { DerivationEngine({this.log, this.background = false, this.offloadActive}); final void Function(String)? log; @@ -3069,6 +3143,14 @@ class DerivationEngine { } } + /// Run [body] under the derive latch if no pass is in flight, or return + /// [busy] without running it. For opportunistic storage work (the + /// automatic VACUUM) that must never overlap a derive or the archive + /// replay inside one, and is fine to skip. [body] must not call a locked + /// entry point. + static Future ifIdle(T busy, Future Function() body) => + _withRunLock(busy, body); + /// Run [body] once no pass is in flight, holding the latch so none starts /// until it returns. For user deletes: a pass that built its substrate /// before the delete keeps writing `day_result`, `metric_series` and the @@ -3089,6 +3171,79 @@ class DerivationEngine { } } + /// The user's "Compact database": the retention prune and archive eviction + /// a derive pass ends with, then a full `VACUUM` and a truncating WAL + /// checkpoint, all under the derive latch (waiting, like [whileIdle], for a + /// pass in flight). The prune is the pass's own: it holds unfinalized days + /// exactly as [_pruneOldDecoded] does, and it never touches anything a + /// drain has not committed, so commit-before-ACK is untouched — a drain that + /// starts meanwhile only waits on the database lock, and a commit that + /// cannot land is never acknowledged. + /// + /// The VACUUM runs only when [freeDiskBytes] reports at least twice the + /// database (plus its WAL) free: it writes a whole second copy before it + /// drops the first. Unknown free space counts as not enough. + /// + /// [busy] is asked again right before the VACUUM: the caller's own check + /// ran before the prune, and a sync that started since would otherwise sit + /// on the write lock for the whole rewrite. + Future compactStorage({ + required Future Function() freeDiskBytes, + void Function(CompactStep step)? onStep, + bool Function()? busy, + }) => + whileIdle(() async { + final before = await LocalDb.databaseBytesOnDisk(); + onStep?.call(CompactStep.prune); + try { + // The scope a pass would prune against: every day with substrate, + // the wearable's included, so a pending one is held. + final scope = await _deriveScope(heavy: true, force: false); + final dataNowSec = + await _withWearableEdge(await LocalDb.lastDecodedRecTs()) ?? 0; + if (dataNowSec > 0) { + await _pruneOldDecoded(scope.rawDays, dataNowSec); + } + } catch (e) { + _log('compact: prune skipped: $e'); + } + await _runStorageHousekeeping(long: true); + final size = await LocalDb.databaseBytesOnDisk(); + final free = await freeDiskBytes(); + if (free == null || free < 2 * size) { + return CompactResult( + beforeBytes: before, + afterBytes: size, + freeBytes: free, + neededBytes: 2 * size, + vacuumed: false, + ); + } + if (busy?.call() ?? false) { + return CompactResult( + beforeBytes: before, + afterBytes: size, + freeBytes: free, + neededBytes: 2 * size, + vacuumed: false, + deferred: true, + ); + } + onStep?.call(CompactStep.vacuum); + if (!await LocalDb.vacuumAndCheckpoint()) { + // The size below is measured on disk, WAL included, so the result + // stays honest; the WAL shrinks at the next checkpoint. + _log('compact: WAL not truncated, a reader held it'); + } + return CompactResult( + beforeBytes: before, + afterBytes: await LocalDb.databaseBytesOnDisk(), + freeBytes: free, + neededBytes: 2 * size, + vacuumed: true, + ); + }); + final Map _diag = { 'running': false, 'stage': 'idle', @@ -3107,6 +3262,7 @@ class DerivationEngine { 'todo_days': 0, 'done_days': 0, 'skipped_days': 0, + 'unchanged_days': 0, // List, not a single day — several days can be in flight concurrently // (see run()'s bounded worker pool). 'active_days': [], @@ -3124,6 +3280,80 @@ class DerivationEngine { /// Any later pass on this shared engine (a drain's light pass) resets them /// too, so a caller that must attribute the outcome to its own run passes /// [run]'s `onOutcome` instead of reading these after other awaits. + /// + /// Days [run] left alone in its last pass because nothing they read had + /// moved (see [_dayInputFingerprint]). + int lastRunUnchangedDays = 0; + + /// Test seam: derive every targeted day, as before the unchanged-day skip. + @visibleForTesting + static bool debugDeriveEveryDay = false; + + /// Test seam: the version [_dayInputFingerprint] keys on, to stand in for a + /// [kAlgoVersion] bump. + @visibleForTesting + static int? debugFingerprintAlgoVersion; + + /// Everything a derive of [dayId] reads, hashed, or null when it must + /// derive regardless (today: its result follows the wall clock). + /// + /// [run] re-derives every unfinalized day on every pass — on a soak with + /// no new data, four to six days at 8-14 s each, the whole pass for + /// nothing. A day is skipped when this matches the value stored by a + /// derive of it that was a FIXED POINT: the fingerprint read just before + /// that derive equalled the one read just after, so the derive rewrote + /// nothing it reads (its own row, its neighbours', the review state, the + /// sessions it scores). Deterministic compute over unchanged inputs then + /// gives the same result again, so not running it is the same as running + /// it. A day still converging (tonight's sleep landing on the next pass, a + /// review row it just wrote) derives again until it is one. + /// + /// Mixed in: the algorithm version, the profile, the data edge (stored in + /// the payload, so new data anywhere is a new result), the timezone the + /// day's window is cut in, the session-sensor globals, the baseline history + /// before the day, and [LocalDb.dayInputSignature] — the rows themselves. + Future _dayInputFingerprint( + String dayId, + Profile profile, + int dataNowSec, + _BaselineHistoryCache history, + ) async { + if (dayId == todayLabel()) return null; + final dayStart = _localDayLabelToSec(dayId); + final dayEnd = _localNextDayLabelToSec(dayId); + final wearable = await activeWearable(); + final rows = await LocalDb.dayInputSignature( + dayId: dayId, + prevDayId: localDateLabel(dayStart - 43200), + nextDayId: localDateLabel(dayEnd + 43200), + fromSec: dayStart - kNocturnalSearchLookbackSec - 86400, + toSec: dayEnd + napBoundaryBufferSec + 86400, + wearableDeviceId: wearable?.$1, + ); + if (rows == null) return null; + final now = DateTime.now(); + final finalized = (await LocalDb.finalizedDayIds(kAlgoVersion)).toList() + ..sort(); + final head = [ + 'v${debugFingerprintAlgoVersion ?? kAlgoVersion}', + dataNowSec, + // The edges a day is aged against (band and wearable; see + // [_agingEdgeFor]), from the pass's own memo. + await _agingEdgeFor(null, dataNowSec, history), + await (history._wearableEdge ??= wearableLastTs()), + now.timeZoneName, + now.timeZoneOffset.inMinutes, + dayStart, + dayEnd, + jsonEncode(profile.toMap()), + (LocalDb.sessionSensorSources.toList()..sort()).join(','), + LocalDb.sessionWearableSource, + '$wearable', + history.inputsBefore(dayId), + finalized.join(','), + ].join('|'); + return sha1.convert(utf8.encode('$head#$rows')).toString(); + } String? lastRunError; int lastRunFailedDays = 0; @@ -3163,11 +3393,13 @@ class DerivationEngine { ..['todo_days'] = 0 ..['done_days'] = 0 ..['skipped_days'] = 0 + ..['unchanged_days'] = 0 ..['active_days'] = [] ..['concurrency'] = _deriveConcurrency ..['last_error'] = null; lastRunError = null; lastRunFailedDays = 0; + lastRunUnchangedDays = 0; FirebaseTraceHandle? runTrace; try { @@ -3325,6 +3557,7 @@ class DerivationEngine { var done = 0; var completed = 0; var failures = 0; + var unchanged = 0; final activeDays = {}; _diag['stage'] = 'per_day'; _diag['active_days'] = const []; @@ -3339,10 +3572,59 @@ class DerivationEngine { // which Dart's cooperative single-threaded scheduler makes atomic // relative to the other concurrent workers even though the actual // isolate CPU work they await genuinely runs in parallel across cores. + // A forced pass and a day the user just edited always derive. + Future fingerprint(String dayId) async { + if (force || editedDays.contains(dayId) || debugDeriveEveryDay) { + return null; + } + try { + return await _dayInputFingerprint( + dayId, profile, dataNowSec, history); + } catch (e) { + _log('derive day $dayId: fingerprint failed, deriving: $e'); + return null; + } + } + + // The stored fingerprint is only an optimization: a read or write that + // fails (SQLITE_BUSY with two isolates deriving) must cost the skip, + // never the day. + Future storedFingerprint(String dayId) async { + try { + return await LocalDb.deriveFingerprint(dayId); + } catch (e) { + _log('derive day $dayId: fingerprint read failed, deriving: $e'); + return null; + } + } + + Future storeFingerprint(String dayId, String? value) async { + try { + await LocalDb.putDeriveFingerprint(dayId, value); + return true; + } catch (e) { + _log('derive day $dayId: fingerprint write failed: $e'); + return false; + } + } + Future processDay(String dayId) async { activeDays.add(dayId); _diag['active_days'] = activeDays.toList(); try { + final before = await fingerprint(dayId); + if (before != null && before == await storedFingerprint(dayId)) { + unchanged++; + _diag['unchanged_days'] = unchanged; + activeDays.remove(dayId); + _diag['active_days'] = activeDays.toList(); + completed++; + onDayDone?.call(dayId, completed, orderedDays.length); + return; + } + // Cleared before the derive so an interrupted one cannot leave a + // fingerprint vouching for the row it was rewriting. + final cleared = await storeFingerprint(dayId, null); final prepared = await _prepareTargetDay(dayId); // Override day whose raw has been pruned (≥14 d): re-deriving would // produce an empty/absent result and clobber the user's manual sleep. @@ -3353,9 +3635,21 @@ class DerivationEngine { _log('derive day $dayId skipped: override day, raw pruned — kept'); } else if (prepared != null) { _diag['prepared_days'] = (_diag['prepared_days'] as int) + 1; - await _derivePreparedDay(prepared, profile, dataNowSec, history); + final complete = await _derivePreparedDay( + prepared, profile, dataNowSec, history); done++; _diag['done_days'] = done; + // Stored only when the derive was a fixed point; see + // [_dayInputFingerprint]. Never for a partial or declined result + // (two failures in a row can store the same row, and the day + // would then never be retried), and never over a fingerprint + // that could not be cleared first. + if (complete && + cleared && + before != null && + before == await fingerprint(dayId)) { + await storeFingerprint(dayId, before); + } } else { _log('derive day $dayId skipped: no bounded window payload'); await _markDaySkipped( @@ -3393,8 +3687,15 @@ class DerivationEngine { _diag['stage'] = 'quiet_repass'; done += await _quietLevelRepass(profile, dataNowSec, scope.rawDays); - // 4. Cross-day rollup + notifications (best-effort). - if (done > 0) { + lastRunUnchangedDays = unchanged; + if (unchanged > 0) { + _log('derive: $unchanged day(s) unchanged since their last derive'); + } + + // 4. Cross-day rollup + notifications (best-effort). A day left alone + // because nothing it reads moved still counts: the rollup and the + // notifications ran on every such pass before the skip existed. + if (done > 0 || unchanged > 0) { _diag['stage'] = 'baselines'; await _refreshBaselines(); _diag['stage'] = 'cross_day'; @@ -3451,7 +3752,7 @@ class DerivationEngine { // `_running`. See _runStorageHousekeeping — this is the only place every // entry path and every early return actually reaches. _diag['stage'] = 'housekeeping'; - await _runStorageHousekeeping(); + await _runStorageHousekeeping(long: heavy || force); // ONE-SHOT: rescale stored strain onto the recalibrated scale. Days // inside the raw window re-derive from substrate above; everything older // has none, so its headline is rebuilt from the stored TRIMP + wake @@ -3562,6 +3863,7 @@ class DerivationEngine { ..['todo_days'] = 0 ..['done_days'] = 0 ..['skipped_days'] = 0 + ..['unchanged_days'] = 0 ..['active_days'] = [] ..['concurrency'] = _deriveConcurrency ..['last_error'] = null; @@ -5034,6 +5336,11 @@ class DerivationEngine { ); if (todoDays.isEmpty) { _log('rescan: no recent decoded-backed days'); + // Quiet snapshot first: a kill between the two writes must leave the + // signature stale (a cheap full rescan next tick), never a fresh + // signature over an old quiet snapshot that narrows the next rescan + // against the wrong levels. + await _storeQuietEntries(); await LocalDb.setCursor('baseline_sig', sig); return 0; } @@ -5043,11 +5350,31 @@ class DerivationEngine { ); final history = await _BaselineHistoryCache.load(); + // Only the quiet level moved: strain is priced on the quiet levels of + // the days BEFORE each day, so only a day after a changed level can + // score differently. The rest — finalized days included — keep their + // rows, which a re-derive would rewrite unchanged. + final prevQuiet = await LocalDb.getCursor(_baselineQuietCursor); + var rescanDays = todoDays; + if (prev != null && + prevQuiet != null && + _withoutQuietSignature(prev) == _withoutQuietSignature(sig)) { + final changed = changedQuietDates( + _decodeQuietEntries(prevQuiet), history.quietWindowEntries()); + if (changed != null) { + rescanDays = [ + for (final d in todoDays) + if (changed.any((c) => c.compareTo(d) < 0)) d, + ]; + _log('rescan: quiet level moved on $changed — ' + '${rescanDays.length} of ${todoDays.length} day(s) priced on it'); + } + } // Same bounded worker-pool pattern as run()/runDays — up to // _rescanWindowDays (21) days is exactly the kind of sweep that used // to run fully sequentially for no reason (independent day_id-keyed // writes + one frozen baseline snapshot shared read-only here). - final orderedDays = todoDays.reversed.toList(); + final orderedDays = rescanDays.reversed.toList(); var done = 0; var completed = 0; @@ -5069,11 +5396,15 @@ class DerivationEngine { await runWithConcurrency(orderedDays, _deriveConcurrency, processDay); - await _refreshBaselines(); - // Cross-day rollup + notifications reflect the refreshed scalars. - await _runCrossDay(profile); - await _runNotifications(); + if (orderedDays.isNotEmpty) { + await _refreshBaselines(); + // Cross-day rollup + notifications reflect the refreshed scalars. + await _runCrossDay(profile); + await _runNotifications(); + } // Store the new signature so the next tick is a cheap no-op until it moves. + // Quiet snapshot before the signature, for the reason given above. + await _storeQuietEntries(); await LocalDb.setCursor('baseline_sig', await _baselineSignature()); await LocalDb.refreshComputeFreshness(); return done; @@ -5086,6 +5417,57 @@ class DerivationEngine { } } + /// Cursor holding [_BaselineHistoryCache.quietWindowEntries] as of the + /// last `baseline_sig`, so a quiet-only change can be narrowed to the days + /// it reaches. + static const String _baselineQuietCursor = 'baseline_quiet'; + + Future _storeQuietEntries() async => LocalDb.setCursor( + _baselineQuietCursor, + jsonEncode((await _BaselineHistoryCache.load()).quietWindowEntries()), + ); + + /// The baseline signature without its quiet-level part (`|q…`, last). + static String _withoutQuietSignature(String sig) { + final i = sig.lastIndexOf('|q'); + return i < 0 ? sig : sig.substring(0, i); + } + + static List>? _decodeQuietEntries(String json) { + try { + return [ + for (final e in jsonDecode(json) as List) + [(e as List)[0] as String, (e[1] as num).toDouble()], + ]; + } catch (_) { + return null; + } + } + + /// The dates whose quiet level differs between [before] and [after] (both + /// `[date, value]`, ascending), over the dates both lists cover — an entry + /// that merely slid out of the older end of one is not a change. Null when + /// either is unreadable: the caller then rescans everything, as before. + @visibleForTesting + static Set? changedQuietDates( + List>? before, + List> after, + ) { + if (before == null) return null; + final a = {for (final e in before) e[0] as String: e[1]}; + final b = {for (final e in after) e[0] as String: e[1]}; + String? first(Map m) => + m.isEmpty ? null : (m.keys.toList()..sort()).first; + final fa = first(a), fb = first(b); + final cut = fa == null || fb == null + ? '' + : (fa.compareTo(fb) > 0 ? fa : fb); + return { + for (final d in {...a.keys, ...b.keys}) + if (d.compareTo(cut) >= 0 && a[d] != b[d]) d, + }; + } + /// The days [rescanRecent] re-derives: every day with substrate inside the /// rescan window, minus any whose derive window ([_targetDayWindow], from the /// previous noon) reaches below [prunedBeforeSec]. The prune cuts on local @@ -5405,7 +5787,11 @@ class DerivationEngine { ); } - Future _derivePreparedDay( + /// True only when the whole derive ran and its result was stored: a + /// decline that kept the existing row, or a second half that failed (a + /// partial row, however much of it was carried forward), is false, so the + /// caller never records that day as settled. + Future _derivePreparedDay( PreparedDerivationDay day, Profile profile, int dataNowSec, @@ -5604,7 +5990,7 @@ class DerivationEngine { if (_isRealDayResult(existing)) { _log('derive ${day.date}: no substrate (raw pruned) — kept the ' 'existing result rather than blanking it'); - return; + return false; } } @@ -5679,7 +6065,7 @@ class DerivationEngine { "under a day that already had real night scalars — kept the " 'existing result rather than overwriting the readiness baseline ' '(edge#305)'); - return; + return false; } } @@ -6307,6 +6693,7 @@ class DerivationEngine { ], ); } + return secondHalfOk; } /// Bank one day's (phone steps, counter ticks) pair for the (family, @@ -7373,7 +7760,15 @@ class DerivationEngine { /// filled in), and a transient skip never finalizes, so a day whose second /// half fails every single time never becomes finalized. One such day used /// to latch pruning off for the whole install, forever, at ~12 MB/day. - static const int _maxRawHoldDays = 14; + /// + /// Seven days behind the data edge, not fourteen. Nothing that can still + /// finalize needs longer: a day locks once the edge is [_finalizationSec] + /// past it, and a strain score it is owed comes from the days BEFORE it, + /// which [_quietLevelUnreachable] settles as soon as those have derived. A + /// day still unlocked a week on is one whose derive keeps failing, and the + /// second week of retries only cost another week of substrate (~12 MB a + /// day on a WHOOP 4, several times that on an MG). + static const int _maxRawHoldDays = 7; /// Cursor holding the highest `rec_ts` cutoff the raw prune has applied. static const String _prunedBeforeCursor = @@ -7510,10 +7905,25 @@ class DerivationEngine { /// every day reported 0 back to `reanalyzeAll`. This work is a storage /// optimization; nothing it does may change what the derive returns. /// - /// Bounded and resumable, so running it on every pass costs one small batch. + /// Bounded and resumable: a light pass gives it about three seconds (a page + /// of the archive move can run past that, never by more than one page), a + /// heavy or forced pass and an explicit compaction about fifteen. /// Off the path to a durable commit, and never inside a migration: `onUpgrade` /// runs under iOS's CPU watchdog (invariant 11). - Future _runStorageHousekeeping() async { + Future _runStorageHousekeeping({bool long = false}) async { + try { + // The pre-conversion undecodable archive, moved into the frame store a + // bounded slice at a time (see LocalDb.migrateLegacyRawArchive). A + // WHOOP MG's is ~500 MB of hex, so a background or light pass takes + // little. + final moved = await LocalDb.migrateLegacyRawArchive( + budget: Duration(seconds: long && !background ? 15 : 3), + shouldStop: offloadActive, + ); + if (moved > 0) _log('moved $moved archived frame(s) to the frame store'); + } catch (e) { + _log('raw archive move skipped (will retry): $e'); + } try { final reencoded = await LocalDb.reencodeLegacyDayResults(); if (reencoded > 0) { @@ -7521,7 +7931,7 @@ class DerivationEngine { } // Substrate archive retention, measured against the data edge like the // live prune. Off the commit path; vacuumIfBloated reclaims the pages. - final edge = await LocalDb.lastDecodedRecTs(); + final edge = await LocalDb.substrateArchiveEdge(); if (edge != null) { final evicted = await LocalDb.evictSubstrateArchive( await LocalDb.substrateArchivePolicy(), diff --git a/lib/compute/inputs/colmi_inputs.dart b/lib/compute/inputs/colmi_inputs.dart index c95f559ff..b9823df78 100644 --- a/lib/compute/inputs/colmi_inputs.dart +++ b/lib/compute/inputs/colmi_inputs.dart @@ -56,7 +56,7 @@ Future> colmiSkinTemps( // ring's history (kColmiHistoryDays) later: scan only those captures. for (final r in await db.rawQuery( 'SELECT hex, COALESCE(rec_ts * 1000, captured_at) AS anchor_ms ' - 'FROM raw_archive WHERE device_id = ? ' + 'FROM ${LocalDb.rawArchiveView} WHERE device_id = ? ' "AND reason = 'colmi_big_0x${kColmiBigTemperature.toRadixString(16)}' " 'AND captured_at >= ? AND captured_at <= ?', [ @@ -143,7 +143,7 @@ const Set kColmiNightRows = { Future colmiSleepUndecoded(String deviceId) async { final db = await LocalDb.instance; final rows = await db.rawQuery( - 'SELECT hex FROM raw_archive WHERE device_id = ? AND reason = ? ' + 'SELECT hex FROM ${LocalDb.rawArchiveView} WHERE device_id = ? AND reason = ? ' 'ORDER BY captured_at DESC LIMIT 1', [deviceId, 'colmi_cmd_0x${kColmiCmdSetTime.toRadixString(16).padLeft(2, '0')}'], ); diff --git a/lib/compute/inputs/ultrahuman_inputs.dart b/lib/compute/inputs/ultrahuman_inputs.dart index 720c38fea..dd467784d 100644 --- a/lib/compute/inputs/ultrahuman_inputs.dart +++ b/lib/compute/inputs/ultrahuman_inputs.dart @@ -49,7 +49,7 @@ Future> ultrahumanRecords( final db = await LocalDb.instance; final out = []; for (final r in await db.rawQuery( - "SELECT hex FROM raw_archive WHERE device_id = ? " + "SELECT hex FROM ${LocalDb.rawArchiveView} WHERE device_id = ? " "AND reason = 'ultrahuman_record' AND rec_ts >= ? AND rec_ts <= ? " 'ORDER BY rec_ts', [deviceId, from, to], diff --git a/lib/data/db.dart b/lib/data/db.dart index e5b3c8b95..51cf3d383 100644 --- a/lib/data/db.dart +++ b/lib/data/db.dart @@ -17,7 +17,8 @@ import 'dart:io'; import 'dart:isolate'; import 'dart:typed_data'; -import 'package:flutter/foundation.dart' show visibleForTesting; +import 'package:crypto/crypto.dart' show sha1; +import 'package:flutter/foundation.dart' show debugPrint, visibleForTesting; import 'package:openstrap_protocol/openstrap_protocol.dart' as proto; import 'package:path/path.dart' as p; import 'package:path_provider/path_provider.dart'; @@ -576,7 +577,7 @@ class LocalDb { await db.execute('CREATE INDEX idx_samples_ts ON samples(ts)'); await _createEvents(db); await _createBandSignals(db); - await _createRawArchive(db); + await _ensureRawArchiveStore(db); await _createDerived(db); await _createDayResult(db); await _createUserTables(db); @@ -774,7 +775,7 @@ class LocalDb { // the thinning in [thinRawArchiveBefore]) so a future firmware's // records can be re-decoded. Also add `millivolts` to band_battery // for the battery-health series. Both additive. - await _createRawArchive(db); + await _createLegacyRawArchive(db); await _ensureBandBatteryMillivolts(db); } if (oldV < 22) { @@ -886,7 +887,7 @@ class LocalDb { 'ALTER TABLE raw_archive RENAME TO _raw_archive_old', ); await db.execute('DROP INDEX IF EXISTS idx_raw_archive_captured'); - await _createRawArchive(db); + await _createLegacyRawArchive(db); await db.execute( 'INSERT OR IGNORE INTO raw_archive ' '(hex, counter, packet_type, rec_ts, captured_at, reason) ' @@ -895,7 +896,7 @@ class LocalDb { ); await db.execute('DROP TABLE _raw_archive_old'); } else { - await _createRawArchive(db); + await _createLegacyRawArchive(db); } } if (oldV < 33) { @@ -1236,6 +1237,15 @@ class LocalDb { }, onOpen: (db) async { await _repairOpenSchema(db); + // This phone's own database only: [_repairOpenSchema] also opens + // export copies, where it would write a setting the user never chose. + // Best effort — a failure must not get the database quarantined; the + // next open retries it. + try { + await SubstrateArchive.keepLegacyWindowOnce(db); + } catch (e) { + debugPrint('[db] archive window check skipped: $e'); + } }, version: schemaVersion, ); @@ -1253,9 +1263,11 @@ class LocalDb { await _createBandSignals(db); await _ensureBandBatteryMillivolts(db); await _ensureBandBatteryChargeUnits(db); - await _createRawArchive(db); + await _ensureRawArchiveStore(db); await _createDerived(db); await _createDayResult(db); + await _ensureDayResultImportedColumn(db); + await _createDeriveFingerprint(db); await _createUserTables(db); await _createSyncState(db); await _createSyncCursor(db); @@ -1431,7 +1443,8 @@ class LocalDb { } /// The column names [table] currently has (empty if the table is absent). - static Future> _columnsOf(Database db, String table) async { + static Future> _columnsOf( + DatabaseExecutor db, String table) async { final info = await db.rawQuery('PRAGMA table_info($table)'); return { for (final c in info) @@ -4747,15 +4760,19 @@ class LocalDb { // aside BEFORE the caller writes the batch-ACK that lets the band trim. if (archives != null) { for (final a in archives) { - batch.insert('raw_archive', { - 'device_id': deviceId, - 'counter': a.counter, - 'hex': a.hex, - 'packet_type': a.packetType, - 'rec_ts': a.recTs, - 'captured_at': a.capturedAt, - 'reason': a.reason, - }, conflictAlgorithm: ConflictAlgorithm.ignore); + batch.insert( + rawArchiveTable, + rawArchiveRow( + deviceId: deviceId, + hex: a.hex, + counter: a.counter, + packetType: a.packetType, + recTs: a.recTs, + capturedAt: a.capturedAt, + reason: a.reason, + ), + conflictAlgorithm: ConflictAlgorithm.ignore, + ); if (++ops >= chunkOps) await flushChunk(); } } @@ -7629,38 +7646,336 @@ class LocalDb { if (queued > 0) await batch.commit(noResult: true); } + /// The `(device_id, hex)`-keyed table the ladder rungs below v58 build and + /// re-key ([_rekeyByDeviceIdV51]). Only the ladder creates it now; the open + /// pass converts it into [rawArchiveTable] ([_ensureRawArchiveStore]). + static Future _createLegacyRawArchive(Database db) async { + await db.execute(''' + CREATE TABLE IF NOT EXISTS raw_archive ( + device_id TEXT NOT NULL DEFAULT '$kPrimaryDeviceId', + hex TEXT NOT NULL, + counter INTEGER, + packet_type INTEGER NOT NULL, + rec_ts INTEGER, + captured_at INTEGER NOT NULL, + reason TEXT NOT NULL, + PRIMARY KEY (device_id, hex) + ) + '''); + await db.execute( + 'CREATE INDEX IF NOT EXISTS idx_raw_archive_captured ' + 'ON raw_archive(captured_at DESC)', + ); + } + + /// Where archived frames are stored: [_createRawArchiveFrames]. Readers keep + /// reading the `raw_archive` view, which gives the frame back as `hex`. + static const String rawArchiveTable = 'raw_archive_frames'; + + /// A pre-conversion `raw_archive` table, renamed aside on open and moved + /// into [rawArchiveTable] in pages by [migrateLegacyRawArchive]. + static const String _rawArchiveLegacy = 'raw_archive_legacy'; + + /// The view every reader goes through ([_ensureRawArchiveView]). NOT named + /// `raw_archive`: an older build opening this file after a rollback runs + /// `CREATE TABLE IF NOT EXISTS raw_archive` and then `CREATE INDEX IF NOT + /// EXISTS idx_raw_archive_captured ON raw_archive`, and SQLite refuses to + /// index a view before it looks at the name, so a view under the old name + /// failed that open and got the database quarantined. With the name free, + /// the old build makes itself an empty table, the index line is a no-op + /// (the name is taken by [rawArchiveTable]'s), and the next open of this + /// build folds whatever it wrote back in ([_ensureRawArchiveStore]). + static const String rawArchiveView = 'raw_archive_view'; + + /// The frame as the `raw_archive` view's `hex` column: the stored text for a + /// frame kept verbatim, else the bytes in lowercase hex — exactly the + /// string [rawArchiveRow] was handed. + static const String _rawArchiveHexSql = 'COALESCE(hex, lower(hex(frame)))'; + /// Durable archive for historical records we received but could not decode /// (unknown/unsupported version). Thinned to a stated SAMPLE RATE behind the /// retention edge and otherwise kept forever — the point is that a future /// firmware's records survive until we understand the format, and a sample is /// what a decode needs. See [thinRawArchiveBefore] for the numbers. Keyed by - /// frame `hex` (content identity), like `events`/`band_events` — NOT by - /// `counter`. The strap resets its record counter to ~0 on every reboot, so - /// two DISTINCT undecodable frames from different boots can collide on a - /// reused counter; a `counter`-PK + IGNORE silently DROPPED the second, in - /// the one table whose whole purpose is to never lose a frame. Hashing on the - /// bytes means an identical re-flood (missed-ACK redelivery) still dedups, - /// while genuinely distinct frames both survive a counter collision. `counter` - /// is retained as a plain forensic column. - static Future _createRawArchive(Database db) async { + /// frame CONTENT (`hash`), like `events`/`band_events` — NOT by `counter`. + /// The strap resets its record counter to ~0 on every reboot, so two + /// DISTINCT undecodable frames from different boots can collide on a reused + /// counter; a `counter`-PK + IGNORE silently DROPPED the second, in the one + /// table whose whole purpose is to never lose a frame. Keying on the bytes + /// means an identical re-flood (missed-ACK redelivery) still dedups, while + /// genuinely distinct frames both survive a counter collision. `counter` is + /// retained as a plain forensic column. + /// + /// THE BYTES, NOT THEIR HEX, AND KEYED BY A HASH. The table this replaces + /// stored each frame as hex TEXT under `PRIMARY KEY (device_id, hex)` on a + /// rowid table, so every frame was written twice — once in the row and once + /// in the key's autoindex — at two characters a byte: a 2,128-byte WHOOP MG + /// v20 frame cost ~9 KB, ~10,500 of them a day. Here it is one BLOB under a + /// 20-byte SHA-1 of it, ~4.2 KB a frame (a 2.2 KB row cannot share a 4 KB + /// page with a second one, so each takes a page). `frame` is last so a + /// scan of the small columns (thinning, stats) never walks its overflow + /// pages. A rowid table still: [_mergeFromDbFile] pages restores by rowid. + /// + /// `hex` keeps the text verbatim, with `frame` NULL, when it is not the + /// canonical lowercase hex of some bytes — so nothing is ever rewritten on + /// the way in, whatever a caller handed over. + static Future _createRawArchiveFrames(Database db) async { await db.execute(''' - CREATE TABLE IF NOT EXISTS raw_archive ( + CREATE TABLE IF NOT EXISTS $rawArchiveTable ( device_id TEXT NOT NULL DEFAULT '$kPrimaryDeviceId', - hex TEXT NOT NULL, + hash BLOB NOT NULL, counter INTEGER, packet_type INTEGER NOT NULL, rec_ts INTEGER, captured_at INTEGER NOT NULL, reason TEXT NOT NULL, - PRIMARY KEY (device_id, hex) + hex TEXT, + frame BLOB, + UNIQUE (device_id, hash) ) '''); + // Same name as the old table's index. An older build opening this file + // runs `CREATE INDEX IF NOT EXISTS idx_raw_archive_captured ON + // raw_archive`, which the existing name turns into a no-op instead of an + // attempt to index a view (which fails the open). await db.execute( 'CREATE INDEX IF NOT EXISTS idx_raw_archive_captured ' - 'ON raw_archive(captured_at DESC)', + 'ON $rawArchiveTable(captured_at DESC)', + ); + // Every reader filters on reason (the re-drive, the vendor inputs, the + // thinning) and most on device too. + await db.execute( + 'CREATE INDEX IF NOT EXISTS idx_raw_archive_reason ' + 'ON $rawArchiveTable(reason, device_id)', + ); + } + + /// `raw_archive` as every reader has always seen it, over [rawArchiveTable] + /// (and the legacy table while [migrateLegacyRawArchive] is still moving + /// it). The INSTEAD OF triggers keep plain inserts and deletes against the + /// old name working — an older build after a rollback, a test fixture — + /// with the text itself as the key: correct, just without the size saving. + /// Everything in this build writes through [rawArchiveRow] instead. + static Future _ensureRawArchiveView(DatabaseExecutor db) async { + final legacy = await _objectType(db, _rawArchiveLegacy) == 'table'; + await db.execute('DROP TRIGGER IF EXISTS raw_archive_insert'); + await db.execute('DROP TRIGGER IF EXISTS raw_archive_delete'); + // The view's old name, from a build of this layout before the rename. + if (await _objectType(db, 'raw_archive') == 'view') { + await db.execute('DROP VIEW raw_archive'); + } + await db.execute('DROP VIEW IF EXISTS $rawArchiveView'); + await db.execute( + 'CREATE VIEW $rawArchiveView AS ' + 'SELECT device_id, $_rawArchiveHexSql AS hex, counter, packet_type, ' + 'rec_ts, captured_at, reason FROM $rawArchiveTable' + '${legacy ? ' UNION ALL SELECT device_id, hex, counter, packet_type, ' + 'rec_ts, captured_at, reason FROM $_rawArchiveLegacy' : ''}', + ); + await db.execute( + 'CREATE TRIGGER IF NOT EXISTS raw_archive_insert ' + 'INSTEAD OF INSERT ON $rawArchiveView BEGIN ' + 'INSERT OR IGNORE INTO $rawArchiveTable ' + '(device_id, hash, counter, packet_type, rec_ts, captured_at, reason, hex) ' + "VALUES (COALESCE(NEW.device_id, '$kPrimaryDeviceId'), " + "CAST('text:' || NEW.hex AS BLOB), NEW.counter, NEW.packet_type, " + 'NEW.rec_ts, NEW.captured_at, NEW.reason, NEW.hex); END', + ); + await db.execute( + 'CREATE TRIGGER IF NOT EXISTS raw_archive_delete ' + 'INSTEAD OF DELETE ON $rawArchiveView BEGIN ' + 'DELETE FROM $rawArchiveTable WHERE device_id = OLD.device_id ' + 'AND $_rawArchiveHexSql = OLD.hex; ' + '${legacy ? 'DELETE FROM $_rawArchiveLegacy WHERE device_id = ' + 'OLD.device_id AND hex = OLD.hex; ' : ''}END', ); } + /// `'table'`, `'view'`, … for [name] in sqlite_master, or null. + static Future _objectType(DatabaseExecutor db, String name) async { + final rows = await db.rawQuery( + 'SELECT type FROM sqlite_master WHERE name = ? LIMIT 1', + [name], + ); + return rows.isEmpty ? null : rows.first['type'] as String?; + } + + /// The archive store, on every open. A database still holding the old + /// `raw_archive` TABLE has it renamed aside — O(1), no row touched, so safe + /// under the launch watchdog (invariant 11) — and the rows move later, in + /// pages, off the open path ([migrateLegacyRawArchive]). The view unions + /// both until then, so no reader ever sees a frame missing. + static Future _ensureRawArchiveStore(Database db) async { + if (await _objectType(db, 'raw_archive') == 'table') { + // One transaction: a crash between the rename and the index drop would + // leave the frames table without its captured_at index until the + // legacy table is gone, because the index name is still taken. + await db.transaction((txn) async { + // A build from before the v51 rekey, rolled back to, makes this table + // without device_id; the view and the move both read it. Filled the + // way the v51 rekey filled it: the primary device. + if (!(await _columnsOf(txn, 'raw_archive')).contains('device_id')) { + await txn.execute( + 'ALTER TABLE raw_archive ADD COLUMN device_id TEXT NOT NULL ' + "DEFAULT '$kPrimaryDeviceId'", + ); + } + if (await _objectType(txn, _rawArchiveLegacy) == 'table') { + // Only reachable if a file was hand-assembled with both; fold the + // newer table into the aside one rather than lose either. + await txn.execute( + 'INSERT OR IGNORE INTO $_rawArchiveLegacy ' + '(device_id, hex, counter, packet_type, rec_ts, captured_at, ' + 'reason) SELECT device_id, hex, counter, packet_type, rec_ts, ' + 'captured_at, reason FROM raw_archive', + ); + await txn.execute('DROP TABLE raw_archive'); + } else { + await txn + .execute('ALTER TABLE raw_archive RENAME TO $_rawArchiveLegacy'); + } + // Frees the name for [rawArchiveTable]'s index (see there). + await txn.execute('DROP INDEX IF EXISTS idx_raw_archive_captured'); + }); + } + await _createRawArchiveFrames(db); + if (await _objectType(db, rawArchiveView) != 'view' || + await _objectType(db, 'raw_archive') == 'view') { + await _ensureRawArchiveView(db); + } else if ((await _objectType(db, _rawArchiveLegacy) == 'table') != + (await db.rawQuery( + "SELECT 1 FROM sqlite_master WHERE type = 'view' " + "AND name = '$rawArchiveView' AND sql LIKE '%$_rawArchiveLegacy%'", + )) + .isNotEmpty) { + // The view and the legacy table disagree (a restore, a crash between + // the drop and the rebuild): rebuild the view and its triggers. + await db.execute('DROP TRIGGER IF EXISTS raw_archive_insert'); + await db.execute('DROP TRIGGER IF EXISTS raw_archive_delete'); + await _ensureRawArchiveView(db); + } + } + + /// [hex] as bytes when it is canonical lowercase hex (even length, 0-9a-f), + /// else null — so `lower(hex(frame))` gives back exactly [hex]. + static Uint8List? _canonicalHexBytes(String hex) { + final n = hex.length; + if (n == 0 || n.isOdd) return null; + final out = Uint8List(n >> 1); + int nibble(int c) { + if (c >= 0x30 && c <= 0x39) return c - 0x30; + if (c >= 0x61 && c <= 0x66) return c - 0x61 + 10; + return -1; + } + + for (var i = 0; i < n; i += 2) { + final hi = nibble(hex.codeUnitAt(i)), lo = nibble(hex.codeUnitAt(i + 1)); + if (hi < 0 || lo < 0) return null; + out[i >> 1] = (hi << 4) | lo; + } + return out; + } + + /// The [rawArchiveTable] row for one archived frame. Every writer in this + /// build goes through here (the drain commit, the fallback path, the + /// legacy move, a restore), so the key is always the same function of the + /// frame: SHA-1 of the bytes, or of the text when it is kept verbatim. + static Map rawArchiveRow({ + required String deviceId, + required String hex, + required int? counter, + required int packetType, + required int? recTs, + required int capturedAt, + required String reason, + }) { + final bytes = _canonicalHexBytes(hex); + return { + 'device_id': deviceId, + 'hash': Uint8List.fromList( + bytes != null + ? sha1.convert(bytes).bytes + : sha1.convert(utf8.encode('text:$hex')).bytes, + ), + 'counter': counter, + 'packet_type': packetType, + 'rec_ts': recTs, + 'captured_at': capturedAt, + 'reason': reason, + 'hex': bytes == null ? hex : null, + 'frame': bytes, + }; + } + + /// Move the renamed-aside legacy archive into [rawArchiveTable], 500 rows a + /// transaction: each page is inserted and deleted from the legacy table in + /// the same commit, so a kill between pages loses nothing and repeats + /// nothing. Once it is empty the legacy table is dropped and the view + /// rebuilt over the new table alone. Returns rows moved; stops after the + /// first page past [budget], or when [shouldStop] says a band offload has + /// started (its commits would queue behind these pages), and resumes on + /// the next call. + static Future migrateLegacyRawArchive({ + Duration? budget, + bool Function()? shouldStop, + }) async { + final db = await instance; + if (await _objectType(db, _rawArchiveLegacy) != 'table') return 0; + final clock = Stopwatch()..start(); + var moved = 0; + while (true) { + final page = await db.rawQuery( + 'SELECT rowid AS _r, device_id, hex, counter, packet_type, rec_ts, ' + 'captured_at, reason FROM $_rawArchiveLegacy ORDER BY rowid LIMIT 500', + ); + if (page.isEmpty) { + await db.transaction((txn) async { + if ((await txn.rawQuery( + 'SELECT 1 FROM $_rawArchiveLegacy LIMIT 1')) + .isNotEmpty) { + return; // a row landed since; the next call moves it + } + await txn.execute('DROP TRIGGER IF EXISTS raw_archive_insert'); + await txn.execute('DROP TRIGGER IF EXISTS raw_archive_delete'); + await txn.execute('DROP VIEW IF EXISTS $rawArchiveView'); + await txn.execute('DROP TABLE $_rawArchiveLegacy'); + // Rebuilt in the same transaction: between a commit and a rebuild + // outside it there is no raw_archive at all, and a reader on + // another isolate fails with "no such table". + await _ensureRawArchiveView(txn); + }); + return moved; + } + final hi = (page.last['_r'] as num).toInt(); + await db.transaction((txn) async { + final batch = txn.batch(); + for (final r in page) { + batch.insert( + rawArchiveTable, + rawArchiveRow( + deviceId: r['device_id'] as String? ?? kPrimaryDeviceId, + hex: r['hex'] as String, + counter: (r['counter'] as num?)?.toInt(), + packetType: (r['packet_type'] as num?)?.toInt() ?? 0, + recTs: (r['rec_ts'] as num?)?.toInt(), + capturedAt: (r['captured_at'] as num?)?.toInt() ?? 0, + reason: r['reason'] as String? ?? 'unknown', + ), + conflictAlgorithm: ConflictAlgorithm.ignore, + ); + } + batch.rawDelete( + 'DELETE FROM $_rawArchiveLegacy WHERE rowid <= ?', + [hi], + ); + await batch.commit(noResult: true); + }); + moved += page.length; + if (budget != null && clock.elapsed > budget) return moved; + if (shouldStop?.call() ?? false) return moved; + } + } + /// One compressed, verified bucket of pruned decoded substrate per /// (device_id, utc_day) — see substrate_archive.dart. `utc_day` is /// `rec_ts ~/ 86400`, an absolute storage partition and NEVER a day label. @@ -7988,15 +8303,19 @@ class LocalDb { String deviceId = kPrimaryDeviceId, }) async { final db = await instance; - await db.insert('raw_archive', { - 'device_id': deviceId, - 'counter': a.counter, - 'hex': a.hex, - 'packet_type': a.packetType, - 'rec_ts': a.recTs, - 'captured_at': a.capturedAt, - 'reason': a.reason, - }, conflictAlgorithm: ConflictAlgorithm.ignore); + await db.insert( + rawArchiveTable, + rawArchiveRow( + deviceId: deviceId, + hex: a.hex, + counter: a.counter, + packetType: a.packetType, + recTs: a.recTs, + capturedAt: a.capturedAt, + reason: a.reason, + ), + conflictAlgorithm: ConflictAlgorithm.ignore, + ); } /// The `raw_archive.reason` values [redriveArchivedRecords] will retry. @@ -8072,6 +8391,9 @@ class LocalDb { /// `compute_freshness` key for [redriveArchiveOnce]. static const String kArchiveRedriveKey = 'archive_redrive_v12_v1'; + /// Marks a [_redrive] resume position that is a [rawArchiveTable] rowid. + static const String _redriveRowidCursor = '\u0000rowid'; + /// One-shot replay of every [redrivableArchiveReasons] row, off the launch /// path (called from a heavy derive). Marked done only after it finishes, /// so an interrupted pass repeats; the replay skips seconds already decoded, @@ -8090,6 +8412,12 @@ class LocalDb { if (v is Map) state = v.cast(); } if (state['done'] == true) return 0; + // Wait for the legacy archive to finish moving: the replay reads the new + // store only, and marking it done over a half-moved archive would skip + // the rest for good. + if (await _objectType(await instance, _rawArchiveLegacy) == 'table') { + return 0; + } final row = await computeFreshness(kRedriveExcludedKey); final excluded = row == null ? const <(int, int)>[] : _parseWindows(row['payload_json']); @@ -8119,6 +8447,23 @@ class LocalDb { return r.recovered; } + /// Drop an unfinished [redriveArchiveOnce]'s resume point before a VACUUM. + /// That position is a [rawArchiveTable] rowid, and the table has no + /// INTEGER PRIMARY KEY, so VACUUM is free to renumber it: resuming there + /// could skip frames. The replay skips seconds already decoded, so starting + /// over from the top only costs time. + static Future _resetRedriveResume() async { + final row = await computeFreshness(kArchiveRedriveKey); + if (row?['payload_json'] case final String j) { + final v = jsonDecode(j); + if (v is! Map || v['done'] == true || v['after_hex'] == null) return; + await putComputeFreshness( + kArchiveRedriveKey, + jsonEncode({'recovered': v['recovered'] ?? 0}), + ); + } + } + /// GATES 4b — replay the archive through the CURRENT decoders, into /// `decoded_onehz` / `decoded_rr`. Returns the seconds recovered. /// @@ -8165,13 +8510,26 @@ class LocalDb { // Both are cheap and both are required: a DB whose ladder has not created // the archive yet (or is mid-ladder) must no-op rather than throw — a throw // in here rolls the WHOLE upgrade back and quarantines the database. - for (final t in const ['raw_archive', 'decoded_onehz']) { + // The archive is the `raw_archive` table on the ladder's older layouts, + // and [rawArchiveView] over [rawArchiveTable] once converted. + final framesLayout = await _objectType(db, rawArchiveView) == 'view'; + for (final t in [framesLayout ? rawArchiveView : 'raw_archive', + 'decoded_onehz']) { final present = await db.rawQuery( - "SELECT 1 FROM sqlite_master WHERE type='table' AND name=?", + "SELECT 1 FROM sqlite_master WHERE type IN ('table', 'view') " + 'AND name=?', [t], ); if (present.isEmpty) return (recovered: 0, resume: null); } + // The converted store pages on its own rowid (the hex is computed, so + // paging on it would sort the archive every page). Its cursor travels in + // the same (String, String) slot, marked by [_redriveRowidCursor]; a + // position from the old hex-ordered pass restarts at the top, which the + // replay's skip-what-is-decoded rule makes safe. + var afterRowid = framesLayout && startDevice == _redriveRowidCursor + ? int.tryParse(startHex) ?? 0 + : 0; // SELF-DETECTING, because this runs from BOTH sides of the v47 rung: from // the oldV<44 ladder step, where `decoded_onehz` is still keyed by rec_ts // alone and naming `device_id` would throw inside onUpgrade (quarantining @@ -8183,7 +8541,7 @@ class LocalDb { // rekey (v51): this function runs from the oldV<44 rung too, i.e. BEFORE // the v51 rekey, so at that point `raw_archive` is still hex-keyed and the // row-value comparison below must fall back to comparing `hex` alone. - final preArchiveDeviceKey = + final preArchiveDeviceKey = !framesLayout && !(await _columnsOf(db, 'raw_archive')).contains('device_id'); final marks = List.filled(reasons.length, '?').join(','); @@ -8192,11 +8550,19 @@ class LocalDb { // Post-v51, `hex` alone is no longer unique // (two devices can share a byte-identical undecodable frame), so a plain // `hex > ?` cursor would silently skip the second device's row. - var afterHex = startHex; - var afterDevice = startDevice; + var afterHex = framesLayout ? '$afterRowid' : startHex; + var afterDevice = framesLayout ? _redriveRowidCursor : startDevice; var recovered = 0; while (true) { - final rows = preArchiveDeviceKey + final rows = framesLayout + ? await db.rawQuery( + 'SELECT rowid AS _r, device_id, $_rawArchiveHexSql AS hex, ' + 'counter, packet_type, rec_ts, captured_at ' + 'FROM $rawArchiveTable WHERE reason IN ($marks) AND rowid > ? ' + 'ORDER BY rowid ASC LIMIT 500', + [...reasons, afterRowid], + ) + : preArchiveDeviceKey ? await db.rawQuery( 'SELECT hex, counter, packet_type, rec_ts, captured_at ' 'FROM raw_archive WHERE reason IN ($marks) AND hex > ? ' @@ -8213,8 +8579,14 @@ class LocalDb { ); if (rows.isEmpty) return (recovered: recovered, resume: null); final pageHex = afterHex, pageDevice = afterDevice; - afterHex = rows.last['hex'] as String; - afterDevice = preArchiveDeviceKey ? '' : rows.last['device_id'] as String; + if (framesLayout) { + afterRowid = (rows.last['_r'] as num).toInt(); + afterHex = '$afterRowid'; + } else { + afterHex = rows.last['hex'] as String; + afterDevice = + preArchiveDeviceKey ? '' : rows.last['device_id'] as String; + } // Decode first, keyed by (device, second) each record claims. Two // archived frames for the SAME device and second collapse here rather @@ -8394,14 +8766,24 @@ class LocalDb { DatabaseExecutor txn, int beforeMs, ) async { - return txn.delete( - 'raw_archive', - // `counter` is NOT NULL in practice but the column is nullable, and - // `NULL % 60` is NULL — so a counter-less row would never match and would - // be kept. Keeping an unsampleable frame is the safe direction. - where: 'reason = ? AND captured_at < ? AND counter % ? != 0', - whereArgs: [_thinnedArchiveReason, beforeMs, rawArchiveKeepEvery], - ); + var deleted = 0; + // The store and, until [migrateLegacyRawArchive] has emptied it, the + // renamed-aside legacy table: the same rule over both. + for (final t in [ + rawArchiveTable, + if (await _objectType(txn, _rawArchiveLegacy) == 'table') + _rawArchiveLegacy, + ]) { + deleted += await txn.delete( + t, + // `counter` is NOT NULL in practice but the column is nullable, and + // `NULL % 60` is NULL — so a counter-less row would never match and + // would be kept. Keeping an unsampleable frame is the safe direction. + where: 'reason = ? AND captured_at < ? AND counter % ? != 0', + whereArgs: [_thinnedArchiveReason, beforeMs, rawArchiveKeepEvery], + ); + } + return deleted; } /// How many undecodable records we've archived, and by reason — for the sync @@ -8410,11 +8792,11 @@ class LocalDb { final db = await instance; final count = Sqflite.firstIntValue( - await db.rawQuery('SELECT COUNT(*) FROM raw_archive'), + await db.rawQuery('SELECT COUNT(*) FROM $rawArchiveView'), ) ?? 0; final byReason = await db.rawQuery( - 'SELECT reason, COUNT(*) AS n FROM raw_archive ' + 'SELECT reason, COUNT(*) AS n FROM $rawArchiveView ' 'GROUP BY reason ORDER BY n DESC, reason ASC', ); return { @@ -9659,6 +10041,286 @@ class LocalDb { return {for (final r in rows) r['day_id'] as String}; } + /// Every stored input a derive of [dayId] reads, as one string, or null + /// when the derive can depend on the wall clock (a session with no end, + /// which activity review treats as running until now). The derivation + /// engine hashes this with the profile, the algorithm version, the data + /// edge and the baseline history, and skips a day whose hash has not moved + /// since a derive that changed nothing it reads (see + /// `DerivationEngine._dayInputFingerprint`). + /// + /// CONSERVATIVE BY CONSTRUCTION: windows are wider than any one the derive + /// loads ([fromSec]..[toSec] is the night search through the nap buffer + /// with a day of slack either side), whole small tables are read where the + /// derive reads part of them, and the only columns left out are write + /// stamps (`computed_at`, `updated_at`, `updated_ts`) and the device row's + /// battery/last-seen readings, none of which a derive reads. Over-reading + /// costs a derive that could have been skipped; under-reading would serve + /// a stale day, so anything in doubt is in. + static Future dayInputSignature({ + required String dayId, + required String prevDayId, + required String nextDayId, + required int fromSec, + required int toSec, + required String? wearableDeviceId, + }) async { + final db = await instance; + final out = StringBuffer(); + String rows(List> rs, {Set drop = const {}}) { + final keys = rs.isEmpty + ? const [] + : (rs.first.keys.where((k) => !drop.contains(k)).toList()..sort()); + return jsonEncode([ + for (final r in rs) [for (final k in keys) r[k]], + ]); + } + + Future add(String label, String sql, + [List args = const [], Set drop = const {}]) async { + out + ..write(label) + ..write('=') + ..write(rows(await db.rawQuery(sql, args), drop: drop)) + ..write(';'); + } + + Future has(String table) async => + (await _objectType(db, table)) != null; + + final days = [prevDayId, dayId, nextDayId]; + const stamps = {'computed_at', 'updated_at', 'updated_ts'}; + final openSession = await db.rawQuery( + 'SELECT 1 FROM sessions WHERE end_ts IS NULL AND start_ts <= ? LIMIT 1', + [toSec + 86400], + ); + if (openSession.isNotEmpty) return null; + + final oneHzCols = { + for (final c in await db.rawQuery('PRAGMA table_info(decoded_onehz)')) + c['name'] as String, + }..removeAll(const {'device_id', 'device_family', 'source'}); + await add( + 'onehz', + 'SELECT COUNT(*), TOTAL(rowid), MAX(rowid), ' + '${[for (final c in oneHzCols.toList()..sort()) 'TOTAL($c)'].join(', ')}, ' + 'TOTAL(LENGTH(device_id)), TOTAL(LENGTH(device_family)), ' + 'TOTAL(LENGTH(source)) ' + 'FROM decoded_onehz WHERE rec_ts BETWEEN ? AND ?', + [fromSec, toSec], + ); + await add( + 'rr', + 'SELECT COUNT(*), TOTAL(rowid), MAX(rowid), TOTAL(rr_ms), ' + 'TOTAL(rr_ts_ms), TOTAL(LENGTH(device_id)) ' + 'FROM decoded_rr WHERE rec_ts BETWEEN ? AND ?', + [fromSec, toSec], + ); + // The wear and charge transitions are read back past the window (the + // state already open at its start), so all of them count. + await add( + 'band_events', + 'SELECT COUNT(*), TOTAL(rowid), MAX(rowid) FROM band_events ' + 'WHERE (ts BETWEEN ? AND ?) OR event_id IN (?, ?, ?, ?)', + [ + fromSec, + toSec, + proto.EventId.chargingOn, + proto.EventId.chargingOff, + proto.EventId.wristOn, + proto.EventId.wristOff, + ], + ); + await add('live_coverage', + 'SELECT * FROM live_coverage WHERE day = ? ORDER BY rowid', [dayId]); + await add( + 'sessions', + 'SELECT * FROM sessions WHERE start_ts BETWEEN ? AND ? ORDER BY id', + [fromSec - 2 * 86400, toSec + 86400], + ); + for (final t in const [ + 'session_sensor', + 'session_score_mask', + 'signal_priority', + 'step_calibration', + 'activity_review_meta', + 'activity_review_days', + ]) { + if (await has(t)) { + await add(t, 'SELECT * FROM $t ORDER BY rowid', const [], stamps); + } + } + final marks = List.filled(days.length, '?').join(','); + await add('sleep_override', + 'SELECT * FROM sleep_override WHERE day_id IN ($marks) ORDER BY day_id', + days, stamps); + if (await has('sleep_nap')) { + await add( + 'sleep_nap', + 'SELECT * FROM sleep_nap WHERE day_id IN ($marks) ' + 'OR (start_ts < ? AND end_ts > ?) ORDER BY rowid', + [...days, toSec, fromSec], + stamps, + ); + } + if (await has('activity_suggestions')) { + await add( + 'activity_suggestions', + 'SELECT * FROM activity_suggestions WHERE day_id IN ($marks) ' + 'OR (detection_start < ? AND detection_end > ?) ORDER BY id', + [...days, toSec + 86400, fromSec - 86400], + stamps, + ); + } + await add( + 'device', + 'SELECT * FROM device ORDER BY id', + const [], + {...stamps, 'last_seen', 'battery_pct', 'battery_ts', 'name'}, + ); + await add( + 'cursor', + "SELECT name, value FROM sync_cursor WHERE name IN (?, ?) " + "OR name LIKE 'wearable_enabled:%' ORDER BY name", + ['active_wearable', kDecodedPrunedBeforeCursor], + ); + await add( + 'device_coverage', + 'SELECT * FROM device_coverage WHERE end_ts > ? AND start_ts < ? ' + 'ORDER BY rowid', + [fromSec, toSec], + ); + if (await has('vendor_sleep_epoch')) { + await add( + 'vendor_sleep', + 'SELECT COUNT(*), TOTAL(rowid), MAX(rowid) FROM vendor_sleep_epoch ' + 'WHERE night_onset_ts BETWEEN ? AND ?', + [fromSec - 86400, toSec + 86400], + ); + } + if (wearableDeviceId != null) { + // A wearable's substrate is read without a window (its whole HR + // history sets its edge) and from the archive (rings bank replies + // there), so all of that device's rows count. + await add( + 'wearable_rows', + 'SELECT COUNT(*), TOTAL(rowid), MAX(rowid), TOTAL(hr) ' + 'FROM decoded_onehz WHERE device_id = ?', + [wearableDeviceId], + ); + // Read off the tables, not the view: the view's hex is built from each + // frame's bytes, which was text per frame per day for a length. A + // stored blob's hex is exactly twice its bytes, and the columns keep the + // view query's names (the signature orders values by them), so the + // value — and every stored fingerprint — is the same as before. + final legacyArchive = await has(_rawArchiveLegacy); + await add( + 'wearable_archive', + 'SELECT COUNT(*) AS "COUNT(*)", ' + 'MAX(captured_at) AS "MAX(captured_at)", ' + 'TOTAL(captured_at) AS "TOTAL(captured_at)", ' + 'TOTAL(n) AS "TOTAL(LENGTH(hex))" FROM (' + 'SELECT captured_at, CASE WHEN hex IS NOT NULL THEN LENGTH(hex) ' + 'ELSE LENGTH(frame) * 2 END AS n FROM $rawArchiveTable ' + 'WHERE device_id = ?' + '${legacyArchive ? ' UNION ALL SELECT captured_at, LENGTH(hex) ' + 'FROM $_rawArchiveLegacy WHERE device_id = ?' : ''})', + [wearableDeviceId, if (legacyArchive) wearableDeviceId], + ); + } + await add( + 'baselines', + "SELECT key, payload_json FROM baselines " + "WHERE key IN ('sleep_user_profile', 'movement_floor') ORDER BY key", + ); + await add( + 'freshness', + 'SELECT key, payload_json FROM compute_freshness WHERE key = ?', + ['quiet_repass:$dayId'], + ); + // The day's own rows and its neighbours' — read back by the carry-forward + // guards, tonight's sleep and activity review. Hashed, not inlined. + final results = await db.rawQuery( + 'SELECT day_id, algo_version, finalized, partial, skipped, rhr, rmssd, ' + 'readiness, window_json, payload_json FROM day_result ' + 'WHERE day_id IN ($marks) ORDER BY day_id, algo_version', + days, + ); + out.write('day_result='); + for (final r in results) { + final payload = r['payload_json']; + out + ..write([ + for (final k in const [ + 'day_id', 'algo_version', 'finalized', 'partial', 'skipped', + 'rhr', 'rmssd', 'readiness', 'window_json', + ]) + r[k], + ].join(',')) + ..write(',') + ..write(payload is String + ? sha1.convert(utf8.encode(payload)).toString() + : 'null') + ..write('|'); + } + out.write(';'); + await add( + 'candidates', + 'SELECT day_id, algo_version, payload_json FROM sleep_session_candidates ' + 'WHERE day_id = ? ORDER BY algo_version', + [dayId], + ); + out + ..write('windows=') + ..write(rows(await sleepWindowRows(60))) + ..write(';'); + return out.toString(); + } + + /// The input fingerprint the last derive of [dayId] stored, when that + /// derive changed nothing it reads — see `DerivationEngine`'s skip. + static Future deriveFingerprint(String dayId) async { + final db = await instance; + final r = await db.query(_deriveFingerprintTable, + columns: ['fp'], where: 'day_id = ?', whereArgs: [dayId], limit: 1); + return r.isEmpty ? null : r.first['fp'] as String?; + } + + /// Store (or, with null, clear) [dayId]'s fingerprint. Rows two months + /// older than [dayId] go with it: only days still re-deriving are ever + /// looked up. Measured from the day, not the wall clock, so a backfill of + /// old days does not delete its own rows. + static Future putDeriveFingerprint(String dayId, String? fp) async { + final db = await instance; + await db.transaction((txn) async { + if (fp == null) { + await txn.delete(_deriveFingerprintTable, + where: 'day_id = ?', whereArgs: [dayId]); + return; + } + await txn.insert(_deriveFingerprintTable, {'day_id': dayId, 'fp': fp}, + conflictAlgorithm: ConflictAlgorithm.replace); + final t = DateTime.tryParse(dayId); + if (t != null) { + await txn.delete(_deriveFingerprintTable, + where: 'day_id < ?', + whereArgs: [dayLabelOf(DateTime(t.year, t.month - 2, t.day))]); + } + }); + } + + static const String _deriveFingerprintTable = 'derive_fingerprint'; + + /// One row per day: the fingerprint above. Recomputable bookkeeping — + /// losing it costs one derive per day — so additive on the open pass with + /// no version bump, like the other repair-path tables. + static Future _createDeriveFingerprint(Database db) async { + await db.execute( + 'CREATE TABLE IF NOT EXISTS $_deriveFingerprintTable ' + '(day_id TEXT PRIMARY KEY, fp TEXT NOT NULL)', + ); + } + /// The set of day_id labels that are FINALIZED at [algoVersion] (locked). A /// finalized day is never recomputed even on a version bump. /// @@ -9666,18 +10328,110 @@ class LocalDb { /// no rows for yet, and band rows for it arriving later must still derive /// and replace the snapshot ([isMeasuredDayRow]). Every caller intersects /// this with days that have raw, so a raw-less import is still left alone. + /// + /// Read off [_dayResultFinalizedIndex] once [_ensureDayResultImportedFlag] + /// has filled `day_result.imported`. The LIKE it replaces read every + /// finalized day's whole bundle — and `finalized` itself sits after + /// `payload_json` in the row, so even the plain column test walked each + /// bundle's overflow pages — several times per derive pass. static Future> finalizedDayIds(int algoVersion) async { final db = await instance; + final indexed = await _ensureDayResultImportedFlag(db); final rows = await db.query( 'day_result', columns: ['day_id'], - where: 'algo_version = ? AND finalized = 1 ' - "AND payload_json NOT LIKE '%\"imported\":true%'", + where: indexed + ? 'algo_version = ? AND finalized = 1 AND imported = 0' + : 'algo_version = ? AND finalized = 1 ' + 'AND payload_json NOT LIKE $_importedPayloadLike', whereArgs: [algoVersion], ); return {for (final r in rows) r['day_id'] as String}; } + /// The one test for "this bundle is an imported snapshot". The `imported` + /// column is computed from it by trigger, so the column and the old scan + /// cannot disagree about a row. + static const String _importedPayloadLike = "'%\"imported\":true%'"; + + /// Covering index for [finalizedDayIds]. Created only once every existing + /// row's `imported` is filled, so its presence is also the "backfill done" + /// marker: a database restored from a file without it backfills again. + static const String _dayResultFinalizedIndex = 'idx_day_result_finalized'; + + /// `day_result.imported` and the triggers that keep it equal to + /// [_importedPayloadLike] on every insert and payload update — whichever + /// path writes the row (derive, import, restore merge, raw SQL). Cheap and + /// idempotent: ADD COLUMN with a constant default rewrites nothing, so this + /// is safe on the every-open repair path. The backfill of existing rows is + /// NOT here; see [_ensureDayResultImportedFlag]. + static Future _ensureDayResultImportedColumn(Database db) async { + if ((await _columnsOf(db, 'day_result')).isEmpty) return; + await _addColumnIfMissing( + db, 'day_result', 'imported', 'INTEGER NOT NULL DEFAULT 0', + ); + await db.execute( + 'CREATE TRIGGER IF NOT EXISTS trg_day_result_imported_ins ' + 'AFTER INSERT ON day_result BEGIN ' + 'UPDATE day_result ' + 'SET imported = (NEW.payload_json LIKE $_importedPayloadLike) ' + 'WHERE rowid = NEW.rowid; END', + ); + await db.execute( + 'CREATE TRIGGER IF NOT EXISTS trg_day_result_imported_upd ' + 'AFTER UPDATE OF payload_json ON day_result BEGIN ' + 'UPDATE day_result ' + 'SET imported = (NEW.payload_json LIKE $_importedPayloadLike) ' + 'WHERE rowid = NEW.rowid; END', + ); + } + + /// True once `day_result.imported` is trustworthy for every row and + /// [_dayResultFinalizedIndex] exists. The first call on a database that + /// predates the column fills it in pages of 200 rows, each its own short + /// statement, then builds the index — off `openDatabase`, so a long history + /// never runs it under the launch watchdog. Rows written meanwhile are + /// already right (the triggers), and re-filling one recomputes the same + /// value, so a concurrent derive cannot race it. Any failure leaves the + /// index absent and the caller on the old scan; the next call resumes. + static Future _ensureDayResultImportedFlag(Database db) async { + Future ready() async => (await db.rawQuery( + "SELECT 1 FROM sqlite_master WHERE type = 'index' AND name = ?", + [_dayResultFinalizedIndex], + )) + .isNotEmpty; + if (await ready()) return true; + try { + if (!(await _columnsOf(db, 'day_result')).contains('imported')) { + await _ensureDayResultImportedColumn(db); + } + var after = 0; + while (true) { + final page = await db.rawQuery( + 'SELECT rowid AS r FROM day_result WHERE rowid > ? ' + 'ORDER BY rowid LIMIT 200', + [after], + ); + if (page.isEmpty) break; + final hi = (page.last['r'] as num).toInt(); + await db.rawUpdate( + 'UPDATE day_result ' + 'SET imported = (payload_json LIKE $_importedPayloadLike) ' + 'WHERE rowid > ? AND rowid <= ?', + [after, hi], + ); + after = hi; + } + await db.execute( + 'CREATE INDEX IF NOT EXISTS $_dayResultFinalizedIndex ' + 'ON day_result(algo_version, finalized, imported, day_id)', + ); + return true; + } catch (_) { + return false; + } + } + /// Normalize a day_result row to also carry a `date` key (== day_id) so legacy /// readers that keyed on `date` keep working. static Map _withDate(Map row) { @@ -9781,6 +10535,52 @@ class LocalDb { return major > 3 || (major == 3 && minor >= 27); } + /// The database file plus its `-wal`: what the user's storage settings + /// show it costing. After a VACUUM in WAL mode the WAL can be as big as the + /// file until a checkpoint truncates it. + static Future databaseBytesOnDisk() async { + final path = p.join(await getDatabasesPath(), dbName); + var total = 0; + for (final f in [File(path), File('$path-wal')]) { + if (await f.exists()) total += await f.length(); + } + return total; + } + + /// Rewrite the file without its free pages, then hand the WAL's copy of + /// that rewrite back to the filesystem. The caller gates it (free space, + /// nothing else in flight); see [vacuumIfBloated] for why this is a one-off + /// VACUUM and not `auto_vacuum`. The checkpoint is a `rawQuery`: the PRAGMA + /// returns a row, which `execute` refuses on iOS's sqflite. + /// + /// True when the WAL was truncated too. A reader still holding a snapshot + /// does not make the checkpoint throw: it returns busy = 1 in its first + /// column and leaves the WAL as it was, so the row is read, and tried once + /// more after a moment. + static Future vacuumAndCheckpoint() async { + final db = await instance; + await _resetRedriveResume(); + await db.execute('VACUUM'); + // VACUUM renumbers every btree root page; see [vacuumIfBloated]. + await CoachDb.close(); + for (var attempt = 0; attempt < 2; attempt++) { + if (attempt > 0) { + await Future.delayed(const Duration(milliseconds: 500)); + } + if (checkpointCompleted( + await db.rawQuery('PRAGMA wal_checkpoint(TRUNCATE)'))) { + return true; + } + } + return false; + } + + /// Whether a `PRAGMA wal_checkpoint` result row says it finished: its first + /// column is 1 when the checkpoint was blocked. + @visibleForTesting + static bool checkpointCompleted(List> rows) => + rows.isEmpty || rows.first.values.first != 1; + static Future databaseFileBytes() async { final dir = await getDatabasesPath(); final path = p.join(dir, dbName); @@ -10697,6 +11497,59 @@ class LocalDb { 'strength_set': (['session_id'], null), }; + /// `raw_archive` out of a backup, whichever layout wrote it: the frame + /// store of this build, the hex-keyed table of an older one, or both (a + /// file saved mid-move). Every row is rebuilt through [rawArchiveRow], so a + /// frame already here, or in the backup twice, lands once. Paged on rowid, + /// like every other table the merge reads. Returns rows offered. + static Future _mergeRawArchive(Database src, Database db) async { + var copied = 0; + Future pageFrom(String table, String hexSql) async { + if (await _objectType(src, table) != 'table') return; + // Pre-v51 files have no device_id: every row is the primary band's. + final hasDevice = (await src.rawQuery('PRAGMA table_info($table)')) + .any((c) => c['name'] == 'device_id'); + final deviceSql = + hasDevice ? 'device_id' : "'$kPrimaryDeviceId' AS device_id"; + var after = 0; + while (true) { + final page = await src.rawQuery( + 'SELECT rowid AS _r, $deviceSql, $hexSql AS hex, counter, ' + 'packet_type, rec_ts, captured_at, reason FROM $table ' + 'WHERE rowid > ? ORDER BY rowid LIMIT 500', + [after], + ); + if (page.isEmpty) return; + after = (page.last['_r'] as num).toInt(); + final batch = db.batch(); + for (final r in page) { + final hex = r['hex']; + if (hex is! String) continue; + batch.insert( + rawArchiveTable, + rawArchiveRow( + deviceId: r['device_id'] as String? ?? kPrimaryDeviceId, + hex: hex, + counter: (r['counter'] as num?)?.toInt(), + packetType: (r['packet_type'] as num?)?.toInt() ?? 0, + recTs: (r['rec_ts'] as num?)?.toInt(), + capturedAt: (r['captured_at'] as num?)?.toInt() ?? 0, + reason: r['reason'] as String? ?? 'unknown', + ), + conflictAlgorithm: ConflictAlgorithm.ignore, + ); + copied++; + } + await batch.commit(noResult: true); + } + } + + await pageFrom('raw_archive', 'hex'); // a table only in older files + await pageFrom(rawArchiveTable, _rawArchiveHexSql); + await pageFrom(_rawArchiveLegacy, 'hex'); + return copied; + } + @visibleForTesting static List get restoreTablesForTest => _restoreTables; @@ -10777,6 +11630,15 @@ class LocalDb { : (srcEdge == null || localEdge > srcEdge ? localEdge : srcEdge); } for (final t in (only ?? tables)) { + if (t == 'raw_archive') { + try { + counts[t] = await _mergeRawArchive(src, db); + } catch (_) { + if (!tolerant) rethrow; + counts[t] = 0; + } + continue; + } try { // PAGED SOURCE READ — never `SELECT *` a whole table. // @@ -13972,6 +14834,17 @@ class LocalDb { int dataNowSec, ) async => SubstrateArchive.evict(await instance, policy, dataNowSec); + /// The data edge the archive window is measured from: the newest band + /// reading, or, on a restored database with an archive but no decoded rows + /// yet, the archive's own newest reading — without it a shorter window + /// would be saved and nothing ever deleted. Null only with neither. + static Future substrateArchiveEdge() async { + final band = await lastDecodedRecTs(); + if (band != null) return band; + final newest = (await substrateArchiveStats())['newest_to_ts']; + return newest is num ? newest.toInt() : null; + } + /// `{buckets, bytes, raw_bytes, oldest_from_ts, newest_to_ts}` of the /// substrate archive. static Future> substrateArchiveStats() async => @@ -14029,6 +14902,7 @@ class LocalDb { final free = await freelistBytes(); if (free < minFreeBytes) return 0; final db = await instance; + await _resetRedriveResume(); await db.execute('VACUUM'); // VACUUM rewrites the file and renumbers every btree root page. CoachDb // caches the root pages of its allow-listed views to decide what a coach diff --git a/lib/data/disk_space.dart b/lib/data/disk_space.dart new file mode 100644 index 000000000..838158cca --- /dev/null +++ b/lib/data/disk_space.dart @@ -0,0 +1,31 @@ +// Free space on the volume that holds the database, from the platform. +// +// Only "Compact database" asks: a VACUUM writes a full second copy of the file +// before it drops the first, so it must not start on a nearly full phone. + +import 'package:flutter/foundation.dart' show visibleForTesting; +import 'package:flutter/services.dart'; +import 'package:sqflite/sqflite.dart' show getDatabasesPath; + +const MethodChannel _channel = MethodChannel('openstrap/storage'); + +/// Test seam: when set, answers instead of the platform. +@visibleForTesting +Future Function()? debugFreeDiskBytes; + +/// Bytes free for the app on the database's volume, or null when the platform +/// cannot say (no channel, a desktop test, an error). Null is "unknown", and +/// every caller treats unknown as not enough. +Future freeDiskBytes() async { + final override = debugFreeDiskBytes; + if (override != null) return override(); + try { + final v = await _channel.invokeMethod( + 'freeBytes', + {'path': await getDatabasesPath()}, + ); + return v is num ? v.toInt() : null; + } catch (_) { + return null; + } +} diff --git a/lib/data/recent_slice.dart b/lib/data/recent_slice.dart index 3cd1f24ee..a97a59523 100644 --- a/lib/data/recent_slice.dart +++ b/lib/data/recent_slice.dart @@ -46,6 +46,8 @@ const Map kSliceTables = { 'decoded_rr': (SliceKeep.highRate, 'rec_ts', 'beat-to-beat intervals'), 'samples': (SliceKeep.highRate, 'ts', 'per-second heart rate'), 'raw_archive': (SliceKeep.highRate, 'captured_at', 'records the app could not decode, by when they arrived'), + 'raw_archive_frames': (SliceKeep.highRate, 'captured_at', 'records the app could not decode, by when they arrived'), + 'raw_archive_legacy': (SliceKeep.highRate, 'captured_at', 'the same records in the older layout, until they are moved'), 'events': (SliceKeep.recent, 'ts', 'strap event stream'), 'band_events': (SliceKeep.recent, 'ts', 'decoded strap events'), 'band_battery': (SliceKeep.recent, 'ts', 'battery readings every few minutes'), @@ -79,6 +81,7 @@ const Map kSliceTables = { 'metric_method': (SliceKeep.all, null, 'how each daily metric was measured'), 'baselines': (SliceKeep.all, null, 'personal baselines'), 'compute_freshness': (SliceKeep.all, null, 'compute bookkeeping'), + 'derive_fingerprint': (SliceKeep.all, null, 'which days changed since they were last scored'), 'sleep_nap': (SliceKeep.all, null, 'naps'), 'sleep_override': (SliceKeep.all, null, 'sleep times the user edited'), // Workouts and activity: one row each. diff --git a/lib/data/substrate_archive.dart b/lib/data/substrate_archive.dart index e27a4125b..e8920bb03 100644 --- a/lib/data/substrate_archive.dart +++ b/lib/data/substrate_archive.dart @@ -39,9 +39,33 @@ import 'substrate_archive_codec.dart'; class SubstrateArchivePolicy { const SubstrateArchivePolicy(this.keepDays); - /// One year. Chosen before a measurement on real exports; revisit with the - /// bytes/day the size probe reports (see substrate_archive_size_probe_test). - static const SubstrateArchivePolicy defaults = SubstrateArchivePolicy(365); + /// Ninety days. The archive costs ~1.6 MB per UTC day of WHOOP substrate, + /// so a year was ~600 MB on disk for history nothing re-derives from yet. + /// What the window gives up: a day older than it has no substrate left + /// anywhere, so a future algorithm change can never re-score it — it keeps + /// the score of the version that derived it. Users who want that history + /// kept pick a longer window ([choices]). + static const SubstrateArchivePolicy defaults = SubstrateArchivePolicy(90); + + /// The windows the data settings offer, shortest first; -1 is forever. + static const List choices = [30, 90, 365, -1]; + + /// The setting after this one in [choices], wrapping to the shortest. A + /// stored value that is not a choice (0 = off, or a hand-set count) moves + /// to the first longer choice. + SubstrateArchivePolicy get next { + int rank(int d) => d < 0 ? 1 << 30 : d; + for (final d in choices) { + if (rank(d) > rank(keepDays)) return SubstrateArchivePolicy(d); + } + return SubstrateArchivePolicy(choices.first); + } + + /// Whether switching to [other] deletes archived days this one keeps. + bool keepsLongerThan(SubstrateArchivePolicy other) { + int rank(int d) => d < 0 ? 1 << 30 : d; + return other.enabled && rank(other.keepDays) < rank(keepDays); + } /// 0 = off (and purged), -1 = forever, otherwise days behind the data edge. final int keepDays; @@ -161,19 +185,85 @@ abstract final class SubstrateArchive { return out; } + /// The window a database that predates the 90-day default was getting: a + /// year. Such a database has no stored setting and already holds history — + /// archived buckets, or band readings a later prune would archive — that + /// the new default would delete with no confirmation and no way back. Null + /// when it does not predate it (a fresh install, or a setting was chosen). + static Future legacyWindow( + DatabaseExecutor ex) async { + Future any(String sql) async { + try { + return (await ex.rawQuery(sql)).isNotEmpty; + } on DatabaseException catch (e) { + if (e.isNoSuchTableError()) return false; + rethrow; + } + } + + // A stored setting, or a database that has already been opened by a + // build with the new default (it carries the marker), is not legacy: a + // backup of one with no setting was getting the 90 days. + if (await any('SELECT 1 FROM compute_freshness ' + "WHERE key IN ('$policyKey', '$_defaultCheckedKey') LIMIT 1")) { + return null; + } + final hasHistory = await any('SELECT 1 FROM $table LIMIT 1') || + await any('SELECT 1 FROM decoded_onehz LIMIT 1') || + // Beat-only records (no 1 Hz row) are history too. + await any('SELECT 1 FROM decoded_rr LIMIT 1'); + return hasHistory ? const SubstrateArchivePolicy(365) : null; + } + + /// Once per install, on the first open of a build with the 90-day default: + /// a database that predates it keeps its year ([legacyWindow]); the user can + /// still shorten it from data settings. A fresh one gets the new default. + static const String _defaultCheckedKey = + 'substrate_archive_policy_default_checked'; + + static Future keepLegacyWindowOnce(Database db) async { + const marker = _defaultCheckedKey; + final done = await db.query('compute_freshness', + columns: ['key'], where: 'key = ?', whereArgs: [marker], limit: 1); + if (done.isNotEmpty) return; + await db.transaction((txn) async { + final legacy = await legacyWindow(txn); + final now = DateTime.now().millisecondsSinceEpoch; + if (legacy != null) { + await txn.insert('compute_freshness', { + 'key': policyKey, + 'payload_json': legacy.toJson(), + 'updated_at': now, + }); + } + await txn.insert('compute_freshness', + {'key': marker, 'payload_json': '{}', 'updated_at': now}, + conflictAlgorithm: ConflictAlgorithm.replace); + }); + } + /// The policy as stored, on [ex] — inside a transaction when the caller /// needs it to agree with what that transaction writes. + /// + /// With no stored setting, a year until [keepLegacyWindowOnce] has run on + /// this database: if that check failed before it finished, housekeeping + /// reading the 90-day default in the meantime would evict exactly what the + /// check is there to keep. The default applies once the check has run. static Future readPolicy(DatabaseExecutor ex) async { final rows = await ex.query( 'compute_freshness', - columns: ['payload_json'], - where: 'key = ?', - whereArgs: [policyKey], - limit: 1, - ); - return SubstrateArchivePolicy.fromJson( - rows.isEmpty ? null : rows.first['payload_json'] as String?, + columns: ['key', 'payload_json'], + where: 'key IN (?, ?)', + whereArgs: [policyKey, _defaultCheckedKey], ); + for (final r in rows) { + if (r['key'] == policyKey) { + return SubstrateArchivePolicy.fromJson(r['payload_json'] as String?); + } + } + return rows.isEmpty + ? const SubstrateArchivePolicy(365) + : SubstrateArchivePolicy.defaults; } /// Store [policy]; switching to Off purges the archive IN THE SAME @@ -194,7 +284,8 @@ abstract final class SubstrateArchive { /// keep. /// /// Never adopted over local history. A local policy row always wins, and so - /// does a local archive under the implicit default: that history was kept + /// does a local archive or any live decoded reading under the implicit + /// default: that history was kept /// on this phone, and importing some other backup's "Off" must not be what /// deletes it (a restore only ever ADDS history). Checked and written in one /// transaction, so an archive cannot appear between the two. @@ -212,10 +303,19 @@ abstract final class SubstrateArchive { if (e.isNoSuchTableError()) return; rethrow; } - if (theirs.isEmpty) return; - final adopted = SubstrateArchivePolicy.fromJson( - theirs.first['payload_json'] as String?, - ); + // A backup from before the setting existed has no row; its archive was + // being kept for a year, and restoring it under the 90-day default would + // drop the older part of what it brings. + final SubstrateArchivePolicy adopted; + if (theirs.isNotEmpty) { + adopted = SubstrateArchivePolicy.fromJson( + theirs.first['payload_json'] as String?, + ); + } else { + final legacy = await legacyWindow(src); + if (legacy == null) return; + adopted = legacy; + } await db.transaction((txn) async { final ours = await txn.query( 'compute_freshness', @@ -227,6 +327,14 @@ abstract final class SubstrateArchive { if (ours.isNotEmpty) return; final anyLocal = await txn.query(table, columns: ['utc_day'], limit: 1); if (anyLocal.isNotEmpty) return; + // Live readings are local history too, the part the next prune moves + // into the archive: under an adopted "Off" that prune deletes them + // without archiving, and a shorter window drops the older ones. + for (final live in const ['decoded_onehz', 'decoded_rr']) { + if ((await txn.rawQuery('SELECT 1 FROM $live LIMIT 1')).isNotEmpty) { + return; + } + } await txn.insert('compute_freshness', { 'key': policyKey, 'payload_json': adopted.toJson(), diff --git a/lib/l10n/app_de.arb b/lib/l10n/app_de.arb index e8ad5b170..8043da17e 100644 --- a/lib/l10n/app_de.arb +++ b/lib/l10n/app_de.arb @@ -2909,5 +2909,72 @@ "notifBatteryForecastReserveBody": "Bei {rate} %/h ist der Akku gegen {time} leer, kurz nach deiner üblichen Aufstehzeit – beim Aufwachen bleiben etwa {left} %. Lade ihn jetzt, damit die Nacht aufgezeichnet wird.", "notifBatteryForecastAfterWakeBody": "Bei {rate} %/h ist der Akku gegen {time} leer, nach deiner üblichen Aufstehzeit.", "notifWindDownBody": "Deine Schlafenszeit ist gegen {time}. Fang an, runterzufahren.", - "notifWorkoutIdleBody": "Seit {minutes} Minuten wurde nichts über Ruheniveau aufgezeichnet. Wenn die Einheit vorbei ist, beende sie im Tab „Training“." + "notifWorkoutIdleBody": "Seit {minutes} Minuten wurde nichts über Ruheniveau aufgezeichnet. Wenn die Einheit vorbei ist, beende sie im Tab „Training“.", + "dataStorageGroup": "Speicher", + "dataKeepHistory": "Rohmesswerte behalten", + "dataKeepHistorySub": "Eine komprimierte Kopie der sekündlichen Messwerte jedes Tages, die bleibt, nachdem die App sie gelöscht hat. Ältere Tage behalten ihre Werte, können aber nach einer verbesserten Auswertung nicht neu berechnet werden", + "dataKeepDays": "{days} Tage", + "@dataKeepDays": { + "placeholders": { + "days": { + "type": "int" + } + } + }, + "dataKeepYear": "1 Jahr", + "dataKeepForever": "Für immer", + "dataKeepShorterTitle": "Rohmesswerte für {window} behalten?", + "@dataKeepShorterTitle": { + "placeholders": { + "window": { + "type": "String" + } + } + }, + "dataKeepShorterBody": "Messwerte, die mehr als etwa {window} vor deinem letzten Messwert liegen, werden jetzt gelöscht. Diese Tage behalten ihre Werte, können nach künftigen Updates aber nicht neu berechnet werden.", + "@dataKeepShorterBody": { + "placeholders": { + "window": { + "type": "String" + } + } + }, + "dataKeepShorterConfirm": "Ältere Messwerte löschen", + "dataKeepShorterCancel": "Behalten", + "dataCompact": "Datenbank verkleinern", + "dataCompactSub": "Löscht, was die Einstellungen oben nicht mehr behalten, und gibt den freien Platz an das Telefon zurück. Braucht etwa doppelt so viel freien Speicher, wie die Datenbank groß ist", + "dataCompactPruning": "Alte Daten werden gelöscht…", + "dataCompactVacuuming": "Wird verkleinert…", + "dataCompactFreed": "{freed} MB freigegeben. Die Datenbank ist jetzt {size} MB groß.", + "@dataCompactFreed": { + "placeholders": { + "freed": { + "type": "int" + }, + "size": { + "type": "int" + } + } + }, + "dataCompactNoSpace": "Alte Daten wurden gelöscht, aber zum Verkleinern sind {need} MB frei nötig und das Telefon hat {free} MB.", + "@dataCompactNoSpace": { + "placeholders": { + "need": { + "type": "int" + }, + "free": { + "type": "int" + } + } + }, + "dataCompactUnknownSpace": "Alte Daten wurden gelöscht, aber der freie Speicher des Telefons ließ sich nicht lesen, daher wurde die Datei nicht verkleinert.", + "dataCompactBusy": "Eine Synchronisierung oder eine laufende Sitzung ist aktiv. Versuche es danach noch einmal.", + "dataSizeMb": "{mb} MB", + "@dataSizeMb": { + "placeholders": { + "mb": { + "type": "int" + } + } + } } diff --git a/lib/l10n/app_en.arb b/lib/l10n/app_en.arb index 44dec25da..724f9d181 100644 --- a/lib/l10n/app_en.arb +++ b/lib/l10n/app_en.arb @@ -16203,5 +16203,117 @@ "type": "String" } } + }, + "dataStorageGroup": "Storage", + "@dataStorageGroup": { + "description": "Data screen group title: storage settings and compaction." + }, + "dataKeepHistory": "Keep raw readings", + "@dataKeepHistory": { + "description": "Row title: how long the compressed raw-reading archive is kept." + }, + "dataKeepHistorySub": "A compressed copy of each day's second-by-second readings, kept after the app clears them. A day older than this keeps its scores but can't be re-scored when scoring improves", + "@dataKeepHistorySub": { + "description": "Row sub for the archive window setting." + }, + "dataKeepDays": "{days} days", + "@dataKeepDays": { + "description": "Archive window value: a number of days.", + "placeholders": { + "days": { + "type": "int" + } + } + }, + "dataKeepYear": "1 year", + "@dataKeepYear": { + "description": "Archive window value: one year." + }, + "dataKeepForever": "Forever", + "@dataKeepForever": { + "description": "Archive window value: never delete." + }, + "dataKeepShorterTitle": "Keep {window} of raw readings?", + "@dataKeepShorterTitle": { + "description": "Confirm title when the archive window is shortened.", + "placeholders": { + "window": { + "type": "String" + } + } + }, + "dataKeepShorterBody": "Readings more than about {window} before your latest one are deleted now. Those days keep their scores, but can't be re-scored after future updates.", + "@dataKeepShorterBody": { + "description": "Confirm body when the archive window is shortened.", + "placeholders": { + "window": { + "type": "String" + } + } + }, + "dataKeepShorterConfirm": "Delete older readings", + "@dataKeepShorterConfirm": { + "description": "Confirm button: shorten the window and delete older archive." + }, + "dataKeepShorterCancel": "Keep them", + "@dataKeepShorterCancel": { + "description": "Cancel button: keep the current window." + }, + "dataCompact": "Compact database", + "@dataCompact": { + "description": "Row title: compact the database file." + }, + "dataCompactSub": "Clears what the settings above no longer keep, then gives the unused space back to the phone. Needs free space of about twice the database's size", + "@dataCompactSub": { + "description": "Row sub for Compact database." + }, + "dataCompactPruning": "Clearing old data…", + "@dataCompactPruning": { + "description": "Progress line while old data is cleared before compaction." + }, + "dataCompactVacuuming": "Compacting…", + "@dataCompactVacuuming": { + "description": "Progress line while the database file is rewritten." + }, + "dataCompactFreed": "Freed {freed} MB. The database is now {size} MB.", + "@dataCompactFreed": { + "description": "Compaction result: megabytes freed and the new size.", + "placeholders": { + "freed": { + "type": "int" + }, + "size": { + "type": "int" + } + } + }, + "dataCompactNoSpace": "Old data was cleared, but compacting needs {need} MB free and the phone has {free} MB.", + "@dataCompactNoSpace": { + "description": "Compaction refused: not enough free storage.", + "placeholders": { + "need": { + "type": "int" + }, + "free": { + "type": "int" + } + } + }, + "dataCompactUnknownSpace": "Old data was cleared, but the phone's free space couldn't be read, so the file was not compacted.", + "@dataCompactUnknownSpace": { + "description": "Compaction refused: free storage could not be read." + }, + "dataCompactBusy": "A sync or a live session is running. Try again once it's done.", + "@dataCompactBusy": { + "description": "Compaction refused: a sync or session is running." + }, + "dataSizeMb": "{mb} MB", + "@dataSizeMb": { + "description": "A size in megabytes, shown as the row value.", + "placeholders": { + "mb": { + "type": "int" + } + } } } diff --git a/lib/l10n/app_es.arb b/lib/l10n/app_es.arb index 4d2118ed2..e99aba10e 100644 --- a/lib/l10n/app_es.arb +++ b/lib/l10n/app_es.arb @@ -2821,5 +2821,72 @@ "notifBatteryForecastReserveBody": "A {rate} %/h se agota hacia las {time}, poco después de tu hora habitual de despertar: te quedará alrededor del {left} % al levantarte. Cárgala ahora para no perder el sueño de esta noche.", "notifBatteryForecastAfterWakeBody": "A {rate} %/h se agota hacia las {time}, después de tu hora habitual de despertar.", "notifWindDownBody": "Te sueles acostar hacia las {time}. Empieza a bajar el ritmo.", - "notifWorkoutIdleBody": "No se ha registrado nada por encima del reposo en {minutes} minutos. Si la sesión terminó, finalízala desde la pestaña Entrenamiento." + "notifWorkoutIdleBody": "No se ha registrado nada por encima del reposo en {minutes} minutos. Si la sesión terminó, finalízala desde la pestaña Entrenamiento.", + "dataStorageGroup": "Almacenamiento", + "dataKeepHistory": "Conservar lecturas sin procesar", + "dataKeepHistorySub": "Una copia comprimida de las lecturas segundo a segundo de cada día, que se conserva después de que la app las borre. Los días más antiguos mantienen sus puntuaciones, pero no se pueden recalcular cuando mejore el cálculo", + "dataKeepDays": "{days} días", + "@dataKeepDays": { + "placeholders": { + "days": { + "type": "int" + } + } + }, + "dataKeepYear": "1 año", + "dataKeepForever": "Siempre", + "dataKeepShorterTitle": "¿Conservar {window} de lecturas sin procesar?", + "@dataKeepShorterTitle": { + "placeholders": { + "window": { + "type": "String" + } + } + }, + "dataKeepShorterBody": "Las lecturas de más de unos {window} antes de la última se borran ahora. Esos días mantienen sus puntuaciones, pero no se podrán recalcular tras futuras actualizaciones.", + "@dataKeepShorterBody": { + "placeholders": { + "window": { + "type": "String" + } + } + }, + "dataKeepShorterConfirm": "Borrar lecturas antiguas", + "dataKeepShorterCancel": "Conservarlas", + "dataCompact": "Compactar la base de datos", + "dataCompactSub": "Borra lo que los ajustes de arriba ya no conservan y devuelve el espacio libre al teléfono. Necesita un espacio libre de unas dos veces el tamaño de la base de datos", + "dataCompactPruning": "Borrando datos antiguos…", + "dataCompactVacuuming": "Compactando…", + "dataCompactFreed": "Se liberaron {freed} MB. La base de datos ocupa ahora {size} MB.", + "@dataCompactFreed": { + "placeholders": { + "freed": { + "type": "int" + }, + "size": { + "type": "int" + } + } + }, + "dataCompactNoSpace": "Se borraron los datos antiguos, pero compactar necesita {need} MB libres y el teléfono tiene {free} MB.", + "@dataCompactNoSpace": { + "placeholders": { + "need": { + "type": "int" + }, + "free": { + "type": "int" + } + } + }, + "dataCompactUnknownSpace": "Se borraron los datos antiguos, pero no se pudo leer el espacio libre del teléfono, así que el archivo no se compactó.", + "dataCompactBusy": "Hay una sincronización o una sesión en curso. Inténtalo de nuevo cuando termine.", + "dataSizeMb": "{mb} MB", + "@dataSizeMb": { + "placeholders": { + "mb": { + "type": "int" + } + } + } } diff --git a/lib/l10n/app_fr.arb b/lib/l10n/app_fr.arb index d30c7c634..58840a3d3 100644 --- a/lib/l10n/app_fr.arb +++ b/lib/l10n/app_fr.arb @@ -2792,5 +2792,72 @@ "notifBatteryForecastReserveBody": "À {rate} %/h, il sera vide vers {time}, juste après votre heure de réveil habituelle : il restera environ {left} % au lever. Rechargez-le maintenant pour garder la nuit.", "notifBatteryForecastAfterWakeBody": "À {rate} %/h, il sera vide vers {time}, après votre heure de réveil habituelle.", "notifWindDownBody": "Vous vous couchez vers {time}. Commencez à ralentir.", - "notifWorkoutIdleBody": "Rien au-dessus du repos n'a été enregistré depuis {minutes} minutes. Si la séance est finie, terminez-la depuis l'onglet Entraînement." + "notifWorkoutIdleBody": "Rien au-dessus du repos n'a été enregistré depuis {minutes} minutes. Si la séance est finie, terminez-la depuis l'onglet Entraînement.", + "dataStorageGroup": "Stockage", + "dataKeepHistory": "Conserver les mesures brutes", + "dataKeepHistorySub": "Une copie compressée des mesures seconde par seconde de chaque jour, conservée après que l'app les a effacées. Un jour plus ancien garde ses scores mais ne peut pas être recalculé quand le calcul s'améliore", + "dataKeepDays": "{days} jours", + "@dataKeepDays": { + "placeholders": { + "days": { + "type": "int" + } + } + }, + "dataKeepYear": "1 an", + "dataKeepForever": "Toujours", + "dataKeepShorterTitle": "Conserver {window} de mesures brutes ?", + "@dataKeepShorterTitle": { + "placeholders": { + "window": { + "type": "String" + } + } + }, + "dataKeepShorterBody": "Les mesures de plus d'environ {window} avant la dernière sont supprimées maintenant. Ces jours gardent leurs scores, mais ne pourront pas être recalculés après les prochaines mises à jour.", + "@dataKeepShorterBody": { + "placeholders": { + "window": { + "type": "String" + } + } + }, + "dataKeepShorterConfirm": "Supprimer les anciennes mesures", + "dataKeepShorterCancel": "Les garder", + "dataCompact": "Compacter la base de données", + "dataCompactSub": "Efface ce que les réglages ci-dessus ne gardent plus, puis rend l'espace inutilisé au téléphone. Nécessite un espace libre d'environ deux fois la taille de la base", + "dataCompactPruning": "Effacement des anciennes données…", + "dataCompactVacuuming": "Compactage…", + "dataCompactFreed": "{freed} Mo libérés. La base fait maintenant {size} Mo.", + "@dataCompactFreed": { + "placeholders": { + "freed": { + "type": "int" + }, + "size": { + "type": "int" + } + } + }, + "dataCompactNoSpace": "Les anciennes données ont été effacées, mais le compactage nécessite {need} Mo libres et le téléphone en a {free}.", + "@dataCompactNoSpace": { + "placeholders": { + "need": { + "type": "int" + }, + "free": { + "type": "int" + } + } + }, + "dataCompactUnknownSpace": "Les anciennes données ont été effacées, mais l'espace libre du téléphone n'a pas pu être lu, donc le fichier n'a pas été compacté.", + "dataCompactBusy": "Une synchronisation ou une session est en cours. Réessayez une fois terminée.", + "dataSizeMb": "{mb} Mo", + "@dataSizeMb": { + "placeholders": { + "mb": { + "type": "int" + } + } + } } diff --git a/lib/l10n/app_hi.arb b/lib/l10n/app_hi.arb index 89f1e08a4..027cccad3 100644 --- a/lib/l10n/app_hi.arb +++ b/lib/l10n/app_hi.arb @@ -2792,5 +2792,72 @@ "notifBatteryForecastReserveBody": "{rate}%/घंटा की दर से यह लगभग {time} तक खत्म हो जाएगी, आपके सामान्य जागने के समय के ठीक बाद — उठते समय लगभग {left}% बचेगी। आज रात की नींद बचाने के लिए अभी चार्ज करें।", "notifBatteryForecastAfterWakeBody": "{rate}%/घंटा की दर से यह लगभग {time} तक खत्म हो जाएगी, आपके सामान्य जागने के समय के बाद।", "notifWindDownBody": "आपके सोने का समय लगभग {time} है। धीमे होना शुरू करें।", - "notifWorkoutIdleBody": "{minutes} मिनट से आराम से ऊपर की कोई मेहनत दर्ज नहीं हुई है। अगर सेशन खत्म हो गया है, तो उसे वर्कआउट टैब से खत्म करें।" + "notifWorkoutIdleBody": "{minutes} मिनट से आराम से ऊपर की कोई मेहनत दर्ज नहीं हुई है। अगर सेशन खत्म हो गया है, तो उसे वर्कआउट टैब से खत्म करें।", + "dataStorageGroup": "स्टोरेज", + "dataKeepHistory": "कच्ची रीडिंग रखें", + "dataKeepHistorySub": "हर दिन की सेकंड-दर-सेकंड रीडिंग की एक संपीड़ित कॉपी, जो ऐप के उन्हें हटाने के बाद भी रहती है। इससे पुराने दिन अपने स्कोर रखते हैं, पर स्कोरिंग बेहतर होने पर दोबारा नहीं आंके जा सकते", + "dataKeepDays": "{days} दिन", + "@dataKeepDays": { + "placeholders": { + "days": { + "type": "int" + } + } + }, + "dataKeepYear": "1 साल", + "dataKeepForever": "हमेशा", + "dataKeepShorterTitle": "{window} की कच्ची रीडिंग रखें?", + "@dataKeepShorterTitle": { + "placeholders": { + "window": { + "type": "String" + } + } + }, + "dataKeepShorterBody": "आपकी आख़िरी रीडिंग से लगभग {window} से ज़्यादा पहले की रीडिंग अभी हटा दी जाएंगी। उन दिनों के स्कोर बने रहेंगे, पर आगे के अपडेट के बाद उन्हें दोबारा नहीं आंका जा सकेगा।", + "@dataKeepShorterBody": { + "placeholders": { + "window": { + "type": "String" + } + } + }, + "dataKeepShorterConfirm": "पुरानी रीडिंग हटाएं", + "dataKeepShorterCancel": "उन्हें रखें", + "dataCompact": "डेटाबेस छोटा करें", + "dataCompactSub": "ऊपर की सेटिंग जो अब नहीं रखतीं उसे हटाता है, फिर खाली जगह फ़ोन को लौटाता है। डेटाबेस के आकार से लगभग दोगुनी खाली जगह चाहिए", + "dataCompactPruning": "पुराना डेटा हटाया जा रहा है…", + "dataCompactVacuuming": "छोटा किया जा रहा है…", + "dataCompactFreed": "{freed} MB खाली हुआ। डेटाबेस अब {size} MB है।", + "@dataCompactFreed": { + "placeholders": { + "freed": { + "type": "int" + }, + "size": { + "type": "int" + } + } + }, + "dataCompactNoSpace": "पुराना डेटा हटा दिया गया, पर छोटा करने के लिए {need} MB खाली जगह चाहिए और फ़ोन में {free} MB है।", + "@dataCompactNoSpace": { + "placeholders": { + "need": { + "type": "int" + }, + "free": { + "type": "int" + } + } + }, + "dataCompactUnknownSpace": "पुराना डेटा हटा दिया गया, पर फ़ोन की खाली जगह पढ़ी नहीं जा सकी, इसलिए फ़ाइल छोटी नहीं की गई।", + "dataCompactBusy": "सिंक या कोई सत्र चल रहा है। उसके खत्म होने पर दोबारा कोशिश करें।", + "dataSizeMb": "{mb} MB", + "@dataSizeMb": { + "placeholders": { + "mb": { + "type": "int" + } + } + } } diff --git a/lib/l10n/app_ru.arb b/lib/l10n/app_ru.arb index 0bb1922e0..c1d4b9fe6 100644 --- a/lib/l10n/app_ru.arb +++ b/lib/l10n/app_ru.arb @@ -7833,5 +7833,72 @@ "type": "String" } } + }, + "dataStorageGroup": "Хранилище", + "dataKeepHistory": "Хранить исходные показания", + "dataKeepHistorySub": "Сжатая копия посекундных показаний за каждый день, которая остаётся после того, как приложение их удалит. Более старые дни сохраняют свои оценки, но их нельзя пересчитать после улучшения расчётов", + "dataKeepDays": "{days} дн.", + "@dataKeepDays": { + "placeholders": { + "days": { + "type": "int" + } + } + }, + "dataKeepYear": "1 год", + "dataKeepForever": "Всегда", + "dataKeepShorterTitle": "Хранить исходные показания за {window}?", + "@dataKeepShorterTitle": { + "placeholders": { + "window": { + "type": "String" + } + } + }, + "dataKeepShorterBody": "Показания, снятые более чем примерно за {window} до последнего, будут удалены сейчас. Эти дни сохранят свои оценки, но их нельзя будет пересчитать после будущих обновлений.", + "@dataKeepShorterBody": { + "placeholders": { + "window": { + "type": "String" + } + } + }, + "dataKeepShorterConfirm": "Удалить старые показания", + "dataKeepShorterCancel": "Оставить", + "dataCompact": "Сжать базу данных", + "dataCompactSub": "Удаляет то, что настройки выше больше не хранят, и возвращает свободное место телефону. Нужно свободное место примерно вдвое больше размера базы", + "dataCompactPruning": "Удаление старых данных…", + "dataCompactVacuuming": "Сжатие…", + "dataCompactFreed": "Освобождено {freed} МБ. Теперь база занимает {size} МБ.", + "@dataCompactFreed": { + "placeholders": { + "freed": { + "type": "int" + }, + "size": { + "type": "int" + } + } + }, + "dataCompactNoSpace": "Старые данные удалены, но для сжатия нужно {need} МБ свободного места, а на телефоне {free} МБ.", + "@dataCompactNoSpace": { + "placeholders": { + "need": { + "type": "int" + }, + "free": { + "type": "int" + } + } + }, + "dataCompactUnknownSpace": "Старые данные удалены, но не удалось узнать свободное место на телефоне, поэтому файл не сжат.", + "dataCompactBusy": "Идёт синхронизация или активная сессия. Повторите, когда она закончится.", + "dataSizeMb": "{mb} МБ", + "@dataSizeMb": { + "placeholders": { + "mb": { + "type": "int" + } + } } } diff --git a/lib/l10n/app_zh.arb b/lib/l10n/app_zh.arb index 858d2a12d..786598d17 100644 --- a/lib/l10n/app_zh.arb +++ b/lib/l10n/app_zh.arb @@ -2792,5 +2792,72 @@ "notifBatteryForecastReserveBody": "按每小时 {rate}% 的耗电速度，电量将在 {time} 左右耗尽，略晚于你平时起床的时间，起床时约剩 {left}%。现在充电，才能记录今晚的睡眠。", "notifBatteryForecastAfterWakeBody": "按每小时 {rate}% 的耗电速度，电量将在 {time} 左右耗尽，晚于你平时起床的时间。", "notifWindDownBody": "你通常在 {time} 左右就寝。开始放慢节奏吧。", - "notifWorkoutIdleBody": "已有 {minutes} 分钟没有记录到高于静息水平的运动。如果训练已结束，请在“锻炼”标签页中结束它。" + "notifWorkoutIdleBody": "已有 {minutes} 分钟没有记录到高于静息水平的运动。如果训练已结束，请在“锻炼”标签页中结束它。", + "dataStorageGroup": "存储", + "dataKeepHistory": "保留原始读数", + "dataKeepHistorySub": "每天逐秒读数的压缩副本，在应用清除原始数据后仍会保留。超过此期限的日子保留其评分，但在评分改进后无法重新计算", + "dataKeepDays": "{days} 天", + "@dataKeepDays": { + "placeholders": { + "days": { + "type": "int" + } + } + }, + "dataKeepYear": "1 年", + "dataKeepForever": "永久", + "dataKeepShorterTitle": "保留 {window} 的原始读数？", + "@dataKeepShorterTitle": { + "placeholders": { + "window": { + "type": "String" + } + } + }, + "dataKeepShorterBody": "比最新读数早约 {window} 以上的读数将立即删除。这些日子会保留评分，但在以后的更新后无法重新计算。", + "@dataKeepShorterBody": { + "placeholders": { + "window": { + "type": "String" + } + } + }, + "dataKeepShorterConfirm": "删除较早的读数", + "dataKeepShorterCancel": "保留", + "dataCompact": "压缩数据库", + "dataCompactSub": "清除上面设置不再保留的数据，然后把未使用的空间还给手机。需要约为数据库两倍大小的可用空间", + "dataCompactPruning": "正在清除旧数据…", + "dataCompactVacuuming": "正在压缩…", + "dataCompactFreed": "已释放 {freed} MB。数据库现在为 {size} MB。", + "@dataCompactFreed": { + "placeholders": { + "freed": { + "type": "int" + }, + "size": { + "type": "int" + } + } + }, + "dataCompactNoSpace": "旧数据已清除，但压缩需要 {need} MB 可用空间，手机只有 {free} MB。", + "@dataCompactNoSpace": { + "placeholders": { + "need": { + "type": "int" + }, + "free": { + "type": "int" + } + } + }, + "dataCompactUnknownSpace": "旧数据已清除，但无法读取手机的可用空间，因此未压缩文件。", + "dataCompactBusy": "正在同步或有进行中的会话。完成后再试。", + "dataSizeMb": "{mb} MB", + "@dataSizeMb": { + "placeholders": { + "mb": { + "type": "int" + } + } + } } diff --git a/lib/state/app_state.dart b/lib/state/app_state.dart index d758d9790..7398bec59 100644 --- a/lib/state/app_state.dart +++ b/lib/state/app_state.dart @@ -93,6 +93,8 @@ import 'prefs.dart'; import '../ble/adapters/signals.dart' show InputSignal; import '../ui2/profile/devices.dart' show liveSources, rankSources; import '../data/db.dart'; +import '../data/disk_space.dart' show freeDiskBytes; +import '../data/substrate_archive.dart' show SubstrateArchivePolicy; import '../data/export_retention.dart' show purgeSharedHealthCopies; import '../ecg/ble_ecg_transport.dart'; import '../ecg/ecg_controller.dart'; @@ -2210,7 +2212,14 @@ class AppState extends ChangeNotifier { if (_background || busy || _liveSessionActive) return; _vacuumedThisLaunch = true; try { - final freed = await LocalDb.vacuumIfBloated(); + // Under the derive latch, skipped if a pass holds it: a VACUUM can + // renumber the archive's rowids under a replay reading them. + final freed = + await DerivationEngine.ifIdle(-1, () => LocalDb.vacuumIfBloated()); + if (freed < 0) { + _vacuumedThisLaunch = false; // the next heavy derive tries again + return; + } if (freed > 0) { _log('[db] vacuum returned ${freed >> 20} MB to the filesystem'); } @@ -2596,6 +2605,50 @@ class AppState extends ChangeNotifier { } } + /// Set how long pruned raw readings stay in `substrate_archive`. A shorter + /// window evicts the days it no longer covers right away, behind the derive + /// latch so a prune that is archiving a bucket finishes first. + /// + /// The eviction runs on every set, not only on a shortening: it deletes + /// nothing past what a longer window keeps, and an eviction that failed + /// after the setting was saved is retried by choosing the same window again. + Future setSubstrateArchiveWindow(SubstrateArchivePolicy next) async { + await LocalDb.setSubstrateArchivePolicy(next); + final edge = await LocalDb.substrateArchiveEdge(); + if (edge == null) return; + final evicted = await DerivationEngine.whileIdle( + () => LocalDb.evictSubstrateArchive(next, edge)); + if (evicted > 0) _log('[db] archive window shortened: $evicted bucket(s)'); + } + + /// The data settings' "Compact database": prune per the retention and + /// archive policies, VACUUM, truncate the WAL. Null when it declined to + /// start because a sync, an offload or a live session is running — a + /// VACUUM holds the write lock for the whole rewrite, and those are the + /// writers that must not wait on it. A derive in flight is waited for + /// rather than refused (see [DerivationEngine.compactStorage]). + Future compactDatabase({ + void Function(CompactStep step)? onStep, + }) async { + bool writerRunning() => + busy || + syncingNow || + _liveSessionActive || + _deriveScheduler.offloadActive; + if (writerRunning()) return null; + final result = await _derive.compactStorage( + freeDiskBytes: freeDiskBytes, + onStep: onStep, + busy: writerRunning, + ); + _log('[db] compact: ${result.beforeBytes >> 20} MB -> ' + '${result.afterBytes >> 20} MB (vacuumed=${result.vacuumed}, ' + 'deferred=${result.deferred})'); + // Same answer as refusing up front: the prune ran, the VACUUM waits for + // the sync to finish. + return result.deferred ? null : result; + } + /// User-initiated "Re-analyze data": force-derive EVERY day that has raw, /// ignoring the derived cursor, then refresh the UI. Returns the days derived, /// the days that could not be, and the error when the pass itself failed (for diff --git a/lib/state/derive_coordinator.dart b/lib/state/derive_coordinator.dart index 6fda9a642..234939ac9 100644 --- a/lib/state/derive_coordinator.dart +++ b/lib/state/derive_coordinator.dart @@ -180,7 +180,10 @@ class DeriveCoordinator { if (heavy && _healthShareConsent()) { unawaited(HealthUploader.instance.maybeUpload(consented: true)); } - if (heavy) unawaited(_maybeReclaimDiskSpace()); + // Awaited, so the scheduler's next job starts only once it is done: + // the vacuum holds the derive latch, and a job whose run() found it + // held would come back 0 and be marked complete without deriving. + if (heavy) await _maybeReclaimDiskSpace(); } catch (e, st) { _log('[derive] post-drain failed: $e'); // Was silently swallowed before — this is a real pipeline failure diff --git a/lib/ui2/profile/data.dart b/lib/ui2/profile/data.dart index dec608392..0cc36ac09 100644 --- a/lib/ui2/profile/data.dart +++ b/lib/ui2/profile/data.dart @@ -25,8 +25,10 @@ import 'package:share_plus/share_plus.dart'; import '../../data/auto_backup.dart'; import '../../data/csv_export.dart'; import '../../data/day_label.dart' show dayLabelOf; +import '../../compute/derivation_engine.dart' show CompactStep; import '../../data/db.dart'; import '../../data/export_retention.dart'; +import '../../data/substrate_archive.dart' show SubstrateArchivePolicy; import '../../import/backup_crypto.dart'; import '../../l10n/app_localizations.dart'; import '../../state/app_state.dart'; @@ -64,6 +66,36 @@ class _DataScreenState extends State { bool _noteFailed = false; ImportOutcome? _outcome; + /// How long pruned raw readings are archived. Null until read. + SubstrateArchivePolicy? _archivePolicy; + + /// Database file plus WAL, for the Compact row. Null until read. + int? _dbBytes; + + /// The compaction step in progress, shown under the spinner. + String? _progress; + + @override + void initState() { + super.initState(); + _loadStorage(); + } + + Future _loadStorage() async { + try { + final policy = await LocalDb.substrateArchivePolicy(); + final bytes = await LocalDb.databaseBytesOnDisk(); + if (mounted) { + setState(() { + _archivePolicy = policy; + _dbBytes = bytes; + }); + } + } catch (_) { + // The row stays blank; tapping it re-reads. + } + } + void _say(String s, {bool failed = false}) { if (mounted) { setState(() { @@ -224,6 +256,81 @@ class _DataScreenState extends State { ); } + /// Step to the next archive window. A shorter one deletes archived days + /// for good, so it asks first. + Future<_Note> _stepArchiveWindow(AppState app) async { + final l = AppLocalizations.of(context); + final current = _archivePolicy ?? await LocalDb.substrateArchivePolicy(); + final next = current.next; + if (current.keepsLongerThan(next)) { + if (!mounted) return ('', false); + final window = _windowLabel(l, next.keepDays); + final ok = await confirmRemove( + context, + title: l?.dataKeepShorterTitle(window) ?? + 'Keep $window of raw readings?', + body: l?.dataKeepShorterBody(window) ?? + 'Readings more than about $window before your latest one are ' + 'deleted now. Those days keep their scores, but ' + "can't be re-scored after future updates.", + remove: l?.dataKeepShorterConfirm ?? 'Delete older readings', + keep: l?.dataKeepShorterCancel ?? 'Keep them', + ); + if (!ok) return ('', false); + } + await app.setSubstrateArchiveWindow(next); + if (mounted) setState(() => _archivePolicy = next); + return ('', false); + } + + Future<_Note> _compact(AppState app) async { + final l = AppLocalizations.of(context); + try { + final r = await app.compactDatabase(onStep: (step) { + if (!mounted) return; + setState(() => _progress = switch (step) { + CompactStep.prune => + l?.dataCompactPruning ?? 'Clearing old data…', + CompactStep.vacuum => l?.dataCompactVacuuming ?? 'Compacting…', + }); + }); + if (r == null) { + return ( + l?.dataCompactBusy ?? + "A sync or a live session is running. Try again once it's done.", + true + ); + } + int mb(int bytes) => (bytes / (1 << 20)).round(); + if (mounted) setState(() => _dbBytes = r.afterBytes); + if (!r.vacuumed) { + final free = r.freeBytes; + return free == null + ? ( + l?.dataCompactUnknownSpace ?? + "Old data was cleared, but the phone's free space couldn't " + 'be read, so the file was not compacted.', + true + ) + : ( + l?.dataCompactNoSpace(mb(r.neededBytes), mb(free)) ?? + 'Old data was cleared, but compacting needs ' + '${mb(r.neededBytes)} MB free and the phone has ' + '${mb(free)} MB.', + true + ); + } + return ( + l?.dataCompactFreed(mb(r.freedBytes), mb(r.afterBytes)) ?? + 'Freed ${mb(r.freedBytes)} MB. The database is now ' + '${mb(r.afterBytes)} MB.', + false + ); + } finally { + if (mounted) setState(() => _progress = null); + } + } + Future<_Note> _reanalyze(AppState app) async { final l = AppLocalizations.of(context); final ml = MaterialLocalizations.of(context); @@ -463,6 +570,31 @@ class _DataScreenState extends State { onTap: _busy ? null : () => goto(c, const PhoneImport())), ]), const SizedBox(height: S.x5), + settingsGroup(c, l?.dataStorageGroup ?? 'Storage', [ + SetRow(LucideIcons.archive, C.teal, + l?.dataKeepHistory ?? 'Keep raw readings', + sub: l?.dataKeepHistorySub ?? + "A compressed copy of each day's second-by-second " + 'readings, kept after the app clears them. A day ' + "older than this keeps its scores but can't be " + 're-scored when scoring improves', + value: _archivePolicy == null + ? '' + : _windowLabel(l, _archivePolicy!.keepDays), + onTap: _busy ? null : () => _run(() => _stepArchiveWindow(app))), + SetRow(LucideIcons.minimize2, C.blue, + l?.dataCompact ?? 'Compact database', + sub: l?.dataCompactSub ?? + 'Clears what the settings above no longer keep, then ' + 'gives the unused space back to the phone. Needs ' + "free space of about twice the database's size", + value: _dbBytes == null + ? '' + : (l?.dataSizeMb((_dbBytes! / (1 << 20)).round()) ?? + '${(_dbBytes! / (1 << 20)).round()} MB'), + onTap: _busy ? null : () => _run(() => _compact(app))), + ]), + const SizedBox(height: S.x5), settingsGroup(c, l?.dataRebuildGroup ?? 'Rebuild', [ // The engine puts days on hold after a ≥3 h timezone jump // "until Re-analyze data runs" — and nothing in the app ran @@ -482,6 +614,12 @@ class _DataScreenState extends State { if (_busy) ...[ const SizedBox(height: S.x6), Center(child: CircularProgressIndicator(color: p.on(C.blue))), + if (_progress != null) ...[ + const SizedBox(height: S.x3), + Center( + child: Text(_progress!, + style: F.cap.copyWith(color: p.ink2))), + ], ], if (_note != null && _note!.isNotEmpty) ...[ const SizedBox(height: S.x5), @@ -524,6 +662,13 @@ class _DataScreenState extends State { } } +/// The archive window as the row shows it. +String _windowLabel(AppLocalizations? l, int keepDays) => keepDays < 0 + ? (l?.dataKeepForever ?? 'Forever') + : keepDays == 365 + ? (l?.dataKeepYear ?? '1 year') + : (l?.dataKeepDays(keepDays) ?? '$keepDays days'); + /// Off → Daily → Weekly → Off. Three states cycle in a row; a picker for three /// options is a sheet nobody needs. BackupCadence _nextCadence(BackupCadence c) => BackupCadence diff --git a/test/colmi_link_test.dart b/test/colmi_link_test.dart index cfe1df23e..a4b86e388 100644 --- a/test/colmi_link_test.dart +++ b/test/colmi_link_test.dart @@ -123,7 +123,7 @@ void main() { test('every reply is banked verbatim, never re-drivable', () async { await sync(); final db = await LocalDb.instance; - final archive = await db.query('raw_archive'); + final archive = await db.query(LocalDb.rawArchiveView); final reasons = archive.map((a) => a['reason']).toSet(); expect(reasons, containsAll(['colmi_cmd_0x03', 'colmi_cmd_0x15', 'colmi_big_0x27'])); diff --git a/test/compact_database_test.dart b/test/compact_database_test.dart new file mode 100644 index 000000000..edaae622a --- /dev/null +++ b/test/compact_database_test.dart @@ -0,0 +1,117 @@ +// "Compact database": prune per policy, VACUUM, truncate the WAL — only with +// twice the database free, and never alongside a derive pass. + +import 'dart:io'; +import 'dart:typed_data'; + +import 'package:flutter_test/flutter_test.dart'; +import 'package:openstrap_edge/compute/derivation_engine.dart'; +import 'package:openstrap_edge/data/db.dart'; +import 'package:path/path.dart' as p; +import 'package:sqflite_common_ffi/sqflite_ffi.dart'; + +Future _useFreshDb(String name) async { + await LocalDb.close(); + LocalDb.dbName = name; + final dir = await databaseFactory.getDatabasesPath(); + await databaseFactory.deleteDatabase(p.join(dir, name)); +} + +/// Fill a scratch table and delete it, leaving free pages behind. +Future _bloat() async { + final db = await LocalDb.instance; + await db.execute('CREATE TABLE IF NOT EXISTS _bloat (x BLOB)'); + final blob = Uint8List(64 * 1024)..fillRange(0, 64 * 1024, 7); + await db.transaction((txn) async { + for (var i = 0; i < 160; i++) { + await txn.insert('_bloat', {'x': blob}); + } + }); + await db.execute('DROP TABLE _bloat'); +} + +Future _walBytes() async { + final path = p.join(await getDatabasesPath(), LocalDb.dbName); + final f = File('$path-wal'); + return await f.exists() ? await f.length() : 0; +} + +void main() { + setUpAll(() { + sqfliteFfiInit(); + databaseFactory = databaseFactoryFfi; + }); + + tearDown(() async { + DerivationEngine.debugRunning = false; + await LocalDb.close(); + }); + + test('vacuums and truncates the WAL when there is room', () async { + await _useFreshDb('compact_room.db'); + await _bloat(); + final steps = []; + final r = await DerivationEngine().compactStorage( + freeDiskBytes: () async => 1 << 40, + onStep: steps.add, + ); + expect(r.vacuumed, isTrue); + expect(steps, [CompactStep.prune, CompactStep.vacuum]); + expect(r.freedBytes, greaterThan(5 << 20)); + expect(r.afterBytes, await LocalDb.databaseBytesOnDisk()); + expect(await _walBytes(), 0); + expect(await LocalDb.freelistBytes(), 0); + }); + + test('refuses the VACUUM under twice the database free', () async { + await _useFreshDb('compact_tight.db'); + await _bloat(); + final size = await LocalDb.databaseBytesOnDisk(); + final r = await DerivationEngine().compactStorage( + freeDiskBytes: () async => 2 * size - 1, + ); + expect(r.vacuumed, isFalse); + expect(r.neededBytes, greaterThanOrEqualTo(r.freeBytes!)); + expect(await LocalDb.freelistBytes(), greaterThan(0)); + }); + + test('skips the VACUUM when a sync started while the prune ran', () async { + await _useFreshDb('compact_busy.db'); + await _bloat(); + final steps = []; + final r = await DerivationEngine().compactStorage( + freeDiskBytes: () async => 1 << 40, + onStep: steps.add, + busy: () => true, + ); + expect(r.vacuumed, isFalse); + expect(r.deferred, isTrue); + expect(steps, [CompactStep.prune]); + expect(await LocalDb.freelistBytes(), greaterThan(0)); + }); + + test('unknown free space counts as not enough', () async { + await _useFreshDb('compact_unknown.db'); + await _bloat(); + final r = await DerivationEngine().compactStorage( + freeDiskBytes: () async => null, + ); + expect(r.vacuumed, isFalse); + expect(r.freeBytes, isNull); + }); + + test('waits for a derive pass in flight instead of running beside it', + () async { + await _useFreshDb('compact_latch.db'); + DerivationEngine.debugRunning = true; + var done = false; + final f = DerivationEngine() + .compactStorage(freeDiskBytes: () async => 1 << 40) + .then((_) => done = true); + await Future.delayed(const Duration(milliseconds: 600)); + expect(done, isFalse); + DerivationEngine.debugRunning = false; + await f; + expect(done, isTrue); + }); +} diff --git a/test/data_backup_folder_ui_test.dart b/test/data_backup_folder_ui_test.dart index 18f7e6f6e..1e90b3646 100644 --- a/test/data_backup_folder_ui_test.dart +++ b/test/data_backup_folder_ui_test.dart @@ -118,6 +118,9 @@ void main() { await t.tap(find.text('Backup folder')); await t.pumpAndSettle(); expect(app.backupFolder?.name, 'Backups'); + // The result card sits under the last group, past the fold. + await t.scrollUntilVisible(find.textContaining('No folder picker'), 200, + scrollable: find.byType(Scrollable).first); expect(find.textContaining('No folder picker'), findsOneWidget); }, variant: TargetPlatformVariant.only(TargetPlatform.android), diff --git a/test/db_migration_ladder_test.dart b/test/db_migration_ladder_test.dart index 886d662b7..5d6f3019f 100644 --- a/test/db_migration_ladder_test.dart +++ b/test/db_migration_ladder_test.dart @@ -1414,7 +1414,7 @@ void main() { final db = await LocalDb.instance; for (final t in const [ - 'raw_archive', 'band_events', 'events', 'band_battery', + LocalDb.rawArchiveView, 'band_events', 'events', 'band_battery', ]) { final rows = await db.query(t); expect(rows.length, 1, reason: '$t row count must be unchanged'); @@ -1446,7 +1446,7 @@ void main() { final db2 = await LocalDb.instance; expect(LocalDb.lastRebuild, isNull); for (final t in const [ - 'raw_archive', 'band_events', 'events', 'band_battery', + LocalDb.rawArchiveView, 'band_events', 'events', 'band_battery', ]) { expect((await db2.query(t)).length, 1); } @@ -1506,7 +1506,7 @@ void main() { expect(await _openThroughLocalDb(name), LocalDb.schemaVersion); final db = await LocalDb.instance; - Future archiveAs(String deviceId) => db.insert('raw_archive', { + Future archiveAs(String deviceId) => db.insert(LocalDb.rawArchiveView, { 'device_id': deviceId, 'hex': 'deadbeef', 'counter': 1, @@ -1518,10 +1518,10 @@ void main() { await archiveAs(''); await archiveAs('device-b'); - expect(await db.query('raw_archive'), hasLength(2)); + expect(await db.query(LocalDb.rawArchiveView), hasLength(2)); await archiveAs(''); // re-flood - expect(await db.query('raw_archive'), hasLength(2)); + expect(await db.query(LocalDb.rawArchiveView), hasLength(2)); }, ); @@ -1541,7 +1541,7 @@ void main() { '01570c500b010c020c0100000000000000000000000000000000000000000000' '010053748080000000fcaf98c0000000'; for (final deviceId in const ['', 'device-b']) { - await db.insert('raw_archive', { + await db.insert(LocalDb.rawArchiveView, { 'device_id': deviceId, 'hex': hex, 'counter': 21510400, diff --git a/test/db_p0_fixes_test.dart b/test/db_p0_fixes_test.dart index c809d937d..c0efc5f06 100644 --- a/test/db_p0_fixes_test.dart +++ b/test/db_p0_fixes_test.dart @@ -465,7 +465,7 @@ void main() { expect(b3.first['rr_ts_ms'], t3 * 1000); // Both archived frames survived the restore. - final archived = await db.query('raw_archive', orderBy: 'hex ASC'); + final archived = await db.query(LocalDb.rawArchiveView, orderBy: 'hex ASC'); expect([for (final a in archived) a['hex']], ['deadbeef01', 'deadbeef02'], reason: 'raw_archive was missing from the import merge list, so a ' diff --git a/test/db_v42_retention_and_provenance_test.dart b/test/db_v42_retention_and_provenance_test.dart index 20d87ca14..f41ebb5ef 100644 --- a/test/db_v42_retention_and_provenance_test.dart +++ b/test/db_v42_retention_and_provenance_test.dart @@ -376,7 +376,7 @@ void main() { // 600 rows at 1-in-60 ⇒ counters 0, 60, … 540. expect( await _count( - "SELECT COUNT(*) FROM raw_archive WHERE reason = 'undecodable_rec_v20' " + "SELECT COUNT(*) FROM ${LocalDb.rawArchiveView} WHERE reason = 'undecodable_rec_v20' " 'AND captured_at < ?', [cutoffSec * 1000], ), @@ -384,13 +384,13 @@ void main() { ); expect( await _count( - "SELECT COUNT(*) FROM raw_archive WHERE reason = 'undecodable_rec_v25'", + "SELECT COUNT(*) FROM ${LocalDb.rawArchiveView} WHERE reason = 'undecodable_rec_v25'", ), 1, ); expect( await _count( - "SELECT COUNT(*) FROM raw_archive WHERE reason = 'undecodable_rec_v20' " + "SELECT COUNT(*) FROM ${LocalDb.rawArchiveView} WHERE reason = 'undecodable_rec_v20' " 'AND captured_at >= ?', [cutoffSec * 1000], ), @@ -398,7 +398,7 @@ void main() { ); // Idempotent: running it again takes nothing more. await LocalDb.pruneDecodedBeforeRecTs(cutoffSec); - expect(await _count('SELECT COUNT(*) FROM raw_archive'), 12); + expect(await _count('SELECT COUNT(*) FROM ${LocalDb.rawArchiveView}'), 12); }); test('prune cursor only moves forward', () async { @@ -557,15 +557,15 @@ void main() { expect(await _count('SELECT COUNT(*) FROM metric_series_version'), derived); - final before = await _count('SELECT COUNT(*) FROM raw_archive'); + final before = await _count('SELECT COUNT(*) FROM ${LocalDb.rawArchiveView}'); final byReason = await db.rawQuery( - 'SELECT reason, COUNT(*) AS n FROM raw_archive GROUP BY reason', + 'SELECT reason, COUNT(*) AS n FROM ${LocalDb.rawArchiveView} GROUP BY reason', ); // Thin at "now", so the whole export is behind the retention edge. final thinned = await LocalDb.thinRawArchiveBefore( DateTime.now().millisecondsSinceEpoch, ); - final after = await _count('SELECT COUNT(*) FROM raw_archive'); + final after = await _count('SELECT COUNT(*) FROM ${LocalDb.rawArchiveView}'); // ignore: avoid_print print('[v42] ${p.basename(src)} archive $before → $after ' '(-$thinned) by_reason=$byReason'); @@ -575,7 +575,7 @@ void main() { if (r['reason'] == 'undecodable_rec_v20') continue; expect( await _count( - 'SELECT COUNT(*) FROM raw_archive WHERE reason = ?', + 'SELECT COUNT(*) FROM ${LocalDb.rawArchiveView} WHERE reason = ?', [r['reason']], ), r['n'], diff --git a/test/derive_coordinator_test.dart b/test/derive_coordinator_test.dart index 47344d1d7..29b98d6c4 100644 --- a/test/derive_coordinator_test.dart +++ b/test/derive_coordinator_test.dart @@ -31,6 +31,7 @@ void main() { bool Function()? disposed, bool healthSync = false, Future Function()? runHealthExport, + Future Function()? reclaim, }) => DeriveCoordinator( // Every test installs a debug hook for the engine calls, so any real @@ -52,7 +53,7 @@ void main() { healthSyncEnabled: () => healthSync, telemetryConsent: () => false, healthShareConsent: () => false, - maybeReclaimDiskSpace: () async => reclaims++, + maybeReclaimDiskSpace: reclaim ?? () async => reclaims++, ); test('constructing the coordinator and its scheduler never touches derive', @@ -116,6 +117,27 @@ void main() { expect(reclaims, 1); }); + test('a heavy pass ends only after the disk reclaim does', () async { + // The scheduler starts its next job when a pass returns. The reclaim + // holds the derive latch, so a job started under it would derive nothing. + final reclaim = Completer(); + final coordinator = make(reclaim: () => reclaim.future); + addTearDown(coordinator.dispose); + coordinator.debugDeriveRun = (profile, {heavy = false, onDayDone}) async => 0; + coordinator.debugRescanRecent = (_) async => 0; + + var finished = false; + final heavy = + coordinator.afterDrain(heavy: true).then((_) => finished = true); + for (var i = 0; i < 5; i++) { + await Future.delayed(Duration.zero); + } + expect(finished, isFalse); + reclaim.complete(); + await heavy; + expect(finished, isTrue); + }); + test('a rescan that throws is logged and the pass carries on', () async { final coordinator = make(healthSync: true); addTearDown(coordinator.dispose); diff --git a/test/derive_if_idle_test.dart b/test/derive_if_idle_test.dart new file mode 100644 index 000000000..43648a847 --- /dev/null +++ b/test/derive_if_idle_test.dart @@ -0,0 +1,32 @@ +// The automatic VACUUM runs through DerivationEngine.ifIdle: never while a +// derive (and the archive replay inside it) holds the latch, and skipped +// rather than queued when one does. + +import 'package:flutter_test/flutter_test.dart'; +import 'package:openstrap_edge/compute/derivation_engine.dart'; + +void main() { + test('skips while a pass holds the latch, runs once it is free', () async { + expect(await DerivationEngine.acquireForReset(), isTrue); + var ran = false; + try { + final r = await DerivationEngine.ifIdle(-1, () async { + ran = true; + return 1; + }); + expect(r, -1); + expect(ran, isFalse); + } finally { + DerivationEngine.releaseForReset(); + } + expect( + await DerivationEngine.ifIdle(-1, () async { + ran = true; + // The latch is held while the body runs. + expect(await DerivationEngine.ifIdle(-2, () async => 0), -2); + return 1; + }), + 1); + expect(ran, isTrue); + }); +} diff --git a/test/derive_partial_not_cached_test.dart b/test/derive_partial_not_cached_test.dart new file mode 100644 index 000000000..9384a06b0 --- /dev/null +++ b/test/derive_partial_not_cached_test.dart @@ -0,0 +1,66 @@ +// A derive whose second half failed stores a partial row. Two failures in a +// row can store the same partial row, and if that pass then recorded the +// day's input fingerprint, every later pass would skip the day as unchanged +// and the missing detail would never be filled in. Only a complete derive +// may record one. Its own file: the derive passes are heavy, and the +// equivalence test next door already runs ten of them. + +import 'package:flutter_test/flutter_test.dart'; +import 'package:openstrap_edge/compute/derivation_engine.dart'; +import 'package:openstrap_edge/compute/profile.dart'; +import 'package:openstrap_edge/data/db.dart'; +import 'package:path/path.dart' as p; +import 'package:shared_preferences/shared_preferences.dart'; +import 'package:sqflite_common_ffi/sqflite_ffi.dart'; + +import 'support/synthetic_day.dart'; + +const _profile = Profile(ageYears: 32, weightKg: 72, heightCm: 176, sex: 'm'); + +// One day, mid-July, still pending (the data edge is inside it). +final _day = SyntheticDay(DateTime(2026, 7, 13)); + +void main() { + setUpAll(() { + sqfliteFfiInit(); + databaseFactory = databaseFactoryFfi; + }); + + tearDown(() async { + DerivationEngine.debugSecondHalfHook = null; + await LocalDb.close(); + }); + + test('a partial derive never records its fingerprint', () async { + const name = 'derive_partial_not_cached.db'; + SharedPreferences.setMockInitialValues({}); + await LocalDb.close(); + LocalDb.dbName = name; + await databaseFactory + .deleteDatabase(p.join(await databaseFactory.getDatabasesPath(), name)); + for (final (raws, samples) in _day.primaryBatches()) { + await LocalDb.commitSyncBatch(raws, samples, deviceFamily: 'gen4'); + } + + DerivationEngine.debugSecondHalfHook = + (_) async => throw StateError('second half down'); + // The passes that reach the fixed point: the second stores the same + // partial row as the first, the third derives from it unchanged. + for (var i = 0; i < 3; i++) { + final e = DerivationEngine(); + await e.run(_profile, heavy: true); + final derived = await _derivedDays(); + expect(derived, isNotEmpty); + for (final d in derived) { + expect(await LocalDb.deriveFingerprint(d), isNull, + reason: 'pass $i stored a fingerprint for partial day $d'); + } + } + }, timeout: const Timeout(Duration(minutes: 5))); +} + +Future> _derivedDays() async => [ + for (final r in await (await LocalDb.instance) + .query('day_result', columns: ['day_id'], distinct: true)) + r['day_id'] as String, + ]; diff --git a/test/derive_unchanged_skip_test.dart b/test/derive_unchanged_skip_test.dart new file mode 100644 index 000000000..647b20922 --- /dev/null +++ b/test/derive_unchanged_skip_test.dart @@ -0,0 +1,181 @@ +// A heavy pass leaves a pending day alone when nothing it reads has moved +// since a derive of it that changed nothing it reads. That is only allowed if +// it is invisible: the same passes with every day derived must store exactly +// the same results, and an override, new data, a profile change or an +// algorithm bump must still re-derive. + +import 'dart:convert'; + +import 'package:flutter_test/flutter_test.dart'; +import 'package:openstrap_edge/compute/derivation_engine.dart'; +import 'package:openstrap_edge/compute/profile.dart'; +import 'package:openstrap_edge/data/db.dart'; +import 'package:openstrap_edge/data/models.dart' show RawRecord, Sample; +import 'package:path/path.dart' as p; +import 'package:shared_preferences/shared_preferences.dart'; +import 'package:sqflite_common_ffi/sqflite_ffi.dart'; + +import 'support/synthetic_day.dart'; + +const _profile = Profile(ageYears: 32, weightKg: 72, heightCm: 176, sex: 'm'); + +// Mid-July, no DST edge. The newest days stay pending: the data edge is less +// than 48 h past them. +final _days = [ + for (var i = 0; i < 3; i++) SyntheticDay(DateTime(2026, 7, 13 + i)), +]; + +Future _freshDb(String name) async { + SharedPreferences.setMockInitialValues({}); + await LocalDb.close(); + LocalDb.dbName = name; + await databaseFactory + .deleteDatabase(p.join(await databaseFactory.getDatabasesPath(), name)); + var committed = 0; + for (final d in _days) { + for (final (raws, samples) in d.primaryBatches()) { + final keep = [ + for (var j = 0; j < raws.length; j++) + if (raws[j].recTs! > committed) j, + ]; + if (keep.isEmpty) continue; + await LocalDb.commitSyncBatch( + [for (final j in keep) raws[j]], + [for (final j in keep) samples[j]], + deviceFamily: 'gen4', + ); + committed = raws[keep.last].recTs!; + } + } +} + +Future<({int derived, int unchanged})> _pass({Profile profile = _profile}) async { + final e = DerivationEngine(); + final n = await e.run(profile, heavy: true); + return (derived: n, unchanged: e.lastRunUnchangedDays); +} + +/// Every stored output of a derive, write stamps dropped (the columns, and +/// the wall-clock stamps inside the baseline artifacts). +Future _snapshot() async { + final db = await LocalDb.instance; + Future>> rows(String sql) async => [ + for (final r in await db.rawQuery(sql)) + { + for (final e in r.entries) + if (!e.key.endsWith('_at') && e.key != 'updated_ts') + e.key: e.value is String + ? (e.value as String).replaceAll( + RegExp(r'"(updated|built)[A-Za-z_]*":\s*\d+'), '') + : e.value, + }, + ]; + return jsonEncode({ + 'day_result': await rows( + 'SELECT * FROM day_result ORDER BY day_id, algo_version'), + 'metric_series': + await rows('SELECT * FROM metric_series ORDER BY date, key'), + 'metric_series_version': + await rows('SELECT * FROM metric_series_version ORDER BY date'), + 'wake_day_features': await rows( + 'SELECT * FROM wake_day_features ORDER BY day_id, algo_version'), + 'sleep_session_candidates': await rows('SELECT * FROM ' + 'sleep_session_candidates ORDER BY day_id, algo_version'), + 'activity_suggestions': + await rows('SELECT * FROM activity_suggestions ORDER BY id'), + 'sleep_nap': await rows('SELECT * FROM sleep_nap ORDER BY start_ts'), + 'baselines': await rows('SELECT * FROM baselines ORDER BY key'), + }); +} + +void main() { + setUpAll(() { + sqfliteFfiInit(); + databaseFactory = databaseFactoryFfi; + }); + + tearDown(() async { + DerivationEngine.debugDeriveEveryDay = false; + DerivationEngine.debugFingerprintAlgoVersion = null; + await LocalDb.close(); + }); + + test('skipping unchanged days stores exactly what deriving them does', + () async { + const passes = 5; + await _freshDb('unchanged_skip_on.db'); + final log = <({int derived, int unchanged})>[]; + for (var i = 0; i < passes; i++) { + log.add(await _pass()); + } + final withSkip = await _snapshot(); + + await _freshDb('unchanged_skip_off.db'); + DerivationEngine.debugDeriveEveryDay = true; + for (var i = 0; i < passes; i++) { + final r = await _pass(); + expect(r.unchanged, 0); + } + final everyDay = await _snapshot(); + + // The skip actually happened: by the last pass nothing was derived. + expect(log.last.derived, 0, reason: '$log'); + expect(log.last.unchanged, greaterThan(0), reason: '$log'); + expect(withSkip, everyDay); + }, timeout: const Timeout(Duration(minutes: 10))); + + test('an override, new data, a profile change and a version bump re-derive', + () async { + await _freshDb('unchanged_skip_triggers.db'); + Future settle() async { + for (var i = 0; i < 6; i++) { + if ((await _pass()).derived == 0) return; + } + fail('never settled'); + } + + await settle(); + final settled = await _pass(); + expect(settled.derived, 0); + final pending = settled.unchanged; + expect(pending, greaterThan(0)); + + // The user's own sleep window on a pending day. + final d = _days[1]; + await LocalDb.putSleepOverride( + dayId: '2026-07-14', + onsetTs: SyntheticDay.sec(d.sleepOnset) + 600, + offsetTs: SyntheticDay.sec(d.sleepOffset), + source: 'manual', + ); + expect((await _pass()).derived, greaterThanOrEqualTo(1)); + await settle(); + + // A second re-delivered inside a pending day's window (counter-reset + // recovery replaces the row in place, the data edge does not move). + final t = SyntheticDay.sec(DateTime(2026, 7, 14, 12)); + await LocalDb.commitSyncBatch( + [ + RawRecord( + counter: 1, packetType: 0x2F, hex: 'aa', capturedAt: t * 1000, + recTs: t), + ], + [Sample(tsEpoch: t, counter: 1, hr: 140, ax: 0, ay: 0, az: 1)], + deviceFamily: 'gen4', + ); + expect((await _pass()).derived, greaterThanOrEqualTo(1)); + await settle(); + + expect( + (await _pass( + profile: const Profile( + ageYears: 33, weightKg: 72, heightCm: 176, sex: 'm'))) + .derived, + pending, + ); + await settle(); + + DerivationEngine.debugFingerprintAlgoVersion = kAlgoVersion + 1; + expect((await _pass()).derived, pending); + }, timeout: const Timeout(Duration(minutes: 15))); +} diff --git a/test/finalized_day_ids_index_test.dart b/test/finalized_day_ids_index_test.dart new file mode 100644 index 000000000..230a96645 --- /dev/null +++ b/test/finalized_day_ids_index_test.dart @@ -0,0 +1,119 @@ +// `finalizedDayIds` reads an indexed `day_result.imported` flag instead of +// LIKE-scanning every finalized bundle. The answer must be the same set the +// scan gave, for rows written before the column existed (backfill) and after +// (triggers), on every write path. + +import 'package:flutter_test/flutter_test.dart'; +import 'package:path/path.dart' as p; +import 'package:sqflite_common_ffi/sqflite_ffi.dart'; +import 'package:openstrap_edge/data/db.dart'; + +Future _useFreshDb(String name) async { + await LocalDb.close(); + LocalDb.dbName = name; + final dir = await databaseFactory.getDatabasesPath(); + await databaseFactory.deleteDatabase(p.join(dir, name)); +} + +const _algo = 7; + +Future> _likeScan(Database db) async => { + for (final r in await db.query( + 'day_result', + columns: ['day_id'], + where: + 'algo_version = ? AND finalized = 1 ' + "AND payload_json NOT LIKE '%\"imported\":true%'", + whereArgs: [_algo], + )) + r['day_id'] as String, +}; + +Future _indexed(Database db) async => (await db.rawQuery( + "SELECT 1 FROM sqlite_master WHERE type = 'index' " + "AND name = 'idx_day_result_finalized'", +)).isNotEmpty; + +Future _put(String day, String payload, {bool finalized = true}) => + LocalDb.putDayResult( + dayId: day, + algoVersion: _algo, + payloadJson: payload, + windowJson: '{}', + finalized: finalized, + ); + +void main() { + setUpAll(() { + sqfliteFfiInit(); + databaseFactory = databaseFactoryFfi; + }); + + tearDown(() async { + await LocalDb.close(); + }); + + test('same set as the LIKE scan, through the triggers', () async { + await _useFreshDb('finalized_ids_triggers.db'); + await _put('2026-01-01', '{"scalars":{}}'); + await _put('2026-01-02', '{"imported":true,"source":"whoop_export"}'); + await _put('2026-01-03', '{"scalars":{}}', finalized: false); + await _put('2026-01-04', '{"imported":false}'); + final db = await LocalDb.instance; + expect(await LocalDb.finalizedDayIds(_algo), {'2026-01-01', '2026-01-04'}); + expect(await _indexed(db), isTrue); + + // A measured re-derive replacing the import, and an import replacing a + // measured row, both flip the flag. + await _put('2026-01-02', '{"scalars":{"rhr":50}}'); + await _put('2026-01-04', '{"imported":true}'); + // A raw payload rewrite (the shape `_scrubImportedSkinTempZ` uses). + await db.rawUpdate( + "UPDATE day_result SET payload_json = '{\"imported\":true}' " + "WHERE day_id = '2026-01-01'", + ); + final ids = await LocalDb.finalizedDayIds(_algo); + expect(ids, {'2026-01-02'}); + expect(ids, await _likeScan(db)); + }); + + test('backfills rows written before the column existed', () async { + await _useFreshDb('finalized_ids_backfill.db'); + final db = await LocalDb.instance; + // Emulate a pre-column database: no triggers, no index, stale flags. + await db.execute('DROP TRIGGER IF EXISTS trg_day_result_imported_ins'); + await db.execute('DROP TRIGGER IF EXISTS trg_day_result_imported_upd'); + await db.execute('DROP INDEX IF EXISTS idx_day_result_finalized'); + for (var i = 0; i < 450; i++) { + final day = + '2025-${(1 + i ~/ 28).toString().padLeft(2, '0')}-' + '${(1 + i % 28).toString().padLeft(2, '0')}'; + await db.insert('day_result', { + 'day_id': day, + 'algo_version': _algo, + 'payload_json': i % 3 == 0 ? '{"imported":true}' : '{"x":$i}', + 'window_json': '{}', + 'computed_at': 0, + 'finalized': i % 5 == 0 ? 0 : 1, + // Deliberately wrong, so only a real backfill passes. + 'imported': i % 3 == 0 ? 0 : 1, + }); + } + final expected = await _likeScan(db); + expect(expected, isNotEmpty); + // Reopen: the repair pass restores the triggers, the first read backfills. + await LocalDb.close(); + expect(await LocalDb.finalizedDayIds(_algo), expected); + final reopened = await LocalDb.instance; + expect(await _indexed(reopened), isTrue); + final plan = await reopened.rawQuery( + 'EXPLAIN QUERY PLAN SELECT day_id FROM day_result ' + 'WHERE algo_version = ? AND finalized = 1 AND imported = 0', + [_algo], + ); + expect( + plan.map((r) => '${r['detail']}').join(' '), + contains('COVERING INDEX idx_day_result_finalized'), + ); + }); +} diff --git a/test/miband_link_test.dart b/test/miband_link_test.dart index b8bb12233..6fe190c35 100644 --- a/test/miband_link_test.dart +++ b/test/miband_link_test.dart @@ -76,7 +76,7 @@ void main() { nowSeconds: () => _nowSec, ); final db = await LocalDb.instance; - final archive = await db.query('raw_archive', orderBy: 'hex'); + final archive = await db.query(LocalDb.rawArchiveView, orderBy: 'hex'); expect(archive, hasLength(3)); expect( archive.map((a) => a['reason']).toSet(), diff --git a/test/mt12_dyn_accel_test.dart b/test/mt12_dyn_accel_test.dart index 99a65481c..c7390f3f2 100644 --- a/test/mt12_dyn_accel_test.dart +++ b/test/mt12_dyn_accel_test.dart @@ -130,7 +130,8 @@ void main() { for (final e in before.entries) { final after = - (await db.rawQuery('SELECT COUNT(*) AS n FROM ${e.key}')) + (await db.rawQuery('SELECT COUNT(*) AS n FROM ' + '${e.key == 'raw_archive' ? LocalDb.rawArchiveView : e.key}')) .first['n'] as int; // GREATER-OR-EQUAL, not equal: the v44 rung replays archived records // into decoded_onehz, so that table may legitimately GAIN rows. No diff --git a/test/oura_link_test.dart b/test/oura_link_test.dart index cdde013f7..ed0e2fa61 100644 --- a/test/oura_link_test.dart +++ b/test/oura_link_test.dart @@ -98,7 +98,7 @@ Future _run(List>> batches) async { return ReplayBandLinkResult( writes: [for (final w in link.writes) w.$2], onehz: await db.query('decoded_onehz', orderBy: 'ts_ms'), - archive: await db.query('raw_archive', orderBy: 'captured_at, hex'), + archive: await db.query(LocalDb.rawArchiveView, orderBy: 'captured_at, hex'), ); } diff --git a/test/pebble_link_test.dart b/test/pebble_link_test.dart index 7c4eb6d2b..c9b121645 100644 --- a/test/pebble_link_test.dart +++ b/test/pebble_link_test.dart @@ -51,7 +51,7 @@ void main() { nowSeconds: () => ts + 3600, ); final db = await LocalDb.instance; - expect((await db.query('raw_archive')).length, greaterThanOrEqualTo(1)); + expect((await db.query(LocalDb.rawArchiveView)).length, greaterThanOrEqualTo(1)); final hr = await db.query('decoded_onehz'); expect(hr.single['hr'], 61); expect(hr.single['source'], 'pebble'); diff --git a/test/raw_archive_frames_test.dart b/test/raw_archive_frames_test.dart new file mode 100644 index 000000000..8d25ceabe --- /dev/null +++ b/test/raw_archive_frames_test.dart @@ -0,0 +1,376 @@ +// raw_archive stores each undecodable frame once, as bytes, keyed by a hash of +// them. The old table kept it as hex TEXT under PRIMARY KEY (device_id, hex) +// on a rowid table, so the hex was stored twice. This pins: readers still see +// the exact hex they always did, dedup still holds, text that is not +// canonical hex survives verbatim, and an existing archive moves across in +// pages without losing or duplicating a frame. + +import 'dart:convert'; +import 'dart:io'; +import 'dart:math' as math; + +import 'package:flutter_test/flutter_test.dart'; +import 'package:path/path.dart' as p; +import 'package:shared_preferences/shared_preferences.dart'; +import 'package:sqflite_common_ffi/sqflite_ffi.dart'; +import 'package:openstrap_edge/data/db.dart'; +import 'package:openstrap_edge/data/models.dart'; + +const _frameBytes = 2128; // a WHOOP MG v20 frame + +String _frameHex(int seed) { + final r = math.Random(seed); + final b = StringBuffer(); + for (var i = 0; i < _frameBytes; i++) { + b.write(r.nextInt(256).toRadixString(16).padLeft(2, '0')); + } + return b.toString(); +} + +ArchiveRecord _rec(int counter, String hex, {String reason = 'undecodable_rec_v20'}) => + ArchiveRecord( + counter: counter, + hex: hex, + packetType: 0x2F, + capturedAt: 1750000000000 + counter * 1000, + reason: reason, + ); + +Future _path(String name) async => + p.join(await databaseFactory.getDatabasesPath(), name); + +Future _fresh(String name) async { + await LocalDb.close(); + LocalDb.dbName = name; + await databaseFactory.deleteDatabase(await _path(name)); +} + +Future>> _viewRows() async => + (await LocalDb.instance).rawQuery( + 'SELECT device_id, hex, counter, packet_type, rec_ts, captured_at, ' + 'reason FROM ${LocalDb.rawArchiveView} ORDER BY device_id, hex', + ); + +Future _bytes(Database db, List names) async { + final marks = List.filled(names.length, '?').join(','); + final r = await db.rawQuery( + 'SELECT SUM(pgsize) AS b FROM dbstat WHERE name IN ($marks)', + names, + ); + return (r.first['b'] as num?)?.toInt() ?? 0; +} + +/// A database as the previous build left it: user_version 58 and the +/// hex-keyed table, with [rows] in it. +Future _legacyDb(String name, List> rows) async { + await _fresh(name); + final db = await databaseFactory.openDatabase(await _path(name), + options: OpenDatabaseOptions( + version: LocalDb.schemaVersion, + onCreate: (db, _) async { + await db.execute(''' + CREATE TABLE raw_archive ( + device_id TEXT NOT NULL DEFAULT '', + hex TEXT NOT NULL, + counter INTEGER, + packet_type INTEGER NOT NULL, + rec_ts INTEGER, + captured_at INTEGER NOT NULL, + reason TEXT NOT NULL, + PRIMARY KEY (device_id, hex) + ) + '''); + await db.execute('CREATE INDEX idx_raw_archive_captured ' + 'ON raw_archive(captured_at DESC)'); + }, + )); + final batch = db.batch(); + for (final r in rows) { + batch.insert('raw_archive', r); + } + await batch.commit(noResult: true); + await db.close(); +} + +Map _legacyRow(String device, String hex, int counter) => { + 'device_id': device, + 'hex': hex, + 'counter': counter, + 'packet_type': 0x2F, + 'rec_ts': null, + 'captured_at': 1750000000000 + counter, + 'reason': 'undecodable_rec_v20', + }; + +void main() { + setUpAll(() { + TestWidgetsFlutterBinding.ensureInitialized(); + SharedPreferences.setMockInitialValues({}); + sqfliteFfiInit(); + databaseFactory = databaseFactoryFfi; + }); + + tearDown(() async => LocalDb.close()); + + test('a frame is stored once, as bytes, and reads back as its hex', () async { + await _fresh('raw_frames_store.db'); + final a = _frameHex(1), b = _frameHex(2); + await LocalDb.commitSyncBatch(const [], const [], + archives: [_rec(1, a), _rec(2, b), _rec(3, a)]); + await LocalDb.archiveRawRecord(_rec(4, b)); // a re-flood, other path + final rows = await _viewRows(); + expect([for (final r in rows) r['hex']], unorderedEquals([a, b])); + final db = await LocalDb.instance; + final stored = await db.query(LocalDb.rawArchiveTable); + expect(stored, hasLength(2)); + for (final r in stored) { + expect(r['hex'], isNull); + expect((r['frame'] as List).length, _frameBytes); + expect((r['hash'] as List).length, 20); + } + }); + + test('text that is not canonical hex is kept verbatim', () async { + await _fresh('raw_frames_text.db'); + const odd = ['ABCDEF', 'abc', 'not hex at all', 'Deadbeef']; + await LocalDb.commitSyncBatch(const [], const [], + archives: [for (var i = 0; i < odd.length; i++) _rec(i, odd[i])]); + expect([for (final r in await _viewRows()) r['hex']], + unorderedEquals(odd)); + }); + + test('two devices keep their own copy of a byte-identical frame', () async { + await _fresh('raw_frames_devices.db'); + final hex = _frameHex(9); + await LocalDb.archiveRawRecord(_rec(1, hex)); + await LocalDb.archiveRawRecord(_rec(1, hex), deviceId: 'ring-1'); + expect(await _viewRows(), hasLength(2)); + }); + + test('plain inserts and deletes on the old name still work', () async { + await _fresh('raw_frames_compat.db'); + final db = await LocalDb.instance; + await db.insert(LocalDb.rawArchiveView, _legacyRow('', 'beef', 7)); + await db.insert(LocalDb.rawArchiveView, _legacyRow('', 'beef', 7), + conflictAlgorithm: ConflictAlgorithm.ignore); + expect(await _viewRows(), hasLength(1)); + await db.delete(LocalDb.rawArchiveView, where: 'hex = ?', whereArgs: ['beef']); + expect(await _viewRows(), isEmpty); + }); + + test('an existing archive moves across in pages, losing nothing', () async { + final rows = [ + for (var i = 0; i < 1203; i++) + _legacyRow(i % 7 == 0 ? 'ring-1' : '', _frameHex(100 + i), i), + _legacyRow('', 'Not-Hex', 5000), + ]; + await _legacyDb('raw_frames_migrate.db', rows); + final expected = [ + for (final r in rows) + '${r['device_id']}|${r['hex']}|${r['counter']}|${r['captured_at']}', + ]..sort(); + Future> seen() async => [ + for (final r in await _viewRows()) + '${r['device_id']}|${r['hex']}|${r['counter']}|${r['captured_at']}', + ]..sort(); + + // Opening renames the table aside; the view already shows every row. + expect(await seen(), expected); + final db = await LocalDb.instance; + // The table and its key's autoindex, whatever the rename called it. + final legacyIndex = [ + for (final r in await db.rawQuery("SELECT name FROM sqlite_master " + "WHERE type = 'index' AND tbl_name = 'raw_archive_legacy'")) + r['name'] as String, + ]; + expect(legacyIndex, isNotEmpty); + final legacyBytes = + await _bytes(db, ['raw_archive_legacy', ...legacyIndex]); + + // A frame arriving mid-move, already in the legacy table: one copy after. + await LocalDb.archiveRawRecord(_rec(1, rows[1]['hex'] as String)); + + // One page per call, a frame lands between calls, and the pass resumes. + var calls = 0; + while (await LocalDb.migrateLegacyRawArchive(budget: Duration.zero) > 0) { + calls++; + if (calls == 1) { + await LocalDb.archiveRawRecord(_rec(9999, _frameHex(9999))); + expected.add('|${_frameHex(9999)}|9999|${1750000000000 + 9999000}'); + expected.sort(); + } + } + expect(calls, greaterThanOrEqualTo(3)); + expect( + await db.rawQuery("SELECT 1 FROM sqlite_master WHERE name = 'raw_archive_legacy'"), + isEmpty, + ); + final view = await db.rawQuery( + "SELECT sql FROM sqlite_master WHERE name = '${LocalDb.rawArchiveView}'"); + expect('${view.single['sql']}', isNot(contains('legacy'))); + final after = await seen(); + // The mid-move duplicate collapsed; counter differs, so compare keys. + expect(after.map((s) => s.split('|').take(2).join('|')).toList(), + expected.map((s) => s.split('|').take(2).join('|')).toList()); + expect(after.length, expected.length); + + final newBytes = await _bytes(db, [ + LocalDb.rawArchiveTable, + for (final r in await db.rawQuery("SELECT name FROM sqlite_master " + "WHERE type = 'index' AND tbl_name = ?", [LocalDb.rawArchiveTable])) + r['name'] as String, + ]); + // ignore: avoid_print + print('raw_archive: ${rows.length} frames, legacy ${legacyBytes >> 10} KiB' + ' -> ${newBytes >> 10} KiB ' + '(${(legacyBytes / rows.length).round()} -> ' + '${(newBytes / after.length).round()} bytes/frame)'); + // A 2.2 KB row cannot share a 4 KB page with another, so a frame costs a + // page; the hex row and its key copy cost two and change. + expect(newBytes, lessThan(legacyBytes * 0.55)); + + // Reopening a converted database changes nothing. + await LocalDb.close(); + expect(await seen(), after); + }); + + test('the move stops after a page once an offload starts', () async { + await _legacyDb('raw_frames_offload.db', [ + for (var i = 0; i < 1200; i++) _legacyRow('', _frameHex(100 + i), i), + ]); + // No time budget: only the offload signal can stop it early. + expect(await LocalDb.migrateLegacyRawArchive(shouldStop: () => true), 500); + expect(await LocalDb.migrateLegacyRawArchive(), 700); + }); + + test('an older build can still open the file, and its frames come back', + () async { + await _fresh('raw_frames_rollback.db'); + await LocalDb.close(); + LocalDb.dbName = 'raw_frames_rollback.db'; + await LocalDb.archiveRawRecord(_rec(1, _frameHex(1))); + await LocalDb.close(); + // What the build before this layout runs on every open, verbatim. With + // a view named raw_archive the index line failed ("views may not be + // indexed") and the open with it. + final old = await databaseFactory.openDatabase( + await _path('raw_frames_rollback.db')); + await old.execute(''' + CREATE TABLE IF NOT EXISTS raw_archive ( + device_id TEXT NOT NULL DEFAULT '', + hex TEXT NOT NULL, + counter INTEGER, + packet_type INTEGER NOT NULL, + rec_ts INTEGER, + captured_at INTEGER NOT NULL, + reason TEXT NOT NULL, + PRIMARY KEY (device_id, hex) + ) + '''); + await old.execute('CREATE INDEX IF NOT EXISTS idx_raw_archive_captured ' + 'ON raw_archive(captured_at DESC)'); + // The old build archives a frame of its own. + await old.insert('raw_archive', _legacyRow('', _frameHex(2), 2)); + await old.close(); + + // Back on this build: both frames are there. + final keys = [ + for (final r in await (await LocalDb.instance).rawQuery( + 'SELECT hex FROM ${LocalDb.rawArchiveView} ORDER BY hex')) + r['hex'], + ]; + expect(keys, [_frameHex(1), _frameHex(2)]..sort()); + }); + + test('a rollback to a build from before device keys still migrates', + () async { + await _fresh('raw_frames_rollback_v50.db'); + await LocalDb.instance; + await LocalDb.close(); + // A schema-50 build: raw_archive keyed by hex alone, no device_id. + final old = await databaseFactory.openDatabase( + await _path('raw_frames_rollback_v50.db')); + await old.execute(''' + CREATE TABLE IF NOT EXISTS raw_archive ( + hex TEXT PRIMARY KEY, + counter INTEGER, + packet_type INTEGER NOT NULL, + rec_ts INTEGER, + captured_at INTEGER NOT NULL, + reason TEXT NOT NULL + ) + '''); + await old.insert('raw_archive', { + 'hex': _frameHex(50), + 'counter': 50, + 'packet_type': 0x2F, + 'captured_at': 1750000050000, + 'reason': 'undecodable_rec_v20', + }); + await old.close(); + + await LocalDb.instance; + while (await LocalDb.migrateLegacyRawArchive() > 0) {} + final rows = await _viewRows(); + expect(rows.map((r) => '${r['device_id']}|${r['hex']}'), + ['|${_frameHex(50)}']); + }); + + test('the input signature reads the archive tables to the same value as ' + 'the view', () async { + // Mid-move: legacy rows still aside, and the frame store holding both a + // blob frame and a verbatim text one. + await _legacyDb('raw_frames_signature.db', [ + for (var i = 0; i < 3; i++) _legacyRow('ring-1', _frameHex(300 + i), i), + ]); + await LocalDb.migrateLegacyRawArchive(budget: Duration.zero); + await LocalDb.archiveRawRecord(_rec(7, _frameHex(77)), deviceId: 'ring-1'); + await LocalDb.archiveRawRecord(_rec(8, 'Not-Hex'), deviceId: 'ring-1'); + final db = await LocalDb.instance; + final viewRow = (await db.rawQuery( + 'SELECT COUNT(*), MAX(captured_at), TOTAL(captured_at), ' + 'TOTAL(LENGTH(hex)) FROM ${LocalDb.rawArchiveView} WHERE device_id = ?', + ['ring-1'], + )).single; + final keys = viewRow.keys.toList()..sort(); + final expected = jsonEncode([ + [for (final k in keys) viewRow[k]], + ]); + final sig = await LocalDb.dayInputSignature( + dayId: '2025-06-15', + prevDayId: '2025-06-14', + nextDayId: '2025-06-16', + fromSec: 1749900000, + toSec: 1750100000, + wearableDeviceId: 'ring-1', + ); + expect(sig, contains('wearable_archive=$expected;')); + }); + + test('a backup of the new layout restores every frame', () async { + await _fresh('raw_frames_backup_src.db'); + final frames = [_frameHex(1), 'Not-Hex', _frameHex(2)]; + await LocalDb.commitSyncBatch(const [], const [], + archives: [for (var i = 0; i < frames.length; i++) _rec(i, frames[i])]); + final before = await _viewRows(); + final copy = await LocalDb.exportCopy( + into: Directory.systemTemp.createTempSync('raw_frames')); + await _fresh('raw_frames_backup_dst.db'); + await LocalDb.importFromDbFile(copy); + expect(await _viewRows(), before); + }); + + test('thinning keeps the same 1-in-60 sample in both tables', () async { + final rows = [ + for (var i = 0; i < 240; i++) _legacyRow('', _frameHex(i), i), + ]; + await _legacyDb('raw_frames_thin.db', rows); + await LocalDb.instance; + // Half moved, half still aside, when the retention pass thins. + await LocalDb.migrateLegacyRawArchive(budget: Duration.zero); + final deleted = await LocalDb.thinRawArchiveBefore(1750000000000 + 1000); + expect(deleted, 240 - 4); // counters 0, 60, 120, 180 stay + expect([for (final r in await _viewRows()) r['counter']]..sort(), + [0, 60, 120, 180]); + }); +} diff --git a/test/raw_archive_test.dart b/test/raw_archive_test.dart index 72daaf58f..dd6163f9a 100644 --- a/test/raw_archive_test.dart +++ b/test/raw_archive_test.dart @@ -360,7 +360,7 @@ void main() { // Opening runs the whole ladder, v44 included. final db = await LocalDb.instance; final archived = (await db.rawQuery( - 'SELECT COUNT(*) AS n FROM raw_archive WHERE reason IN ' + 'SELECT COUNT(*) AS n FROM ${LocalDb.rawArchiveView} WHERE reason IN ' '(${List.filled(LocalDb.redrivableArchiveReasons.length, '?').join(',')})', LocalDb.redrivableArchiveReasons, )).first['n']; @@ -377,4 +377,38 @@ void main() { } }, timeout: const Timeout(Duration(minutes: 20))); } + + + test('a VACUUM drops an unfinished replay\'s rowid resume point', () async { + await LocalDb.putComputeFreshness( + LocalDb.kArchiveRedriveKey, + jsonEncode({'recovered': 5, 'after_hex': '42', 'after_device': 'x'}), + ); + await LocalDb.vacuumAndCheckpoint(); + final state = jsonDecode((await LocalDb.computeFreshness( + LocalDb.kArchiveRedriveKey))!['payload_json'] as String) as Map; + expect(state, {'recovered': 5}); + + // A finished replay stays finished. + await LocalDb.putComputeFreshness( + LocalDb.kArchiveRedriveKey, jsonEncode({'recovered': 5, 'done': true})); + await LocalDb.vacuumAndCheckpoint(); + final done = jsonDecode((await LocalDb.computeFreshness( + LocalDb.kArchiveRedriveKey))!['payload_json'] as String) as Map; + expect(done['done'], true); + await LocalDb.deleteComputeFreshness(LocalDb.kArchiveRedriveKey); + }); + + test('a blocked WAL checkpoint is not reported as done', () { + expect( + LocalDb.checkpointCompleted([ + {'busy': 1, 'log': 40, 'checkpointed': 12}, + ]), + isFalse); + expect( + LocalDb.checkpointCompleted([ + {'busy': 0, 'log': 0, 'checkpointed': 0}, + ]), + isTrue); + }); } diff --git a/test/raw_prune_gating_test.dart b/test/raw_prune_gating_test.dart index 667c63b6a..cf0091e6b 100644 --- a/test/raw_prune_gating_test.dart +++ b/test/raw_prune_gating_test.dart @@ -80,14 +80,14 @@ void main() { final dataNow = _dayStart('2026-05-20') + 12 * 3600; final cutoff = DerivationEngine.rawPruneCutoffSec( dataNowSec: dataNow, - rawDayIds: const ['2026-05-12', '2026-05-13', '2026-05-19'], + rawDayIds: const ['2026-05-12', '2026-05-15', '2026-05-19'], finalizedDayIds: const {'2026-05-12', '2026-05-19'}, ); - // Held at 05-12 (05-13's night search starts 05-12 noon)… - expect(cutoff, _dayStart('2026-05-12')); + // Held at 05-14 (05-15's night search starts 05-14 noon)… + expect(cutoff, _dayStart('2026-05-14')); // …but older finalized days are still reclaimed. The old guard // returned early and kept them too. - expect(cutoff, greaterThan(_dayStart('2026-05-11'))); + expect(cutoff, greaterThan(_dayStart('2026-05-13'))); }); // Day D's night starts the evening before. A day with a result that has @@ -120,7 +120,7 @@ void main() { )!; expect(cutoff, greaterThan(_dayStart('2026-01-05'))); // Never keeps more than the documented ceiling behind the data edge. - expect(dataNow - cutoff, lessThanOrEqualTo(14 * 86400)); + expect(dataNow - cutoff, lessThanOrEqualTo(7 * 86400)); }); test('a day still inside the retention window does not move it', () { diff --git a/test/rescan_quiet_narrowing_test.dart b/test/rescan_quiet_narrowing_test.dart new file mode 100644 index 000000000..cd338b09c --- /dev/null +++ b/test/rescan_quiet_narrowing_test.dart @@ -0,0 +1,44 @@ +// A rescan triggered only by the quiet level re-prices just the days whose +// strain window holds a level that moved. + +import 'package:flutter_test/flutter_test.dart'; +import 'package:openstrap_edge/compute/derivation_engine.dart'; + +void main() { + List> entries(Map m) => [ + for (final e in (m.entries.toList() + ..sort((a, b) => a.key.compareTo(b.key)))) + [e.key, e.value], + ]; + + test('a value that moved is a change, and only that one', () { + final before = entries({ + '2026-07-01': .10, + '2026-07-02': .15, + '2026-07-03': .20, + }); + final after = entries({ + '2026-07-01': .10, + '2026-07-02': .25, + '2026-07-03': .20, + }); + expect(DerivationEngine.changedQuietDates(before, after), {'2026-07-02'}); + }); + + test('a new level and a removed one are changes', () { + final before = entries({'2026-07-01': .10, '2026-07-02': .15}); + final after = entries({'2026-07-01': .10, '2026-07-03': .30}); + expect(DerivationEngine.changedQuietDates(before, after), + {'2026-07-02', '2026-07-03'}); + }); + + test('an entry that slid off the older end is not a change', () { + final before = entries({'2026-07-01': .10, '2026-07-02': .15}); + final after = entries({'2026-07-02': .15, '2026-07-03': .30}); + expect(DerivationEngine.changedQuietDates(before, after), {'2026-07-03'}); + }); + + test('nothing stored yet means rescan everything', () { + expect(DerivationEngine.changedQuietDates(null, const []), isNull); + }); +} diff --git a/test/session_devices_test.dart b/test/session_devices_test.dart index c2f1528f1..a01219cb7 100644 --- a/test/session_devices_test.dart +++ b/test/session_devices_test.dart @@ -72,7 +72,7 @@ void main() { [('body_temp', 36.5), ('body_temp', 36.52)].reversed.toList()); expect(rows.first['ts_ms'], morning.millisecondsSinceEpoch); expect(rows.every((r) => r['attribution'] == kThermometer.label), isTrue); - final archive = await (await LocalDb.instance).query('raw_archive'); + final archive = await (await LocalDb.instance).query(LocalDb.rawArchiveView); expect(archive.length, 3, reason: 'every indication is banked'); }); diff --git a/test/startup_and_rebuild_recovery_test.dart b/test/startup_and_rebuild_recovery_test.dart index ecfa4d8dc..df34ef3b3 100644 --- a/test/startup_and_rebuild_recovery_test.dart +++ b/test/startup_and_rebuild_recovery_test.dart @@ -300,7 +300,7 @@ void main() { 'payload_json': '{"mean":51}', 'updated_at': 1, }); - await db.insert('raw_archive', { + await db.insert(LocalDb.rawArchiveView, { 'hex': 'deadbeef', 'counter': 7, 'packet_type': 0x2f, @@ -375,7 +375,10 @@ void main() { reason: 'the old file is parked, never deleted'); for (final t in salvaged) { - expect(await db.query(t), hasLength(1), reason: '$t did not come back'); + // The archive reads back through its view on this layout. + final live = t == 'raw_archive' ? LocalDb.rawArchiveView : t; + expect(await db.query(live), hasLength(1), + reason: '$t did not come back'); expect(r.salvaged[t], 1, reason: '$t was not reported as recovered'); } // Values, not just counts — a merge that dropped columns would pass a @@ -404,7 +407,9 @@ void main() { final again = await LocalDb.instance; expect(LocalDb.lastRebuild, isNull); for (final t in salvaged) { - expect(await again.query(t), hasLength(1), reason: '$t lost on reopen'); + final live = t == 'raw_archive' ? LocalDb.rawArchiveView : t; + expect(await again.query(live), hasLength(1), + reason: '$t lost on reopen'); } await LocalDb.close(); }); diff --git a/test/substrate_archive_db_test.dart b/test/substrate_archive_db_test.dart index d007a39bc..aeddd85ce 100644 --- a/test/substrate_archive_db_test.dart +++ b/test/substrate_archive_db_test.dart @@ -89,10 +89,14 @@ Future _clear() async { 'decoded_onehz', 'decoded_rr', 'substrate_archive', - 'compute_freshness', ]) { await db.delete(t); } + // Everything but the marker every opened phone database carries: without it + // an export here would read as a backup from before the 90-day default. + await db.delete('compute_freshness', + where: 'key != ?', + whereArgs: ['substrate_archive_policy_default_checked']); SubstrateArchive.debugCorruptEncode = false; SubstrateArchive.debugBeforeBucket = null; SubstrateArchive.debugThrowInBucket = null; @@ -437,6 +441,18 @@ void main() { expect(await (await _db()).query('substrate_archive'), isEmpty); }); + test('with no decoded rows left, the window is measured from the archive', + () async { + await _seed(); + await LocalDb.pruneDecodedBeforeRecTs(cutoff, archive: policy); + final db = await _db(); + await db.delete('decoded_onehz'); + expect(await LocalDb.lastDecodedRecTs(), isNull); + final newest = + (await LocalDb.substrateArchiveStats())['newest_to_ts'] as int; + expect(await LocalDb.substrateArchiveEdge(), newest); + }); + test('evictSubstrateArchive drops buckets older than keepDays; -1 keeps ' 'all; setting the policy to 0 purges', () async { await _seed(); @@ -1000,13 +1016,15 @@ void main() { await LocalDb.pruneDecodedBeforeRecTs(cutoff); final snapshot = await LocalDb.exportCopy(); final db = await _db(); - // A fresh phone: no policy row and no archive of its own. + // A fresh phone: no policy row, no archive and no readings of its own. await db.delete( 'compute_freshness', where: 'key = ?', whereArgs: [SubstrateArchive.policyKey], ); await db.delete('substrate_archive'); + await db.delete('decoded_onehz'); + await db.delete('decoded_rr'); await LocalDb.importFromDbFile(snapshot); expect( await LocalDb.substrateArchivePolicy(), @@ -1023,6 +1041,31 @@ void main() { }); }); + test('a backup\'s Off policy is not adopted over live readings with no ' + 'archive yet', () async { + await _seed(); + await LocalDb.setSubstrateArchivePolicy(const SubstrateArchivePolicy(0)); + final offBackup = await LocalDb.exportCopy(); + final db = await _db(); + // This phone: readings, no archive yet, no stored setting. + await db.delete('compute_freshness', + where: 'key = ?', whereArgs: [SubstrateArchive.policyKey]); + await db.delete('substrate_archive'); + final live = (await db.query('decoded_onehz')).length; + expect(live, greaterThan(0)); + + await LocalDb.importFromDbFile(offBackup); + expect( + await db.query('compute_freshness', + where: 'key = ?', whereArgs: [SubstrateArchive.policyKey]), + isEmpty, + ); + // The next prune archives them instead of deleting them outright. + await LocalDb.pruneDecodedBeforeRecTs(cutoff); + expect(await db.query('substrate_archive'), isNotEmpty); + await File(offBackup).delete(); + }); + test('a backup\'s Off policy never deletes this phone\'s archive; on a ' 'fresh phone it is adopted', () async { await _seed(); @@ -1053,8 +1096,11 @@ void main() { isEmpty, ); - // A fresh phone adopts it — and then takes no archive rows in. + // A fresh phone (no archive, no readings) adopts it — and then takes no + // archive rows in. await db.delete('substrate_archive'); + await db.delete('decoded_onehz'); + await db.delete('decoded_rr'); await LocalDb.importFromDbFile(offBackup); expect( await LocalDb.substrateArchivePolicy(), diff --git a/test/substrate_archive_window_test.dart b/test/substrate_archive_window_test.dart new file mode 100644 index 000000000..51df94524 --- /dev/null +++ b/test/substrate_archive_window_test.dart @@ -0,0 +1,124 @@ +// The archive window the data settings step through. + +import 'package:flutter_test/flutter_test.dart'; +import 'package:openstrap_edge/data/substrate_archive.dart'; +import 'package:sqflite_common_ffi/sqflite_ffi.dart'; + +void main() { + test('defaults to 90 days', () { + expect(SubstrateArchivePolicy.defaults.keepDays, 90); + expect(SubstrateArchivePolicy.fromJson(null).keepDays, 90); + }); + + test('steps 30 → 90 → 365 → forever → 30', () { + var p = const SubstrateArchivePolicy(30); + final seen = []; + for (var i = 0; i < 4; i++) { + p = p.next; + seen.add(p.keepDays); + } + expect(seen, [90, 365, -1, 30]); + }); + + test('an off or hand-set window moves to the next longer choice', () { + expect(const SubstrateArchivePolicy(0).next.keepDays, 30); + expect(const SubstrateArchivePolicy(45).next.keepDays, 90); + expect(const SubstrateArchivePolicy(400).next.keepDays, -1); + }); + + test('only a shorter window counts as deleting history', () { + const forever = SubstrateArchivePolicy(-1); + const year = SubstrateArchivePolicy(365); + const month = SubstrateArchivePolicy(30); + expect(forever.keepsLongerThan(month), isTrue); + expect(year.keepsLongerThan(month), isTrue); + expect(month.keepsLongerThan(year), isFalse); + expect(month.keepsLongerThan(forever), isFalse); + expect(year.keepsLongerThan(year), isFalse); + }); + + group('an archive from before the 90-day default', () { + late Database db; + setUpAll(sqfliteFfiInit); + setUp(() async { + db = await databaseFactoryFfi.openDatabase(inMemoryDatabasePath, + options: OpenDatabaseOptions(singleInstance: false)); + await db.execute('CREATE TABLE compute_freshness (key TEXT PRIMARY KEY, ' + 'payload_json TEXT NOT NULL, updated_at INTEGER NOT NULL)'); + await db.execute('CREATE TABLE ${SubstrateArchive.table} (utc_day INTEGER, day TEXT)'); + await db.execute('CREATE TABLE decoded_onehz (rec_ts INTEGER)'); + await db.execute('CREATE TABLE decoded_rr (rec_ts INTEGER)'); + }); + tearDown(() => db.close()); + + test('keeps its year when nothing was ever chosen', () async { + await db.insert(SubstrateArchive.table, {'day': '2025-11-01'}); + await SubstrateArchive.keepLegacyWindowOnce(db); + expect((await SubstrateArchive.readPolicy(db)).keepDays, 365); + }); + + test('a new install gets the default, and the check runs once', () async { + await SubstrateArchive.keepLegacyWindowOnce(db); + expect((await SubstrateArchive.readPolicy(db)).keepDays, 90); + // Archiving starts later on this build: still the default. + await db.insert(SubstrateArchive.table, {'day': '2026-10-01'}); + await SubstrateArchive.keepLegacyWindowOnce(db); + expect((await SubstrateArchive.readPolicy(db)).keepDays, 90); + }); + + test('an empty archive with band readings already stored keeps a year', + () async { + await db.insert('decoded_onehz', {'rec_ts': 1750000000}); + await SubstrateArchive.keepLegacyWindowOnce(db); + expect((await SubstrateArchive.readPolicy(db)).keepDays, 365); + }); + + test('a backup from before the setting restores with its year', () async { + final backup = await databaseFactoryFfi.openDatabase( + inMemoryDatabasePath, + options: OpenDatabaseOptions(singleInstance: false)); + addTearDown(backup.close); + await backup.execute('CREATE TABLE compute_freshness (key TEXT PRIMARY ' + 'KEY, payload_json TEXT NOT NULL, updated_at INTEGER NOT NULL)'); + await backup.execute('CREATE TABLE ${SubstrateArchive.table} (utc_day INTEGER, day TEXT)'); + // Fresh install: the check already ran and found nothing. + await SubstrateArchive.keepLegacyWindowOnce(db); + await SubstrateArchive.adoptPolicy(backup, db); + expect((await SubstrateArchive.readPolicy(db)).keepDays, 90, + reason: 'a backup with no history brings no window'); + await backup.insert(SubstrateArchive.table, {'day': '2025-11-01'}); + // A backup of a phone already on this build, with no setting chosen, + // was getting the default: nothing to adopt. + await backup.insert('compute_freshness', { + 'key': 'substrate_archive_policy_default_checked', + 'payload_json': '{}', + 'updated_at': 0, + }); + await SubstrateArchive.adoptPolicy(backup, db); + expect((await SubstrateArchive.readPolicy(db)).keepDays, 90); + // One from before this build keeps its year. + await backup.delete('compute_freshness'); + await SubstrateArchive.adoptPolicy(backup, db); + expect((await SubstrateArchive.readPolicy(db)).keepDays, 365); + }); + + test('until the one-time check has run, no setting reads as a year', + () async { + // The check failed (or has not run): nothing may be evicted past 90. + expect((await SubstrateArchive.readPolicy(db)).keepDays, 365); + await SubstrateArchive.keepLegacyWindowOnce(db); // empty: the default + expect((await SubstrateArchive.readPolicy(db)).keepDays, 90); + }); + + test('a window the user picked is left alone', () async { + await db.insert(SubstrateArchive.table, {'day': '2025-11-01'}); + await db.insert('compute_freshness', { + 'key': SubstrateArchive.policyKey, + 'payload_json': const SubstrateArchivePolicy(30).toJson(), + 'updated_at': 0, + }); + await SubstrateArchive.keepLegacyWindowOnce(db); + expect((await SubstrateArchive.readPolicy(db)).keepDays, 30); + }); + }); +} diff --git a/test/synthetic_colmi_day_test.dart b/test/synthetic_colmi_day_test.dart index b975a5be3..bab2f3ee0 100644 --- a/test/synthetic_colmi_day_test.dart +++ b/test/synthetic_colmi_day_test.dart @@ -254,7 +254,7 @@ void main() { // counts from. final db = await LocalDb.instance; final stamps = await db.rawQuery( - "SELECT DISTINCT rec_ts FROM raw_archive WHERE reason LIKE 'colmi_big_%' " + "SELECT DISTINCT rec_ts FROM ${LocalDb.rawArchiveView} WHERE reason LIKE 'colmi_big_%' " "AND device_id = 'colmi-synthetic'"); expect(stamps.map((r) => r['rec_ts']), [SyntheticDay.sec(DateTime(2026, 10, 4, 22, 30))]); @@ -267,7 +267,7 @@ void main() { final reply = colmiBigDataRequest( kColmiBigTemperature, [0, 30, for (var k = 0; k < 48; k++) 150]); final db = await LocalDb.instance; - await db.insert('raw_archive', { + await db.insert(LocalDb.rawArchiveView, { 'device_id': 'colmi-midnight', 'hex': [for (final b in reply) b.toRadixString(16).padLeft(2, '0')] .join(), diff --git a/test/ultrahuman_link_test.dart b/test/ultrahuman_link_test.dart index 17e1cd267..452fa7631 100644 --- a/test/ultrahuman_link_test.dart +++ b/test/ultrahuman_link_test.dart @@ -152,7 +152,7 @@ void main() { final db = await LocalDb.instance; expect((await db.query('decoded_onehz')).length, 4, reason: 're-reading today re-banks the same seconds, not duplicates'); - final archive = await db.query('raw_archive'); + final archive = await db.query(LocalDb.rawArchiveView); expect(archive.length, 4, reason: 'raw_archive is keyed on the bytes'); }); @@ -161,7 +161,7 @@ void main() { await UltrahumanLink.instance.ingestForTest(_deviceId, _ring(first), nowSeconds: () => _sec(_today.add(const Duration(hours: 12)))); final db = await LocalDb.instance; - final archive = await db.query('raw_archive', orderBy: 'counter'); + final archive = await db.query(LocalDb.rawArchiveView, orderBy: 'counter'); expect(archive.map((r) => r['counter']), [1, 2, 3]); }); diff --git a/test/whoop_freeze_golden_test.dart b/test/whoop_freeze_golden_test.dart index cfc95b335..3dc238007 100644 --- a/test/whoop_freeze_golden_test.dart +++ b/test/whoop_freeze_golden_test.dart @@ -198,6 +198,9 @@ Future> _snapshot(Profile profile) async { String sha(Object? stored) => sha256.convert(utf8.encode(stored as String)).toString(); for (final r in dayResults) { + // An index over the payload's own "imported" marker (pinned below by the + // payload hash), kept by trigger — storage, not output. + r.remove('imported'); r['payload_sha256'] = sha(r['payload_json']); r['payload_json'] = SeriesCodec.decodePayloadJson(r['payload_json']); r['window_json'] = jsonDecode(r['window_json'] as String);