diff --git a/crates/tracedecay-global-db/Cargo.toml b/crates/tracedecay-global-db/Cargo.toml index 5f2016d07..a8a3948f9 100644 --- a/crates/tracedecay-global-db/Cargo.toml +++ b/crates/tracedecay-global-db/Cargo.toml @@ -51,7 +51,9 @@ tokio = { version = "1", features = ["full", "test-util"] } # Test-only helper surfaces this crate's test targets reach across crate # boundaries: engine fixtures, session normalizers, and temporal page requests. tracedecay-graph-db = { path = "../tracedecay-graph-db", version = "0.1.0", features = ["test-helpers"] } -tracedecay-runtime-core = { path = "../tracedecay-runtime-core", version = "0.1.0", features = ["test-helpers"] } +# `test-transport` exposes the maintenance-authority fixture publisher this +# crate's WAL reclaim tests need to prove exclusive-maintenance truncation. +tracedecay-runtime-core = { path = "../tracedecay-runtime-core", version = "0.1.0", features = ["test-helpers", "test-transport"] } tracedecay-sessions = { path = "../tracedecay-sessions", version = "0.1.0", features = ["test-helpers"] } tracedecay-temporal-query = { path = "../tracedecay-temporal-query", version = "0.1.0", features = ["test-helpers"] } diff --git a/crates/tracedecay-global-db/src/checkpoint_tests.rs b/crates/tracedecay-global-db/src/checkpoint_tests.rs index 68843acde..7f83ecbd7 100644 --- a/crates/tracedecay-global-db/src/checkpoint_tests.rs +++ b/crates/tracedecay-global-db/src/checkpoint_tests.rs @@ -1,67 +1,244 @@ +use crate::RegisteredGlobalDb; +use crate::registered_maintenance::{REGISTERED_WAL_RECLAIM_TRIGGER_BYTES, RegisteredWalReclaimV1}; use crate::tests::harness::RegisteredGlobalDbHarness; -async fn pinned_wal_reader() -> ( - RegisteredGlobalDbHarness, - tracedecay_runtime_core::db::DatabaseEngineReadSnapshot, -) { - let harness = RegisteredGlobalDbHarness::open("pinned-wal-reader").await; - harness - .registered +/// One mebibyte of incompressible payload per insert; each row spans ~257 +/// database pages, so every insert appends about 1 MiB of WAL frames. +const WAL_LOAD_STATEMENT: &str = "INSERT INTO wal_load(value) VALUES (randomblob(1048576))"; + +/// The retained writer's passive checkpoint lane engages at its 32 MiB soft +/// budget; synthetic load must exceed it for a pinned reader to surface as a +/// typed pending failure instead of a below-budget no-op. +const PRESSURED_WAL_LOAD_BATCHES: usize = 40; + +fn wal_file_bytes(database: &RegisteredGlobalDb) -> u64 { + let mut wal = database.db_path().as_os_str().to_owned(); + wal.push("-wal"); + std::fs::metadata(std::path::PathBuf::from(wal)) + .map(|metadata| metadata.len()) + .unwrap_or(0) +} + +/// Creates the load table, pins a read snapshot on the empty WAL position, +/// then appends enough incompressible frames to exceed the writer's soft +/// checkpoint budget. Every frame stays unbackfillable while the returned +/// snapshot lives. +async fn grow_pressured_wal( + database: &RegisteredGlobalDb, +) -> tracedecay_runtime_core::db::DatabaseEngineReadSnapshot { + database .writer_connection() .unwrap() - .execute_batch( - "PRAGMA wal_autocheckpoint=0; - PRAGMA busy_timeout=1; - CREATE TABLE checkpoint_probe(value INTEGER NOT NULL); - INSERT INTO checkpoint_probe(value) VALUES (1);", - ) + .execute_batch("CREATE TABLE wal_load(value BLOB NOT NULL)") .await .unwrap(); - let reader = harness.registered.read_snapshot().await.unwrap(); + let reader = database.read_snapshot().await.unwrap(); let mut rows = reader - .query("SELECT COUNT(*) FROM checkpoint_probe", ()) + .query("SELECT COUNT(*) FROM wal_load", ()) .await .unwrap(); assert_eq!( rows.next().await.unwrap().unwrap().get::(0).unwrap(), - 1 + 0 ); drop(rows); + let writer = database.writer_connection().unwrap(); + for _ in 0..PRESSURED_WAL_LOAD_BATCHES { + writer.execute(WAL_LOAD_STATEMENT, ()).await.unwrap(); + } + reader +} + +#[tokio::test] +async fn pressured_checkpoint_reports_pinned_reader_and_reclaims_after_release() { + let harness = RegisteredGlobalDbHarness::open("pressured-wal-checkpoint").await; + let reader = grow_pressured_wal(&harness.registered).await; + let high_water = wal_file_bytes(&harness.registered); + assert!( + high_water > REGISTERED_WAL_RECLAIM_TRIGGER_BYTES, + "synthetic load must exceed the reclaim trigger, measured {high_water} bytes" + ); + + // A pinned WAL under size pressure is a typed failure, not a vacuous + // success: the runtime checkpoint lane reports it cannot complete. + let error = harness + .registered + .checkpoint_result() + .await + .unwrap_err() + .to_string(); + assert!(error.contains("pending"), "{error}"); + + drop(reader); + let receipt = harness.registered.checkpoint_result().await.unwrap(); + assert!( + receipt.wal_bytes_before >= high_water, + "receipt must report the measured high-water WAL file, got {} for {high_water}", + receipt.wal_bytes_before + ); + // The harness client holds fixture (non-maintenance) write authority, so + // the drained file keeps its high-water size and the receipt says exactly + // why it was not reclaimed. + assert_eq!( + receipt.reclaim, + RegisteredWalReclaimV1::RequiresExclusiveMaintenance { + trigger_bytes: REGISTERED_WAL_RECLAIM_TRIGGER_BYTES + } + ); + assert_eq!(receipt.wal_bytes_after, receipt.wal_bytes_before); + + // Drain evidence: the checkpoint backfilled every frame, so the next + // write rewinds into the existing file instead of appending past the + // high-water mark. harness .registered .writer_connection() .unwrap() - .execute("INSERT INTO checkpoint_probe(value) VALUES (2)", ()) + .execute("INSERT INTO wal_load(value) VALUES (x'01')", ()) .await .unwrap(); - (harness, reader) + let after_write = wal_file_bytes(&harness.registered); + assert!( + after_write <= receipt.wal_bytes_after, + "post-checkpoint write must reuse the drained WAL, grew {} past {}", + after_write, + receipt.wal_bytes_after + ); } +/// The full Plan 38 reclaim journey: a PASSIVE-busy WAL past the trigger +/// stays a typed failure while pinned, and once the reader releases, the +/// exclusive-maintenance truncation lane drains it to a zero-byte file. #[tokio::test] -async fn checkpoint_result_reports_busy_and_recovers_after_reader_finishes() { - let (harness, reader) = pinned_wal_reader().await; +async fn maintenance_truncate_drains_passive_busy_wal_to_zero_bytes() { + crate::register_test_schema_installer(); + let directory = tempfile::tempdir().unwrap(); + let profile_root = directory.path().join("profile"); + tracedecay_runtime_core::storage::PrivateStoreIo::create_dir_all(&profile_root).unwrap(); + let lease = tracedecay_runtime_core::lifecycle_lease::acquire_exclusive_for_profile( + &profile_root, + "wal reclaim maintenance test", + ) + .unwrap(); + let _scope = tracedecay_runtime_core::db::enter_maintenance_database_scope( + &lease, + &profile_root, + "wal reclaim maintenance test", + ) + .unwrap(); + let db_path = profile_root.join("projects/wal-reclaim/store.db"); + std::fs::create_dir_all(db_path.parent().unwrap()).unwrap(); + let authority = tracedecay_runtime_core::db::DatabaseAuthority::for_runtime( + &db_path, + "wal reclaim maintenance test", + ) + .unwrap(); + assert_eq!( + authority.role(), + tracedecay_runtime_core::db::DatabaseAuthorityRole::Maintenance, + "an entered maintenance scope must grant exclusive maintenance authority" + ); + let database = tracedecay_runtime_core::db::Database::publish_maintenance_test_runtime( + &db_path, + &authority, + tracedecay_runtime_core::db::TestDatabaseRuntimeMode::Initialize, + ) + .await + .unwrap() + .0; + let registered = RegisteredGlobalDb::from_database_for_wal_maintenance_test(database); - let error = harness - .registered + let reader = grow_pressured_wal(®istered).await; + let high_water = wal_file_bytes(®istered); + assert!( + high_water > REGISTERED_WAL_RECLAIM_TRIGGER_BYTES, + "synthetic load must exceed the reclaim trigger, measured {high_water} bytes" + ); + + // Maintenance authority does not bypass reader safety: the pinned WAL is + // still a typed pending failure, never a forced truncation. + let error = registered .checkpoint_result() .await .unwrap_err() .to_string(); - assert!(error.contains("WAL checkpoint incomplete"), "{error}"); - assert!(error.contains("busy=1"), "{error}"); - assert!(error.contains("log_frames="), "{error}"); - assert!(error.contains("checkpointed_frames="), "{error}"); + assert!(error.contains("pending"), "{error}"); + assert_eq!(wal_file_bytes(®istered), high_water); drop(reader); - harness.registered.checkpoint_result().await.unwrap(); + let receipt = registered.checkpoint_result().await.unwrap(); + assert!( + receipt.wal_bytes_before >= high_water, + "receipt must report the measured high-water WAL file, got {} for {high_water}", + receipt.wal_bytes_before + ); + assert_eq!( + receipt.reclaim, + RegisteredWalReclaimV1::Truncated { + trigger_bytes: REGISTERED_WAL_RECLAIM_TRIGGER_BYTES + } + ); + assert_eq!( + receipt.wal_bytes_after, 0, + "truncation must reclaim the WAL file" + ); + assert_eq!(wal_file_bytes(®istered), 0); + + // The truncated store keeps serving reads and writes. + let writer = registered.writer_connection().unwrap(); + writer + .execute("INSERT INTO wal_load(value) VALUES (x'01')", ()) + .await + .unwrap(); + let mut rows = registered + .read_connection() + .query("SELECT COUNT(*) FROM wal_load", ()) + .await + .unwrap(); + assert_eq!( + rows.next().await.unwrap().unwrap().get::(0).unwrap(), + (PRESSURED_WAL_LOAD_BATCHES + 1) as i64 + ); +} + +#[tokio::test] +async fn below_trigger_checkpoint_reports_measured_wal_bytes() { + let harness = RegisteredGlobalDbHarness::open("below-trigger-checkpoint").await; + harness + .registered + .writer_connection() + .unwrap() + .execute_batch( + "CREATE TABLE checkpoint_probe(value INTEGER NOT NULL); + INSERT INTO checkpoint_probe(value) VALUES (1);", + ) + .await + .unwrap(); + + let receipt = harness.registered.checkpoint_result().await.unwrap(); + assert!( + receipt.wal_bytes_before > 0, + "the seeded store must have live WAL frames" + ); + assert!(receipt.wal_bytes_before < REGISTERED_WAL_RECLAIM_TRIGGER_BYTES); + assert_eq!( + receipt.reclaim, + RegisteredWalReclaimV1::BelowTrigger { + trigger_bytes: REGISTERED_WAL_RECLAIM_TRIGGER_BYTES + } + ); + assert_eq!(receipt.wal_bytes_after, receipt.wal_bytes_before); } #[tokio::test] async fn public_checkpoint_remains_best_effort_when_reader_is_busy() { - let (harness, reader) = pinned_wal_reader().await; + let harness = RegisteredGlobalDbHarness::open("best-effort-checkpoint").await; + let reader = grow_pressured_wal(&harness.registered).await; + // The best-effort entry point must swallow the pinned-reader failure so + // shutdown paths never abort on a busy WAL. harness.registered.checkpoint().await; drop(reader); diff --git a/crates/tracedecay-global-db/src/lib.rs b/crates/tracedecay-global-db/src/lib.rs index 3d3e4cf14..a641a849a 100644 --- a/crates/tracedecay-global-db/src/lib.rs +++ b/crates/tracedecay-global-db/src/lib.rs @@ -33,6 +33,9 @@ pub mod observation; mod observation_adapter; mod observation_projection; mod registered_maintenance; +pub use registered_maintenance::{ + REGISTERED_WAL_RECLAIM_TRIGGER_BYTES, RegisteredWalCheckpointReceiptV1, RegisteredWalReclaimV1, +}; mod registered_provider_usage; #[cfg(test)] mod stack_delivery_tests; diff --git a/crates/tracedecay-global-db/src/registered.rs b/crates/tracedecay-global-db/src/registered.rs index b0d2b025a..cd651be29 100644 --- a/crates/tracedecay-global-db/src/registered.rs +++ b/crates/tracedecay-global-db/src/registered.rs @@ -532,6 +532,24 @@ impl RegisteredGlobalDb { self.database.checkpoint().await } + /// The write-authority role retained by this client's guarded database. + /// WAL file truncation is authorized by the runtime only for the + /// exclusive maintenance role. + pub(crate) fn write_authority_role( + &self, + ) -> tracedecay_runtime_core::errors::Result + { + Ok(self.database.write_authority()?.role()) + } + + /// Truncates the drained WAL file through the runtime's exclusive + /// maintenance facade. + pub(crate) async fn truncate_database_wal( + &self, + ) -> tracedecay_runtime_core::errors::Result<()> { + self.database.truncate_wal_for_offline_maintenance().await + } + fn from_database(database: Database) -> Self { Self { database, @@ -540,6 +558,18 @@ impl RegisteredGlobalDb { } } + /// Wraps an already-published guarded database for WAL maintenance tests. + /// + /// The exclusive-maintenance truncation lane cannot be exercised through + /// the ordinary registered harness because the registered fixture + /// publisher mints only Test-role authority; this constructor lets a test + /// drive [`RegisteredGlobalDb::checkpoint_result`] over a + /// maintenance-scoped publication without bypassing the database facade. + #[cfg(test)] + pub(crate) fn from_database_for_wal_maintenance_test(database: Database) -> Self { + Self::from_database(database) + } + pub fn read_connection(&self) -> DatabaseEngineReadConnection { self.database.read_connection() } diff --git a/crates/tracedecay-global-db/src/registered_maintenance.rs b/crates/tracedecay-global-db/src/registered_maintenance.rs index 2fe934bd7..ca4a81c91 100644 --- a/crates/tracedecay-global-db/src/registered_maintenance.rs +++ b/crates/tracedecay-global-db/src/registered_maintenance.rs @@ -1,11 +1,94 @@ +use std::path::{Path, PathBuf}; + +use tracedecay_runtime_core::db::DatabaseAuthorityRole; use tracedecay_runtime_core::errors::TraceDecayError; use crate::RegisteredGlobalDb; +/// WAL file size at or above which a completed checkpoint escalates to file +/// truncation (Plan 38 §6 storage reclaim). +/// +/// SQLite's passive checkpoint lane backfills WAL frames into the main +/// database but never shrinks the `-wal` file, so a store that once ballooned +/// keeps its high-water file size forever. This bound matches the retained +/// writer's own soft checkpoint budget: below it the runtime deliberately +/// tolerates the warm WAL (a fresh schema install alone leaves several +/// mebibytes), so truncation there would only churn the file. A file at or +/// above it survived checkpoint pressure and is high-water debris worth +/// reclaiming. +pub const REGISTERED_WAL_RECLAIM_TRIGGER_BYTES: u64 = 32 * 1024 * 1024; + +/// Measured outcome of one registered WAL checkpoint/compaction pass +/// (Plan 38 §6). +/// +/// Byte figures are file-level measurements of the store's `-wal` sidecar, +/// matching Plan 38's rule that published size evidence stays file- and +/// directory-level. A missing sidecar measures zero bytes. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct RegisteredWalCheckpointReceiptV1 { + pub wal_bytes_before: u64, + pub wal_bytes_after: u64, + pub reclaim: RegisteredWalReclaimV1, +} + +/// How the pass disposed of the WAL file after the checkpoint drained. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum RegisteredWalReclaimV1 { + /// The WAL file was below [`REGISTERED_WAL_RECLAIM_TRIGGER_BYTES`], so + /// only the runtime checkpoint lane ran; no file reclaim was warranted. + BelowTrigger { trigger_bytes: u64 }, + /// The drained WAL file was truncated under this client's exclusive + /// maintenance authority. + Truncated { trigger_bytes: u64 }, + /// The checkpoint drained, but the file keeps its high-water size: the + /// runtime authorizes WAL file truncation only under the exclusive + /// maintenance role, which this client (for example the live daemon) + /// does not hold. Reclaim happens on the next maintenance-scoped pass + /// over the same store. + RequiresExclusiveMaintenance { trigger_bytes: u64 }, +} + impl RegisteredGlobalDb { - /// Checkpoints the registered store's WAL through its authorized writer. - pub async fn checkpoint_result(&self) -> Result<(), TraceDecayError> { - self.checkpoint_database().await + /// Runs one WAL checkpoint/compaction pass through this store's + /// authorized writer and reports the measured result (Plan 38 §6). + /// + /// The checkpoint itself goes through the retained runtime's bounded + /// checkpoint lane, so a WAL pinned by a live reader under size pressure + /// surfaces as a typed error instead of a vacuous success. After the + /// drain, a WAL file at or above [`REGISTERED_WAL_RECLAIM_TRIGGER_BYTES`] + /// is truncated when this client holds the exclusive maintenance + /// authority; otherwise the receipt records that reclaim is deferred to a + /// maintenance-scoped pass. + pub async fn checkpoint_result( + &self, + ) -> Result { + let wal_path = registered_wal_path(self.db_path()); + let wal_bytes_before = wal_file_bytes(&wal_path)?; + self.checkpoint_database().await?; + // A successful checkpoint proves the writable scope, so the role read + // cannot race a mode downgrade. + let reclaim = match wal_reclaim_plan(wal_bytes_before, self.write_authority_role()?) { + WalReclaimPlan::BelowTrigger => RegisteredWalReclaimV1::BelowTrigger { + trigger_bytes: REGISTERED_WAL_RECLAIM_TRIGGER_BYTES, + }, + WalReclaimPlan::Truncate => { + self.truncate_database_wal().await?; + RegisteredWalReclaimV1::Truncated { + trigger_bytes: REGISTERED_WAL_RECLAIM_TRIGGER_BYTES, + } + } + WalReclaimPlan::RequiresExclusiveMaintenance => { + RegisteredWalReclaimV1::RequiresExclusiveMaintenance { + trigger_bytes: REGISTERED_WAL_RECLAIM_TRIGGER_BYTES, + } + } + }; + let wal_bytes_after = wal_file_bytes(&wal_path)?; + Ok(RegisteredWalCheckpointReceiptV1 { + wal_bytes_before, + wal_bytes_after, + reclaim, + }) } pub async fn checkpoint(&self) { @@ -30,3 +113,84 @@ impl RegisteredGlobalDb { // Restore it here once `retention` + `config::RetentionConfig` land below // the composition root. } + +/// How one pass disposes of the WAL file, decided purely from the measured +/// size and the client's write-authority role. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum WalReclaimPlan { + BelowTrigger, + Truncate, + RequiresExclusiveMaintenance, +} + +const fn wal_reclaim_plan(wal_bytes_before: u64, role: DatabaseAuthorityRole) -> WalReclaimPlan { + if wal_bytes_before < REGISTERED_WAL_RECLAIM_TRIGGER_BYTES { + WalReclaimPlan::BelowTrigger + } else if matches!(role, DatabaseAuthorityRole::Maintenance) { + WalReclaimPlan::Truncate + } else { + WalReclaimPlan::RequiresExclusiveMaintenance + } +} + +/// The SQLite WAL sidecar for a database path: the full database file name +/// with `-wal` appended. +fn registered_wal_path(database_path: &Path) -> PathBuf { + let mut wal = database_path.as_os_str().to_owned(); + wal.push("-wal"); + PathBuf::from(wal) +} + +/// File-level size of the WAL sidecar. A missing sidecar is the typed +/// "no WAL exists" state and measures zero; any other filesystem failure +/// propagates instead of degrading to a fabricated figure. +fn wal_file_bytes(wal_path: &Path) -> Result { + match std::fs::metadata(wal_path) { + Ok(metadata) => Ok(metadata.len()), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(0), + Err(error) => Err(TraceDecayError::Database { + message: format!( + "failed to measure WAL file '{}': {error}", + wal_path.display() + ), + operation: "measure registered WAL file".to_owned(), + }), + } +} + +#[cfg(test)] +mod tests { + use super::{REGISTERED_WAL_RECLAIM_TRIGGER_BYTES, WalReclaimPlan, wal_reclaim_plan}; + use tracedecay_runtime_core::db::DatabaseAuthorityRole; + + #[test] + fn wal_below_trigger_is_left_alone_for_every_role() { + for role in [ + DatabaseAuthorityRole::Daemon, + DatabaseAuthorityRole::Maintenance, + DatabaseAuthorityRole::Test, + ] { + assert_eq!( + wal_reclaim_plan(REGISTERED_WAL_RECLAIM_TRIGGER_BYTES - 1, role), + WalReclaimPlan::BelowTrigger + ); + } + } + + #[test] + fn triggered_wal_truncates_only_under_exclusive_maintenance() { + assert_eq!( + wal_reclaim_plan( + REGISTERED_WAL_RECLAIM_TRIGGER_BYTES, + DatabaseAuthorityRole::Maintenance + ), + WalReclaimPlan::Truncate + ); + for role in [DatabaseAuthorityRole::Daemon, DatabaseAuthorityRole::Test] { + assert_eq!( + wal_reclaim_plan(REGISTERED_WAL_RECLAIM_TRIGGER_BYTES, role), + WalReclaimPlan::RequiresExclusiveMaintenance + ); + } + } +}