From d0963a700a1303bc8d3d331d6b6a22e2d6ab5052 Mon Sep 17 00:00:00 2001 From: James Ross Date: Wed, 29 Jul 2026 19:16:59 -0700 Subject: [PATCH 001/111] Docs: Specify the version two retention store --- CHANGELOG.md | 5 + docs/formats/README.md | 3 +- docs/formats/segment-store-v2/README.md | 67 ++++ docs/formats/segment-store-v2/gc.md | 193 +++++++++++ .../segment-store-v2/migration-crash.md | 81 +++++ docs/formats/segment-store-v2/rationale.md | 88 ++++++ docs/formats/segment-store-v2/recovery.md | 285 +++++++++++++++++ docs/formats/segment-store-v2/requirements.md | 63 ++++ docs/formats/segment-store-v2/retention.md | 299 ++++++++++++++++++ .../retention_store_v2_protocol_contract.rs | 228 +++++++++++++ 10 files changed, 1311 insertions(+), 1 deletion(-) create mode 100644 docs/formats/segment-store-v2/README.md create mode 100644 docs/formats/segment-store-v2/gc.md create mode 100644 docs/formats/segment-store-v2/migration-crash.md create mode 100644 docs/formats/segment-store-v2/rationale.md create mode 100644 docs/formats/segment-store-v2/recovery.md create mode 100644 docs/formats/segment-store-v2/requirements.md create mode 100644 docs/formats/segment-store-v2/retention.md create mode 100644 xtask/tests/retention_store_v2_protocol_contract.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index 48c1454..0da89d9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -303,6 +303,11 @@ after its public API and format compatibility policies are established. ### Added +- Specified `keep.segment-store/v2` retention values, root generations, + liveness manifests, reader snapshots, one-way staged migration, exact crash + boundaries, and reserved GC/disposition records. Version-1 immutable bytes + remain authoritative; production version-2 writing remains unavailable until + issue #19's executable evidence is complete. - Accepted ADR-0009 defines caller-supplied retention namespaces, `BlobId`/`LayoutId` reconstruction anchors, fail-closed canonical closure, generation-checked retention publication, immutable liveness snapshots, diff --git a/docs/formats/README.md b/docs/formats/README.md index 64db302..6b8ce93 100644 --- a/docs/formats/README.md +++ b/docs/formats/README.md @@ -8,7 +8,8 @@ admitted merely because one Rust type can serialize and deserialize it. | Format | Coordinate | Status | Evidence | | --- | --- | --- | --- | | [Flat Chunk Layout v1](flat-chunk-layout-v1/README.md) | `keep.flat-chunks/v1` | Implemented through verified reconstruction in issues #10 and #13 | [Golden corpus](../../conformance/layout/v1/README.md) | -| [Durable Segment Store v1](segment-store-v1/README.md) | `keep.segment-store/v1` | Specified in issue #14; segment I/O implemented in issue #15; publication and recovery remain in issues #16–#17 | [Golden corpus](../../conformance/segment-store/v1/README.md) | +| [Durable Segment Store v1](segment-store-v1/README.md) | `keep.segment-store/v1` | Implemented through initialization, publication, restart, and recovery in issues #14–#17 | [Golden corpus](../../conformance/segment-store/v1/README.md) | +| [Durable Segment Store v2](segment-store-v2/README.md) | `keep.segment-store/v2` | Retention transition implementation and executable evidence planned in issue #19 | Golden corpus planned in issue #19 | The registry records protocol specifications, including formats whose implementation is still planned. Each format page states its exact proof diff --git a/docs/formats/segment-store-v2/README.md b/docs/formats/segment-store-v2/README.md new file mode 100644 index 0000000..c908aab --- /dev/null +++ b/docs/formats/segment-store-v2/README.md @@ -0,0 +1,67 @@ +# Durable Segment Store Version 2 + +`keep.segment-store/v2` is the accepted successor to +`keep.segment-store/v1`. It preserves every admitted version-1 segment, +catalog, and publication-head byte while adding explicit retention state, +reader fences, migration evidence, and reserved GC and recovery-disposition +namespaces. + +ADR-0009 owns the cross-cutting retention and liveness decision. These pages +own its durable representation. Issue #19 must supply the production retention +implementation and executable evidence before any version-2 writer is +available. Until that implementation lands, version 1 remains the only +admitted production store. + +## Core laws + +Version 2 retains every version-1 physical law and adds these: + +- the format version is explicit and cannot be inferred from path existence; +- a complete version-2 store is entered only by the specified version-1 + migration; direct version-2 initialization is undefined; +- migration is one-way, writer-authorized, durable, and recoverable from every + documented prefix; +- retention authority exists only through one verified retention head and its + complete immutable manifest; +- each manifest binds every admitted namespace to one exact root generation + and canonical digest; +- root closure is derived from a verified catalog, never from paths, caller + claims, recent access, or application identity; +- catalog publication preserves every current retained closure before + replacing the catalog head; +- readers acquire the version-2 reader fence before opening the catalog head; + and +- ambiguous, corrupt, missing, excessive, or unsupported evidence refuses + before mutation. + +## Normative pages + +The following pages form one protocol: + +- [Retention records and publication](retention.md) owns canonical namespace, + root-generation, manifest, retention-head, closure, and transition rules. +- [GC and disposition records](gc.md) owns the canonical planned intent, + completion, and recovery-disposition byte grammars. +- [Migration and recovery](recovery.md) owns the exact root namespace, + version marker, reader fence, one-way migration, crash states, GC reservation, + recovery-disposition reservation, and restart behavior. +- [Migration crash points](migration-crash.md) owns fixed-stage publication and + the exact process-death boundaries for migration. +- [Requirements and evidence](requirements.md) owns stable requirement and + crash identifiers, evidence status, compatibility, and nonclaims. +- [Format rationale](rationale.md) records format-local choices and rejected + alternatives. + +The version-1 [segment](../segment-store-v1/segment.md), +[catalog](../segment-store-v1/catalog.md), and +[publication-head](../segment-store-v1/catalog.md#publication-head) +grammars remain byte-for-byte authoritative. Version 2 does not reinterpret or +re-encode them. + +## Status + +The format contract is frozen by ADR-0009 and this specification. Requirements +marked **Planned in #19** or **Planned in #21** are not implementation evidence. +A store must refuse version-2 state until the relevant parser, corruption, +golden-format, model-based, crash-injection, recovery, and fuzz evidence is +implemented. diff --git a/docs/formats/segment-store-v2/gc.md b/docs/formats/segment-store-v2/gc.md new file mode 100644 index 0000000..708afab --- /dev/null +++ b/docs/formats/segment-store-v2/gc.md @@ -0,0 +1,193 @@ +# GC and Disposition Records + +This page owns the canonical planned `GcRetirementIntent`, +`GcRetirementReceipt`, and `RecoveryDispositionReceipt` byte grammars for +`keep.segment-store/v2`. + +Issue #21 owns their implementation. They are specified now so version 2 has +one exact root grammar, but their presence remains unsupported mandatory state +until every **Planned in #21** requirement becomes executable evidence. + +## Common rules + +All integers are unsigned and big-endian. Flags and reserved bytes are zero. +Every length and count is checked before allocation. Decoders reject truncation, +trailing bytes, unsupported versions, unknown mandatory flags, nonzero reserved +bytes, overflow, noncanonical ordering, duplicates, digest or checksum +mismatch, and values above fixed ceilings. + +Every digest and checksum uses domain-separated BLAKE3-256. Fixed names are +never replaced to obtain idempotence. + +## GC retirement intent + +`GcRetirementIntent` consists of: + +```text +320-byte fixed-width header +candidate-count × 72-byte candidate entries +32-byte intent digest +32-byte checksum +``` + +The maximum candidate count is 65,536. Its maximum encoded length is +4,718,976 bytes. + + + +| Offset | Width | Field | Canonical value | +| ---: | ---: | --- | --- | +| 0 | 16 | magic | `KEEP:GC:INTENT2\0` | +| 16 | 2 | version | `2` | +| 18 | 2 | header length | `320` | +| 20 | 4 | flags | `0` | +| 24 | 8 | total record length | derived exact length | +| 32 | 8 | GC generation | positive checked successor | +| 40 | 2 | candidate width | `72` | +| 42 | 2 | reserved | zero | +| 44 | 4 | candidate count | `1..=65,536` | +| 48 | 8 | liveness generation | exact current value | +| 56 | 32 | retention-manifest digest | exact current digest | +| 88 | 8 | catalog generation | exact successor value | +| 96 | 32 | catalog digest | names no candidate segment | +| 128 | 4 | realization-profile identity | exact retained profile | +| 132 | 4 | realization-profile version | exact retained profile | +| 136 | 32 | realization-profile digest | exact retained profile | +| 168 | 32 | catalog-successor proof digest | complete verified proof | +| 200 | 32 | segment-pool identity digest | exact admitted pool | +| 232 | 32 | disposition-set digest | exact admitted receipts | +| 264 | 8 | reader-lock device identity | exact locked file | +| 272 | 8 | reader-lock mount identity | exact locked file | +| 280 | 8 | reader-lock file identity | exact locked file | +| 288 | 32 | candidate-entry-set digest | exact canonical entries | + + + +Each 72-byte candidate entry is: + +| Offset | Width | Field | +| ---: | ---: | --- | +| 0 | 32 | segment digest | +| 32 | 8 | segment length | +| 40 | 32 | complete verification-evidence digest | + +Candidate entries use canonical segment-digest order and are duplicate-free. +The entry-set, intent, and checksum domains are: + +```text +keep.gc-candidate-set/v2\0 +keep.gc-retirement-intent/v2\0 +keep.gc-retirement-intent-checksum/v2\0 +``` + +The checksum covers header, entries, and intent digest. The intent digest +covers the header and entries. + +## GC retirement receipt + +`GcRetirementReceipt` is exactly 320 bytes: + + + +| Offset | Width | Field | Canonical value | +| ---: | ---: | --- | --- | +| 0 | 16 | magic | `KEEP:GC:RECEIPT2` | +| 16 | 2 | version | `2` | +| 18 | 2 | record length | `320` | +| 20 | 4 | flags | `0` | +| 24 | 8 | GC generation | exact intent generation | +| 32 | 32 | intent digest | exact durable intent | +| 64 | 32 | retired candidate-set digest | exact intent set | +| 96 | 32 | post-retirement pool-state digest | verified synchronized pool | +| 128 | 8 | liveness generation | revalidated exact value | +| 136 | 32 | retention-manifest digest | revalidated exact value | +| 168 | 8 | catalog generation | revalidated exact value | +| 176 | 32 | catalog digest | revalidated exact value | +| 208 | 8 | reader-lock device identity | exact exclusive lock | +| 216 | 8 | reader-lock mount identity | exact exclusive lock | +| 224 | 8 | reader-lock file identity | exact exclusive lock | +| 232 | 8 | completed synchronization count | exact intent-derived count | +| 240 | 48 | reserved | zero | +| 288 | 32 | checksum | BLAKE3-256 over bytes `0..288` | + + + +The checksum domain is `keep.gc-retirement-receipt-checksum/v2\0`. + +## Recovery disposition receipt + +`RecoveryDispositionReceipt` is exactly 320 bytes: + + + +| Offset | Width | Field | Canonical value | +| ---: | ---: | --- | --- | +| 0 | 16 | magic | `KEEP:REC:DISP2\0\0` | +| 16 | 2 | version | `2` | +| 18 | 2 | record length | `320` | +| 20 | 4 | flags | `0` | +| 24 | 2 | artifact kind | registered enum | +| 26 | 2 | decision | finalize or retire | +| 28 | 2 | admitted recovery classification | registered enum | +| 30 | 2 | reserved | zero | +| 32 | 8 | artifact length | exact observed length | +| 40 | 32 | artifact identity digest | physical evidence identity | +| 72 | 32 | artifact content digest | exact verified bytes | +| 104 | 8 | publication-head generation | exact observed value | +| 112 | 32 | publication-head checksum | exact observed value | +| 144 | 8 | catalog generation | exact observed value | +| 152 | 32 | catalog digest | exact observed value | +| 184 | 8 | liveness generation | exact observed value | +| 192 | 32 | retention-manifest digest | exact observed value | +| 224 | 8 | reader-lock device identity | exact safety coordinate | +| 232 | 8 | reader-lock mount identity | exact safety coordinate | +| 240 | 8 | reader-lock file identity | exact safety coordinate | +| 248 | 32 | decision-evidence digest | complete canonical proof | +| 280 | 8 | reserved | zero | +| 288 | 32 | checksum | BLAKE3-256 over bytes `0..288` | + + + +The checksum domain is `keep.recovery-disposition-receipt-checksum/v2\0`. +Unknown artifact kinds, decisions, or classifications refuse. + +The pool coordinate is: + +```text +recovery/dispositions/.receipt +``` + +The version-2 maximum is 65,536 disposition receipts. A future successor must +migrate the namespace before raising the ceiling. + +## State and recovery + +GC admits these states: + + + +| State | Evidence | Recovery | +| --- | --- | --- | +| idle | no `gc/intent` or `gc/receipt` | no retirement authority | +| active | exact intent, every candidate present | begin execution | +| partial | exact intent, one canonical absent candidate prefix | continue at first present candidate | +| completion pending | exact intent, every candidate absent | publish receipt | +| receipt transition | exact intent and exact receipt | synchronize receipt, remove intent, synchronize `gc` | +| complete | exact receipt only | return exact completion | + + + +An absent candidate outside the canonical absent candidate prefix, substituted +candidate, changed pool, stale coordinate, conflicting receipt, malformed +record, or unexplained absence is unrecoverable ambiguity. Recovery never +guesses which deletion occurred. + +A disposition transition writes and synchronizes +`recovery/disposition.next`, verifies and links the immutable receipt without +replacement, synchronizes `recovery/dispositions`, removes the stage, and +synchronizes `recovery`. Until that completes, the artifact remains +recovery-protected. + +These grammars, their golden fixtures, parsers, corruption matrices, crash +points, model, benchmarks, and fuzz targets are **Planned in #21**. Issue #19 +must refuse their physical presence without mutating it. diff --git a/docs/formats/segment-store-v2/migration-crash.md b/docs/formats/segment-store-v2/migration-crash.md new file mode 100644 index 0000000..c5d7041 --- /dev/null +++ b/docs/formats/segment-store-v2/migration-crash.md @@ -0,0 +1,81 @@ +# Migration Crash Points + +This page owns fixed-record publication and process-death boundaries for the +one-way `keep.segment-store/v1` to `keep.segment-store/v2` migration. + +## Fixed-stage law + +Migration never writes canonical fixed names in place: + +| Stage | Canonical target | +| --- | --- | +| `migration.intent.next` | `migration.intent` | +| `FORMAT.next` | `FORMAT` | +| `migration.receipt.next` | `migration.receipt` | + +For each pair, migration: + +1. creates the stage exclusively as a pinned regular file; +2. writes bounded complete bytes, synchronizes, reopens, and verifies them; +3. links the stage to the canonical target without replacement; +4. synchronizes the store root; +5. removes the retained stage; and +6. synchronizes the store root again. + +The verified stage is linked without replacement. The canonical target is +immutable. Recovery never truncates, replaces, or repairs it. An exact stage +with an absent target resumes at the link. Exact stage and target bytes resume +at the required synchronization or cleanup. Different bytes, a substituted +inode, a link, or a wrong file kind refuse. + +A pre-effect incomplete stage may be removed only when its canonical target and +every later-ordered migration effect are absent and every earlier effect admits +exactly. Recovery pins the stage, removes it, synchronizes the store root, and +returns a typed discard report. Any later effect makes incomplete or corrupt +stage bytes unrecoverable ambiguity. + +The fixed stage is not authority. `migration.intent` becomes migration +authority only after its canonical link and store-root synchronization. +`migration.receipt` becomes completion evidence at the equivalent boundary. + +## Namespace prefix + +After durable intent publication, migration creates persistent `reader.lock` +and the exact nested directory prefix in the order specified by +[migration recovery](recovery.md). Each existing name must be the exact pinned +file or directory expected at that position. Each new nested name is followed +by synchronization of its parent. The final store-root synchronization admits +the complete prefix. A wrong kind, link, out-of-order name, or unknown entry +refuses. + +## Process-death matrix + +| Identifier | Boundary | +| --- | --- | +| `KEEP-CRASH-053` | migration-intent stage write | +| `KEEP-CRASH-054` | migration-intent stage synchronization | +| `KEEP-CRASH-055` | migration-intent canonical link | +| `KEEP-CRASH-056` | store-root synchronization after intent link | +| `KEEP-CRASH-057` | migration-intent stage removal | +| `KEEP-CRASH-058` | store-root synchronization after intent cleanup | +| `KEEP-CRASH-059` | persistent reader-fence creation | +| `KEEP-CRASH-060` | canonical nested directory-prefix creation | +| `KEEP-CRASH-061` | store-root synchronization after namespace creation | +| `KEEP-CRASH-062` | format-marker stage write | +| `KEEP-CRASH-063` | format-marker stage synchronization | +| `KEEP-CRASH-064` | format-marker canonical link | +| `KEEP-CRASH-065` | store-root synchronization after marker link | +| `KEEP-CRASH-066` | format-marker stage removal | +| `KEEP-CRASH-067` | store-root synchronization after marker cleanup | +| `KEEP-CRASH-068` | migration-receipt stage write | +| `KEEP-CRASH-069` | migration-receipt stage synchronization | +| `KEEP-CRASH-070` | migration-receipt canonical link | +| `KEEP-CRASH-071` | store-root synchronization after receipt link | +| `KEEP-CRASH-072` | migration-receipt stage removal | +| `KEEP-CRASH-073` | final store-root synchronization | + +Every identifier requires before, during, and after process-death evidence. +`KEEP-CRASH-060` additionally requires one case for every admitted directory +prefix length. Restart must classify exact stages, canonical targets, namespace +prefix, marker, receipt, and cleanup state without depending on a clock, +filesystem iteration order, or file existence alone. diff --git a/docs/formats/segment-store-v2/rationale.md b/docs/formats/segment-store-v2/rationale.md new file mode 100644 index 0000000..315bee8 --- /dev/null +++ b/docs/formats/segment-store-v2/rationale.md @@ -0,0 +1,88 @@ +# Format Rationale + +This note records choices local to `keep.segment-store/v2`. ADR-0009 remains +authoritative for the cross-cutting retention and GC decision. + +## Use a successor store version + +Extending the version-1 root shape was rejected. Version 1 deliberately refuses +unknown entries, so treating new retention state as optional would weaken its +admission law and make old readers misclassify a mutated store. A durable +migration intent creates an explicit authority boundary. + +Direct version-2 initialization was rejected for this version. Requiring one +admitted version-1 predecessor gives migration, compatibility, and recovery one +starting law instead of defining a second initialization protocol without a +consumer requirement. + +## Stage fixed migration records + +Writing `migration.intent`, `FORMAT`, or `migration.receipt` in place was +rejected because process death can expose partial canonical bytes. Exact +`.next` stages make incomplete bytes non-authoritative and publish each +canonical fixed record through an immutable no-replacement link. + +## Preserve version-1 immutable bytes + +Re-encoding segments, catalogs, or publication heads during migration was +rejected. Their bytes are already canonical and independently evidenced. +Preservation narrows migration to new authority and namespace state and permits +byte-for-byte rollback analysis without promising an automatic downgrade. + +## Keep namespace bytes out of paths + +Using caller namespace text as a directory name was rejected. Namespace bytes +may contain separators, zero bytes, or non-Unicode data and have no filesystem +semantics. A domain-separated digest supplies the physical coordinate while +the root record retains the exact bytes to detect collision or substitution. + +## Retain empty namespace generations + +Deleting empty namespaces was rejected because an old absent-state compare and +swap could become valid again. Persistent empty generations prevent that ABA +hazard. The fixed 4,096-namespace ceiling bounds the resulting manifest and +makes capacity refusal explicit. + +## Use one global manifest + +Enumerating mutable namespace directories during GC was rejected. One immutable +manifest binds the complete namespace map under a `LivenessGeneration`, so a +reader or GC planner cannot miss a concurrently created namespace. + +## Store semantic records, not serializer output + +Serde-defined persistence was rejected. Fixed headers, explicit widths, +big-endian integers, zero reserved bytes, canonical ordering, named digest +domains, and golden fixtures keep the protocol independent of Rust layout and +dependency defaults. + +## Start with one realization profile + +Version-2 catalogs expose one canonical location per logical record identity. +Pretending to support multiple representation policies would add an unproved +abstraction. The registered single-witness profile states the current law +exactly; another profile requires a successor specification and evidence. + +## Use a kernel reader fence + +A durable reader registry, lease, clock, and process liveness inference were +rejected. A persistent file with shared reader locks and an exclusive GC lock +has an observable process-death lifecycle. The fixed writer-then-reader lock +order avoids lock inversion. Publication does not take the reader lock because +it deletes no published immutable segment. Readers therefore double-collect +both mutable heads around transitive admission and reject a mixed view. + +## Derive logical store identity + +Random or physical-location store identifiers were rejected. A deterministic +digest of the admitted version-1 catalog, immutable pools, and target format +definition gives byte-identical stores one logical identity while the migration +intent separately binds physical coordinates for in-place recovery. + +## Reserve GC names but refuse their state + +Leaving the future GC namespace undefined was rejected because adding it later +would mutate the exact version-2 root grammar. Accepting placeholder bytes was +also rejected. Version 2 reserves the names, while their presence remains an +unsupported mandatory state until issue #21 supplies complete byte, parser, +crash, recovery, corruption, and fuzz evidence. diff --git a/docs/formats/segment-store-v2/recovery.md b/docs/formats/segment-store-v2/recovery.md new file mode 100644 index 0000000..871e675 --- /dev/null +++ b/docs/formats/segment-store-v2/recovery.md @@ -0,0 +1,285 @@ +# Migration and Recovery + +This page owns the version-2 filesystem namespace, format marker, reader fence, +one-way migration, fixed-stage recovery, GC reservation, and +recovery-disposition reservation. + +## Exact filesystem namespace + +Version 2 preserves the version-1 files and directories and admits these new +coordinates: + +```text +reader.lock +FORMAT +migration.intent +migration.intent.next +migration.receipt +migration.receipt.next +FORMAT.next +retention/HEAD +retention/head.next +retention/root.next +retention/manifest.next +retention/roots//-.root +retention/manifests/-.manifest +gc/intent +gc/receipt +recovery/disposition.next +recovery/dispositions/.receipt +``` + +`retention/HEAD`, every fixed `.next` stage, `gc/intent`, and `gc/receipt` are +optional according to the exact state tables below. Immutable-pool coordinates +are data-dependent but canonically named. Every other root or +protocol-directory entry is an unknown entry and unrecoverable ambiguity. +Operations are capability-relative and never follow links. + +## Format marker + +`FORMAT` is exactly 96 bytes: + +| Offset | Width | Field | Canonical value | +| ---: | ---: | --- | --- | +| 0 | 16 | magic | `KEEP:STORE:V2\0\0\0` | +| 16 | 2 | version | `2` | +| 18 | 2 | record length | `96` | +| 20 | 4 | flags | `0` | +| 24 | 32 | format-definition digest | registered v2 digest | +| 56 | 4 | maximum namespace count | `4,096` | +| 60 | 4 | reserved | zero | +| 64 | 32 | checksum | BLAKE3-256 over bytes `0..64` | + +The definition and checksum domains are +`keep.segment-store-definition/v2\0` and +`keep.segment-store-marker-checksum/v2\0`. A missing marker is version 1 only +when the exact version-1 namespace admits. An unsupported, corrupt, +substituted, or same-name/different-digest marker refuses. + +## Reader fence + +`reader.lock` is a persistent regular zero-length file. Its contents and +existence alone prove nothing. + +A version-2 reader acquires a kernel-managed shared lock on `reader.lock` +before opening catalog `HEAD` or `retention/HEAD`. The returned `ReaderFence` +owns that lock for the complete snapshot lifetime. Close, drop, or process +death releases only the kernel lock and never deletes the persistent file. + +GC acquires the store writer authority and then an exclusive `reader.lock`, in +that fixed order. New readers wait and existing readers drain before GC +revalidation or physical deletion. Catalog and retention publication may +proceed beside readers because they publish immutable successors and delete no +published segment. + +## Migration records + +Migration is a one-way explicit migration under exclusive writer authority. +Version 1 is never extended in place without durable migration evidence. + +`migration.intent` is exactly 256 bytes: + + + +| Offset | Width | Field | Canonical value | +| ---: | ---: | --- | --- | +| 0 | 16 | magic | `KEEP:MIG:INT2\0\0\0` | +| 16 | 2 | version | `2` | +| 18 | 2 | record length | `256` | +| 20 | 4 | flags | `0` | +| 24 | 8 | catalog generation named by version-1 `HEAD` | positive | +| 32 | 8 | catalog length named by version-1 `HEAD` | exact admitted length | +| 40 | 32 | catalog digest named by version-1 `HEAD` | exact admitted digest | +| 72 | 32 | predecessor catalog digest | zero for generation 1 | +| 104 | 32 | immutable-pool inventory digest | canonical complete inventory | +| 136 | 8 | root device identity | admitted platform value | +| 144 | 8 | root mount identity | admitted platform value | +| 152 | 8 | root file identity | admitted platform value | +| 160 | 32 | target format-definition digest | exact registered v2 digest | +| 192 | 32 | new store identifier | deterministic derivation below | +| 224 | 32 | checksum | BLAKE3-256 over bytes `0..224` | + + + +The checksum domain is `keep.store-migration-intent-checksum/v2\0`. The pool +inventory digest uses `keep.store-v1-pool-inventory/v2\0` over the sorted, +duplicate-free canonical names, lengths, and verified content digests from +both immutable pools. The intent therefore binds the exact catalog generation, +length, and digest named by the admitted version-1 `HEAD`. + +The deterministically derived store identifier is: + +```text +BLAKE3-256("keep.store-identifier/v2\0" || + catalog-generation-u64 || + catalog-length-u64 || + catalog-digest || + predecessor-catalog-digest || + immutable-pool-inventory-digest || + target-format-definition-digest) +``` + +Integer fields use their fixed-width big-endian bytes. Root device, mount, file +identity, caller identity, path, and time do not enter the identifier. The +migration intent separately binds the physical root coordinates so in-place +recovery refuses a substituted store. + +`migration.receipt` is exactly 256 bytes: + + + +| Offset | Width | Field | Canonical value | +| ---: | ---: | --- | --- | +| 0 | 16 | magic | `KEEP:MIG:REC2\0\0\0` | +| 16 | 2 | version | `2` | +| 18 | 2 | record length | `256` | +| 20 | 4 | flags | `0` | +| 24 | 32 | migration-intent digest | exact durable intent | +| 56 | 32 | store identifier | exact intent value | +| 88 | 32 | format-marker digest | exact verified marker | +| 120 | 32 | initial retention-state digest | exact no-payload digest below | +| 152 | 32 | initial GC-state digest | exact no-payload digest below | +| 184 | 32 | disposition namespace digest | exact no-payload digest below | +| 216 | 8 | completed synchronization mask | every mandatory bit set | +| 224 | 32 | checksum | BLAKE3-256 over bytes `0..224` | + + + +Its checksum domain is `keep.store-migration-receipt-checksum/v2\0`. Unknown +synchronization bits, a missing mandatory bit, or any mismatch with the intent +refuses. + +The three initial-state fields are the no-payload digests +`BLAKE3-256("keep.initial-retention-state/v2\0")`, +`BLAKE3-256("keep.initial-gc-state/v2\0")`, and +`BLAKE3-256("keep.empty-disposition-set/v2\0")`. At completed migration, +absence of `retention/HEAD` is the canonical empty retention state only while +all retention stages and pools are empty. Any retention artifact routes through +recovery instead. Direct version-2 initialization is undefined. + +The byte-exact offset tables and golden fixtures are requirements +`KEEP-MIGRATION-002` and `KEEP-MIGRATION-007`; no production writer exists +until those planned items become implemented evidence. + +## One-way migration protocol + +Migration performs these ordered steps: + +1. Admit and completely recover the exact version-1 store. +2. Revalidate its head, catalog, pools, root identity, and writer authority. +3. Publish `migration.intent` from `migration.intent.next` through the + no-replacement fixed-stage protocol. +4. Create and verify persistent `reader.lock`. +5. Create the exact `retention`, `retention/roots`, + `retention/manifests`, `gc`, `recovery`, and + `recovery/dispositions` directories. +6. Synchronize every created parent and the store root. +7. Publish `FORMAT` from `FORMAT.next` through the fixed-stage protocol. +8. Reopen and verify the complete version-2 view. +9. Publish `migration.receipt` from `migration.receipt.next` through the + fixed-stage protocol. + +The [migration crash-point specification](migration-crash.md) owns that +protocol and spans `KEEP-CRASH-053` through `KEEP-CRASH-073`. + +Migration never rewrites or deletes admitted version-1 immutable bytes and +provides no automatic downgrade. + +Version-1 admission refuses once any migration stage, `migration.intent`, +`reader.lock`, `FORMAT`, or version-2 directory is present. Once the canonical +intent is durable, only version-2 migration recovery may continue. + +## Partial migration recovery + +The migration recovery boundary admits only these ordered prefixes: + + + +| State | Required response | +| --- | --- | +| no migration artifact | admit exact version 1 | +| intent stage only | finalize an exact stage or explicitly discard an incomplete pre-effect stage | +| durable intent only | verify intent and continue | +| intent plus a canonical prefix of v2 names | verify each name and continue | +| complete v2 shape without marker | verify directories and write marker | +| marker without receipt | reopen full v2 view and publish receipt | +| exact receipt with optional exact receipt stage | clean the stage and admit complete migration | + + + +A partial migration retry revalidates the intent and every existing byte, +continues idempotently at the first absent canonical step, and never replaces +an existing entry. A missing predecessor, changed version-1 coordinate, +out-of-order name, wrong file kind, substituted byte, conflicting receipt, +unknown entry, or changed root identity is unrecoverable ambiguity. + +Process death before durable canonical intent leaves version 1 plus at most its +non-authoritative stage. Process death after durable intent leaves +recovery-required version-2 migration state. + +## Retention publication recovery + +At restart, a fixed retention stage is classified from its exact framing and +transitive evidence: + +The forward protocol guarantees that `root.next` is durable before a new +namespace directory is created. A new digest-named directory is created +exclusively, verified as the exact regular directory rather than a link, and +followed by synchronization of `retention/roots` before the immutable root is +linked. An existing exact directory is idempotent; any wrong kind, substituted +namespace, or unexpected entry refuses. Directory existence alone never proves +a retained root. + + + +| Fixed stage | Complete evidence | Recovery | +| --- | --- | --- | +| `root.next` | canonical successor root, matching namespace and closure proof | finalize its immutable pool link and retain the stage | +| `manifest.next` | canonical successor manifest naming only admitted roots | finalize its immutable pool link and retain both stages | +| `head.next` | canonical successor head naming the staged manifest | finalize the head, synchronize it, then remove retained stages | + + + +A pre-effect incomplete stage may be removed only when every later-ordered +effect is absent and all earlier evidence admits exactly. Recovery pins that +regular file, removes it, synchronizes `retention`, and returns a typed discard +report. Any later effect, stale generation, mismatched digest, missing +transitive member, reappeared stage, conflicting pool entry, or other +corruption is a typed refusal. A complete valid orphan remains +recovery-protected until explicit disposition. + +The retention crash points are: + +| Identifier | Boundary | +| --- | --- | +| `KEEP-CRASH-036` | root stage write | +| `KEEP-CRASH-037` | root stage synchronization | +| `KEEP-CRASH-038` | new namespace-directory creation or exact admission | +| `KEEP-CRASH-039` | namespace-pool synchronization after creation | +| `KEEP-CRASH-040` | immutable root link | +| `KEEP-CRASH-041` | root namespace-directory synchronization | +| `KEEP-CRASH-042` | manifest stage write | +| `KEEP-CRASH-043` | manifest stage synchronization | +| `KEEP-CRASH-044` | immutable manifest link | +| `KEEP-CRASH-045` | manifest pool synchronization | +| `KEEP-CRASH-046` | retention-head stage write | +| `KEEP-CRASH-047` | retention-head stage synchronization | +| `KEEP-CRASH-048` | retention-head atomic replacement | +| `KEEP-CRASH-049` | committed retention namespace synchronization | +| `KEEP-CRASH-050` | retained root-stage removal | +| `KEEP-CRASH-051` | retained manifest-stage removal | +| `KEEP-CRASH-052` | retention cleanup synchronization | + +Each point requires before, during, and after process-death evidence. Restart +must establish exact catalog visibility, retention head, namespace generation, +orphan classification, stage disposition, and recovery report. + +## GC and recovery-disposition recovery + +The [GC and disposition record specification](gc.md) owns the exact +`GcRetirementIntent`, `GcRetirementReceipt`, and +`RecoveryDispositionReceipt` grammars and state transitions. Issue #21 owns +their executable parser, corruption, crash, recovery, and fuzz evidence. +Issue #19 admits only the absent `gc/intent`, `gc/receipt`, +`recovery/disposition.next`, and disposition-receipt pool. Any presence is +unsupported mandatory state and refuses without mutation. diff --git a/docs/formats/segment-store-v2/requirements.md b/docs/formats/segment-store-v2/requirements.md new file mode 100644 index 0000000..b011772 --- /dev/null +++ b/docs/formats/segment-store-v2/requirements.md @@ -0,0 +1,63 @@ +# Requirements and Evidence + +This ledger owns stable requirements for `keep.segment-store/v2`. A planned +case is not evidence. + +## Retention transitions + + + +| ID | Requirement | Evidence | Status | +| --- | --- | --- | --- | +| `KEEP-RETENTION-001` | `RetentionNamespace`, `RootGeneration`, `LivenessGeneration`, profile coordinates, limits, anchors, and digests are validated typed values | unit and public API tests | Planned in #19 | +| `KEEP-RETENTION-002` | Root, manifest, and head codecs implement the exact canonical grammars and fixed bounds | golden-format fixtures plus independent oracle | Planned in #19 | +| `KEEP-RETENTION-003` | Every structural field, truncation boundary, ordering law, duplicate, overflow, flag, reserved byte, digest, checksum, and trailing byte has a precise refusal | corruption matrix and fuzz | Planned in #19 | +| `KEEP-RETENTION-004` | Retain and release compare expected and observed generations and publish exact successors only | unit and model-based transition tests | Planned in #19 | +| `KEEP-RETENTION-005` | Closure derivation is deterministic, bounded, cycle-safe, fail-closed, and verifies complete blob reconstruction | property, corruption, and adversarial catalog tests | Planned in #19 | +| `KEEP-RETENTION-006` | Publication follows the exact ordered durability protocol, including new namespace-directory admission and retention of fixed-stage evidence until head commit, and returns only after cleanup synchronization | operation-order and `KEEP-CRASH-036..=052` crash-injection tests | Planned in #19 | +| `KEEP-RETENTION-007` | Restart resolves every fixed-stage crash prefix to one documented lawful state or typed ambiguity | debug and release crash matrix | Planned in #19 | +| `KEEP-RETENTION-008` | Readers double-collect catalog and retention heads and bind one complete catalog, manifest, and root-generation view under a `ReaderFence` | immutable snapshot and concurrency tests | Planned in #19 | +| `KEEP-RETENTION-009` | Exact already-committed retry is idempotent only while its successor remains current | retry and stale-successor tests | Planned in #19 | +| `KEEP-RETENTION-010` | Model operation sequences agree with a deterministic namespace-to-anchor-set map and never admit caller identity, paths, clocks, or application policy | model-based and source-architecture tests | Planned in #19 | + + + +## Migration + + + +| ID | Requirement | Evidence | Status | +| --- | --- | --- | --- | +| `KEEP-MIGRATION-001` | Exact version-1 stores remain admitted until a durable migration artifact exists | compatibility fixtures | Planned in #19 | +| `KEEP-MIGRATION-002` | Intent and receipt have complete fixed byte tables, named domains, bounds, checksums, deterministic store identity, and exact initial-state digests | golden-format fixtures | Planned in #19 | +| `KEEP-MIGRATION-003` | Migration revalidates version-1 head, catalog, pools, root identity, and writer authority before mutation | capability-relative integration tests | Planned in #19 | +| `KEEP-MIGRATION-004` | Every partial migration prefix continues idempotently under writer authority | state-machine and recovery tests | Planned in #19 | +| `KEEP-MIGRATION-005` | Unknown, out-of-order, substituted, corrupt, conflicting, or changed evidence is unrecoverable ambiguity | corruption and mutation matrix | Planned in #19 | +| `KEEP-MIGRATION-006` | Migration never rewrites or deletes admitted version-1 immutable bytes | byte-for-byte before/after witness | Planned in #19 | +| `KEEP-MIGRATION-007` | Process death around every intent stage, canonical link, namespace prefix, marker stage, receipt stage, cleanup, and synchronization boundary reaches a documented lawful state | `KEEP-CRASH-053..=073` crash-injection matrix | Planned in #19 | +| `KEEP-MIGRATION-008` | Version-1 admission refuses every version-2 or partial-migration artifact after migration begins | compatibility and fuzz tests | Planned in #19 | + + + +## Garbage collection reservation + + + +| ID | Requirement | Evidence | Status | +| --- | --- | --- | --- | +| `KEEP-GC-001` | Version 2 specifies exact bounded GC intent, receipt, and recovery-disposition grammars but refuses their presence until their parser and recovery protocol are implemented | namespace admission tests | Planned in #21 | +| `KEEP-GC-002` | GC intent, receipt, disposition, reader-fence, retirement, compaction, and recovery laws implement ADR-0009 without changing logical identity | golden-format, model-based, corruption, crash-injection, benchmark, and fuzz evidence | Planned in #21 | + + + +## Compatibility and nonclaims + +- Version 2 preserves exact version-1 segment, catalog, and publication-head + bytes. +- Migration is one-way and provides no downgrade. +- Retention evidence proves a bounded physical reconstruction claim, not + application meaning, causal ownership, future policy, or secure erasure. +- A version-2 format specification is not proof that a version-2 production + writer exists. +- Benchmarks are required before performance-sensitive retention or migration + optimization. diff --git a/docs/formats/segment-store-v2/retention.md b/docs/formats/segment-store-v2/retention.md new file mode 100644 index 0000000..ca4811b --- /dev/null +++ b/docs/formats/segment-store-v2/retention.md @@ -0,0 +1,299 @@ +# Retention Records and Publication + +This page owns retention values, root-generation records, manifests, heads, +closure admission, and publication for `keep.segment-store/v2`. + +## Scalar and identity rules + +All integers are unsigned and big-endian. Reserved bytes and unassigned flags +are zero. Decoders reject unknown mandatory flags, nonzero reserved bytes, +truncation, trailing bytes, unsupported versions, noncanonical ordering, +duplicates, overflow, and values above a fixed limit. + +Checksums and durable digests use each record's named domain-separated +BLAKE3-256 profile, including the domain string's terminating zero byte. No +digest covers a serializer-owned value. + +### Retention namespace + +`RetentionNamespace` is an opaque, nonempty byte string of 1 through 255 bytes. +Every byte is admitted and canonical as-is; the value is not Unicode, a path, +an account, a process, or an application identity. No normalization, case +folding, alias, implicit namespace, or alternate encoding exists. + +The namespace digest is: + +```text +BLAKE3-256("keep.retention-namespace/v1\0" || + namespace-length-u16 || + namespace-bytes) +``` + +The 32-byte digest supplies the physical namespace-directory coordinate. The +root-generation record also stores the exact namespace bytes, so a digest +collision or substituted spelling refuses instead of aliasing two authorities. + +### Generations + +`RootGeneration` and `LivenessGeneration` are positive `u64` values. Generation +1 is initial. A successor is exactly the observed value plus one under checked +arithmetic. Zero and overflow refuse. An empty root set remains a new +`RootGeneration`; namespace identity and generation history are never deleted +or reused in version 2. + +The maximum admitted namespace count is 4,096, including current manifest +namespaces, empty generations, and recovery-protected orphan namespace +directories; directory existence alone is not authority. Admission computes +the attempted total with checked arithmetic and refuses above that maximum +before any namespace-generation or manifest bytes are staged. Existing +namespaces may transition while the store is at capacity. + +### Reconstruction anchor + +One anchor is exactly 119 bytes: + +| Offset | Width | Field | +| ---: | ---: | --- | +| 0 | 59 | canonical `BlobId` binary bytes | +| 59 | 60 | canonical `LayoutId` binary bytes | + +Anchors are ordered by the lexicographic order of their complete canonical +bytes. The set is sorted, duplicate-free before admission. The maximum +anchor count in one namespace generation is 65,536. + +### Realization profile and limits + +Version 2 admits one realization profile: + +- identity `1`; +- version `1`; +- canonical name `keep.retention-single-canonical-witness/v1`; +- exact witness count `1` for each layout and chunk identity; and +- selection by canonical physical catalog coordinate. + +The stored profile coordinate is the `u32` identity, `u32` version, and +BLAKE3-256 digest of its canonical definition bytes. Any unknown or mismatched +coordinate refuses. A future profile requires a successor specification. + +Each root generation stores caller-selected limits no greater than these +implementation ceilings: + +| Limit | Ceiling | +| --- | ---: | +| anchors | 65,536 | +| closure nodes | 1,048,576 | +| closure depth | 8 | +| encoded bytes inspected | 16,777,216 | +| physical bytes inspected | 1,073,741,824 | + +All limits are positive. Cross-field validation and the ceiling check complete +before traversal or materialization. + +## Root-generation record + +One root-generation file is: + +```text +192-byte fixed-width header +namespace bytes +anchor-count × 119-byte anchors +32-byte root digest +32-byte checksum +``` + +Its total maximum length is 7,799,295 bytes. + + + +| Offset | Width | Field | Canonical value | +| ---: | ---: | --- | --- | +| 0 | 16 | magic | `KEEP:RET:ROOT2\0\0` | +| 16 | 2 | version | `2` | +| 18 | 2 | header length | `192` | +| 20 | 4 | flags | `0` | +| 24 | 8 | total record length | derived exact length | +| 32 | 8 | root generation | positive | +| 40 | 2 | namespace length | `1..=255` | +| 42 | 2 | anchor width | `119` | +| 44 | 4 | anchor count | `0..=65,536` | +| 48 | 4 | profile identity | `1` | +| 52 | 4 | profile version | `1` | +| 56 | 32 | profile-definition digest | registered exact digest | +| 88 | 8 | closure-node limit | positive and at most ceiling | +| 96 | 2 | closure-depth limit | positive and at most ceiling | +| 98 | 2 | reserved | zero | +| 100 | 8 | encoded-byte limit | positive and at most ceiling | +| 108 | 8 | physical-byte limit | positive and at most ceiling | +| 116 | 32 | predecessor root digest | zero for generation 1 | +| 148 | 32 | anchor-set digest | exact body-anchor digest | +| 180 | 12 | reserved | zero | + + + +The anchor-set digest is: + +```text +BLAKE3-256("keep.retention-anchor-set/v2\0" || + anchor-count-u32 || + canonical-anchor-bytes) +``` + +The root digest covers the header and body: + +```text +BLAKE3-256("keep.retention-root/v2\0" || header || body) +``` + +The checksum covers the header, body, and root digest: + +```text +BLAKE3-256("keep.retention-root-checksum/v2\0" || + header || body || root-digest) +``` + +The pool coordinate is: + +```text +retention/roots// + -.root +``` + +Names with alternate width, case, suffix, generation, or digest refuse. + +## Global retention manifest + +One manifest binds every admitted namespace to its exact root generation and +canonical digest: + +```text +160-byte fixed-width header +entry-count × 72-byte entries +32-byte manifest digest +32-byte checksum +``` + +Each entry is: + +| Offset | Width | Field | +| ---: | ---: | --- | +| 0 | 32 | namespace digest | +| 32 | 8 | root generation | +| 40 | 32 | root digest | + +Entries are sorted by namespace digest and duplicate-free. The maximum entry +count is 4,096 and the maximum manifest length is 295,136 bytes. + + + +| Offset | Width | Field | Canonical value | +| ---: | ---: | --- | --- | +| 0 | 16 | magic | `KEEP:RET:LIVE2\0\0` | +| 16 | 2 | version | `2` | +| 18 | 2 | header length | `160` | +| 20 | 4 | flags | `0` | +| 24 | 8 | total record length | derived exact length | +| 32 | 8 | liveness generation | positive | +| 40 | 2 | entry width | `72` | +| 42 | 2 | reserved | zero | +| 44 | 4 | entry count | `0..=4,096` | +| 48 | 32 | predecessor manifest digest | zero for generation 1 | +| 80 | 32 | entry-set digest | exact canonical entries | +| 112 | 48 | reserved | zero | + + + +The entry-set, manifest, and checksum domains are respectively: + +```text +keep.retention-manifest-entries/v2\0 +keep.retention-manifest/v2\0 +keep.retention-manifest-checksum/v2\0 +``` + +The manifest pool coordinate is: + +```text +retention/manifests/ + -.manifest +``` + +## Retention head + +`retention/HEAD` and `retention/head.next` use one exact 144-byte record: + +| Offset | Width | Field | Canonical value | +| ---: | ---: | --- | --- | +| 0 | 16 | magic | `KEEP:RET:HEAD2\0\0` | +| 16 | 2 | version | `2` | +| 18 | 2 | record length | `144` | +| 20 | 4 | flags | `0` | +| 24 | 8 | liveness generation | positive | +| 32 | 8 | manifest length | exact admitted length | +| 40 | 32 | manifest digest | exact pool digest | +| 72 | 32 | predecessor manifest digest | zero for generation 1 | +| 104 | 8 | reserved | zero | +| 112 | 32 | checksum | BLAKE3-256 over bytes `0..112` | + +The checksum domain is `keep.retention-head-checksum/v2\0`. + +## Closure admission + +Before publication, Keep pins one completely verified catalog generation and +derives the complete closure for every anchor: + +1. Resolve and admit the exact layout record named by `LayoutId`. +2. Require its embedded `BlobId` to equal the anchor `BlobId`. +3. Resolve and admit every ordered chunk identity required by that layout. +4. Verify each physical record, identity, checksum, digest, and catalog + coordinate under the stored realization profile. +5. Enforce the stored limits with checked counters and a visited set. +6. Reconstruct and authenticate the complete blob identity. + +A missing or corrupt closure member, ambiguous catalog claim, unsupported +profile, limit breach, cycle, unknown mandatory edge, identity mismatch, or +ordering error refuses the entire transition. Keep never omits one failed +member and continues with a smaller live set. + +Version-2 catalog publication holds the same writer authority and proves every +current retained closure against its candidate catalog before replacing the +catalog `HEAD`. + +## Generation transition + +A transition supplies a namespace, an expected state of absent or one exact +`RootGeneration`, a complete canonical anchor set, the exact realization +profile coordinate, and admitted limits. + +Under exclusive writer authority, publication: + +1. completes recovery of every fixed retention stage; +2. admits the current retention head, manifest, and selected namespace root; +3. compares expected and observed generations; +4. verifies the candidate closure against one pinned catalog; +5. writes and synchronizes `retention/root.next`; +6. for a new namespace, exclusively creates and verifies its exact digest-named + directory, then synchronizes `retention/roots`; +7. links and verifies the root pool entry, then synchronizes its directory; +8. writes and synchronizes `retention/manifest.next`; +9. links and verifies the manifest pool entry and synchronizes its directory; +10. writes and synchronizes `retention/head.next`; +11. atomically replaces `retention/HEAD` and synchronizes `retention`; + `root.next` and `manifest.next` remain durable until the retention head + commits, then are removed and `retention` is synchronized again; and +12. returns a consequential `#[must_use]` receipt. + +The receipt binds the namespace, expected and observed generations, committed +root generation and digest, global manifest generation and digest, profile +coordinate, anchor-set and closure digests, catalog generation and digest, and +every durable publication outcome. + +A stale transition preserves expected and observed generations. A +byte-identical retry returns **already committed** only while that exact root +successor remains current; otherwise it returns the precise stale state. + +A reader holds one shared `ReaderFence` and double-collects the catalog and +retention heads around complete transitive admission. It accepts only the same +coordinates before and after for both heads. Any generation, length, digest, or +checksum change discards the view and retries within a bounded attempt limit; +exhaustion refuses. The accepted view observes one complete root generation for +its snapshot lifetime. diff --git a/xtask/tests/retention_store_v2_protocol_contract.rs b/xtask/tests/retention_store_v2_protocol_contract.rs new file mode 100644 index 0000000..7c80151 --- /dev/null +++ b/xtask/tests/retention_store_v2_protocol_contract.rs @@ -0,0 +1,228 @@ +//! Written-contract evidence for the version-2 retention store. + +#![cfg(feature = "repository-tasks")] + +use std::fs; +use std::io; +use std::path::{Path, PathBuf}; + +const FORMAT_ROOT: &str = "docs/formats/segment-store-v2"; +const DOCUMENT_REVIEW_LIMIT_LINES: usize = 300; + +fn repository_root() -> Result { + Path::new(env!("CARGO_MANIFEST_DIR")) + .parent() + .map(Path::to_path_buf) + .ok_or_else(|| io::Error::other("xtask manifest directory has no parent")) +} + +fn read(relative: &str) -> Result { + fs::read_to_string(repository_root()?.join(relative)) +} + +fn normalized(document: &str) -> String { + document.split_whitespace().collect::>().join(" ") +} + +#[test] +fn version_two_is_one_routed_protocol() -> Result<(), Box> { + let format_index = read("docs/formats/README.md")?; + let changelog = read("CHANGELOG.md")?; + let overview = normalized(&read(&format!("{FORMAT_ROOT}/README.md"))?); + + assert!( + format_index.contains("segment-store-v2/README.md"), + "format index does not route to segment-store v2" + ); + assert!( + changelog.contains("`keep.segment-store/v2`"), + "changelog does not record the segment-store v2 contract" + ); + for required in [ + "`keep.segment-store/v2`", + "successor to `keep.segment-store/v1`", + "[Retention records and publication](retention.md)", + "[GC and disposition records](gc.md)", + "[Migration and recovery](recovery.md)", + "[Migration crash points](migration-crash.md)", + "[Requirements and evidence](requirements.md)", + "[Format rationale](rationale.md)", + ] { + assert!( + overview.contains(required), + "segment-store v2 overview omits `{required}`" + ); + } + Ok(()) +} + +#[test] +fn retention_records_have_exact_canonical_grammars() -> Result<(), Box> { + let retention = normalized(&read(&format!("{FORMAT_ROOT}/retention.md"))?); + + for required in [ + "`RetentionNamespace`", + "1 through 255 bytes", + "`RootGeneration`", + "`LivenessGeneration`", + "big-endian", + "fixed-width header", + "sorted, duplicate-free", + "BLAKE3-256", + "domain-separated", + "trailing bytes", + "unknown mandatory flags", + "maximum admitted namespace count", + "before any namespace-generation or manifest bytes are staged", + "expected and observed generations", + "already committed", + "one complete root generation", + "remain durable until the retention head", + "double-collects the catalog and retention heads", + "same coordinates before and after", + ] { + assert!( + retention.contains(required), + "segment-store v2 retention grammar omits `{required}`" + ); + } + Ok(()) +} + +#[test] +fn migration_and_recovery_define_every_authority_boundary() -> Result<(), Box> +{ + let recovery = normalized(&read(&format!("{FORMAT_ROOT}/recovery.md"))?); + + for required in [ + "one-way explicit migration", + "`migration.intent`", + "`migration.intent.next`", + "`migration.receipt`", + "`migration.receipt.next`", + "`FORMAT.next`", + "`migration.intent` is exactly 256 bytes", + "`migration.receipt` is exactly 256 bytes", + "catalog generation, length, and digest", + "deterministically derived store identifier", + "absence of `retention/HEAD` is the canonical empty retention state", + "pre-effect incomplete stage", + "keep.initial-retention-state/v2\\0", + "keep.initial-gc-state/v2\\0", + "keep.empty-disposition-set/v2\\0", + "root.next` is durable before a new namespace directory", + "`KEEP-CRASH-036`", + "`KEEP-CRASH-073`", + "partial migration", + "Version-1 admission refuses", + "`reader.lock`", + "`GcRetirementIntent`", + "`GcRetirementReceipt`", + "`RecoveryDispositionReceipt`", + "unknown entry", + "unrecoverable ambiguity", + "idempotent", + "process death", + ] { + assert!( + recovery.contains(required), + "segment-store v2 recovery contract omits `{required}`" + ); + } + Ok(()) +} + +#[test] +fn migration_never_writes_canonical_fixed_names_in_place() -> Result<(), Box> +{ + let migration = normalized(&read(&format!("{FORMAT_ROOT}/migration-crash.md"))?); + + for required in [ + "never writes canonical fixed names in place", + "`migration.intent.next`", + "`FORMAT.next`", + "`migration.receipt.next`", + "linked without replacement", + "pre-effect incomplete stage", + "`KEEP-CRASH-053`", + "`KEEP-CRASH-073`", + "before, during, and after process-death evidence", + ] { + assert!( + migration.contains(required), + "segment-store v2 migration crash protocol omits `{required}`" + ); + } + Ok(()) +} + +#[test] +fn gc_records_are_bounded_before_their_implementation() -> Result<(), Box> { + let gc = normalized(&read(&format!("{FORMAT_ROOT}/gc.md"))?); + + for required in [ + "`GcRetirementIntent`", + "320-byte fixed-width header", + "72-byte candidate entries", + "65,536", + "`GcRetirementReceipt`", + "exactly 320 bytes", + "`RecoveryDispositionReceipt`", + "canonical absent candidate prefix", + "unrecoverable ambiguity", + "Planned in #21", + ] { + assert!( + gc.contains(required), + "segment-store v2 GC grammar omits `{required}`" + ); + } + Ok(()) +} + +#[test] +fn requirement_ledger_names_planned_and_executable_evidence() +-> Result<(), Box> { + let requirements = normalized(&read(&format!("{FORMAT_ROOT}/requirements.md"))?); + + for required in [ + "`KEEP-RETENTION-001`", + "`KEEP-RETENTION-010`", + "`KEEP-MIGRATION-001`", + "`KEEP-MIGRATION-008`", + "`KEEP-GC-001`", + "Planned in #19", + "Planned in #21", + "golden-format", + "model-based", + "corruption", + "crash-injection", + "fuzz", + ] { + assert!( + requirements.contains(required), + "segment-store v2 requirement ledger omits `{required}`" + ); + } + Ok(()) +} + +#[test] +fn version_two_pages_stay_within_the_review_threshold() -> Result<(), Box> { + for name in [ + "README.md", + "gc.md", + "migration-crash.md", + "rationale.md", + "recovery.md", + "requirements.md", + "retention.md", + ] { + let line_count = read(&format!("{FORMAT_ROOT}/{name}"))?.lines().count(); + assert!( + line_count <= DOCUMENT_REVIEW_LIMIT_LINES, + "{name} has {line_count} lines; review threshold is {DOCUMENT_REVIEW_LIMIT_LINES}" + ); + } + Ok(()) +} From b1b4f23467c89542eb5d55c9936c7d0575e922db Mon Sep 17 00:00:00 2001 From: James Ross Date: Wed, 29 Jul 2026 19:45:55 -0700 Subject: [PATCH 002/111] Test: Freeze version two retention bytes --- conformance/segment-store/v2/ORIGIN.md | 67 ++++++ conformance/segment-store/v2/README.md | 74 +++++++ conformance/segment-store/v2/artifacts.tsv | 8 + conformance/segment-store/v2/definition.tsv | 90 ++++++++ .../segment-store/v2/format-marker.hex | 1 + conformance/segment-store/v2/inventory.tsv | 4 + .../segment-store/v2/migration-intent.hex | 1 + .../segment-store/v2/migration-receipt.hex | 1 + .../segment-store/v2/migration-source.tsv | 3 + .../segment-store/v2/one-anchor-root.hex | 1 + .../segment-store/v2/one-root-head.hex | 1 + .../segment-store/v2/one-root-manifest.hex | 1 + .../segment-store/v2/retention-profile.tsv | 3 + docs/formats/README.md | 2 +- docs/formats/segment-store-v2/README.md | 6 + .../segment-store-v2/migration-crash.md | 23 ++ .../segment-store-v2/migration-inventory.md | 49 +++++ docs/formats/segment-store-v2/recovery.md | 15 +- docs/formats/segment-store-v2/retention.md | 18 +- ...retention_store_v2_conformance_contract.rs | 61 ++++++ .../tests/retention_store_v2_format_oracle.rs | 75 +++++++ .../artifacts.rs | 145 +++++++++++++ .../artifacts/migration.rs | 96 +++++++++ .../artifacts/retention.rs | 155 ++++++++++++++ .../encoding.rs | 126 +++++++++++ .../fixture_assertion.rs | 198 ++++++++++++++++++ .../retention_store_v2_protocol_contract.rs | 11 + 27 files changed, 1220 insertions(+), 15 deletions(-) create mode 100644 conformance/segment-store/v2/ORIGIN.md create mode 100644 conformance/segment-store/v2/README.md create mode 100644 conformance/segment-store/v2/artifacts.tsv create mode 100644 conformance/segment-store/v2/definition.tsv create mode 100644 conformance/segment-store/v2/format-marker.hex create mode 100644 conformance/segment-store/v2/inventory.tsv create mode 100644 conformance/segment-store/v2/migration-intent.hex create mode 100644 conformance/segment-store/v2/migration-receipt.hex create mode 100644 conformance/segment-store/v2/migration-source.tsv create mode 100644 conformance/segment-store/v2/one-anchor-root.hex create mode 100644 conformance/segment-store/v2/one-root-head.hex create mode 100644 conformance/segment-store/v2/one-root-manifest.hex create mode 100644 conformance/segment-store/v2/retention-profile.tsv create mode 100644 docs/formats/segment-store-v2/migration-inventory.md create mode 100644 xtask/tests/retention_store_v2_conformance_contract.rs create mode 100644 xtask/tests/retention_store_v2_format_oracle.rs create mode 100644 xtask/tests/retention_store_v2_format_oracle/artifacts.rs create mode 100644 xtask/tests/retention_store_v2_format_oracle/artifacts/migration.rs create mode 100644 xtask/tests/retention_store_v2_format_oracle/artifacts/retention.rs create mode 100644 xtask/tests/retention_store_v2_format_oracle/encoding.rs create mode 100644 xtask/tests/retention_store_v2_format_oracle/fixture_assertion.rs diff --git a/conformance/segment-store/v2/ORIGIN.md b/conformance/segment-store/v2/ORIGIN.md new file mode 100644 index 0000000..305c4d2 --- /dev/null +++ b/conformance/segment-store/v2/ORIGIN.md @@ -0,0 +1,67 @@ +# Version 2 Corpus Origin + +The corpus was constructed on 2026-07-29 with: + +- `rustc 1.96.0 (ac68faa20 2026-05-25)`; +- `cargo 1.96.0 (30a34c682 2026-05-25)`; and +- `b3sum 1.8.5`. + +## Independent inputs + +The oracle imports exact bytes only from these previously accepted fixtures: + +- `conformance/segment-store/v1/one-zero-segment.hex`; +- `conformance/segment-store/v1/one-zero-catalog.hex`; +- `conformance/segment-store/v1/one-zero-head.hex`; +- the one-zero `BlobId` canonical text and `LayoutId` binary identity from + `conformance/layout/v1/layouts.tsv`. + +It parses the version-1 head coordinate, catalog predecessor, and segment and +catalog semantic digests directly from fixed offsets. The oracle constructs the +59-byte `BlobId` from the accepted binary grammar and verifies its length and +digest against the layout table; the table directly supplies the 60-byte +`LayoutId`. It does not call a production encoder, decoder, retention type, +migration adapter, serializer, or filesystem implementation. + +## Definition verification + +The profile digest was checked independently with: + +```bash +{ + printf 'keep.retention-realization-profile/v1\0' + cat conformance/segment-store/v2/retention-profile.tsv +} | b3sum --no-names +``` + +Exact output: + +```text +db1c1c1a50613ef11f7c0ee0882e37b6d24e2db2ca57783d01197ba51b61ce59 +``` + +The format-definition digest was checked independently with: + +```bash +{ + printf 'keep.segment-store-definition/v2\0' + cat conformance/segment-store/v2/definition.tsv +} | b3sum --no-names +``` + +Exact output: + +```text +32381f1ac332d1277a7e1faf8f11576993cb55b7e85d2a110b74dc9c3b873427 +``` + +## Materialization boundary + +A temporary ignored Rust test wrote the initially reviewed TSV and hexadecimal +artifacts from the handwritten oracle. That write path was removed immediately +after materialization. The committed oracle is read-only and rejects drift. + +Changing any fixture requires a deliberate specification change, an updated +definition or profile digest when affected, fresh independent construction, +and review of every dependent migration and retention coordinate. A fixture is +never regenerated to make a production implementation pass. diff --git a/conformance/segment-store/v2/README.md b/conformance/segment-store/v2/README.md new file mode 100644 index 0000000..7417eb2 --- /dev/null +++ b/conformance/segment-store/v2/README.md @@ -0,0 +1,74 @@ +# Durable Segment Store Version 2 Corpus + +This corpus freezes independent canonical inputs and golden bytes for +`keep.segment-store/v2`. It proves the written format has one executable byte +interpretation. It does not prove that a production encoder, decoder, +migration, retention transition, or garbage collector exists. + +## Corpus files + +| File | Contract | +| --- | --- | +| `definition.tsv` | Sorted format-definition key/value bytes | +| `retention-profile.tsv` | Registered realization-profile definition | +| `inventory.tsv` | Canonical one-segment, one-catalog migration inventory | +| `migration-source.tsv` | Exact version-1 and derived migration coordinates | +| `artifacts.tsv` | Golden artifact lengths, digests, checksums, and filenames | +| `format-marker.hex` | Canonical 96-byte `FORMAT` record | +| `migration-intent.hex` | Canonical 256-byte migration intent | +| `migration-receipt.hex` | Canonical 256-byte migration receipt | +| `one-anchor-root.hex` | Generation-1 root with one nontext namespace | +| `one-root-manifest.hex` | Generation-1 one-namespace manifest | +| `one-root-head.hex` | Generation-1 retention head | +| `ORIGIN.md` | Construction provenance and verification boundary | + +Every text file uses UTF-8 or ASCII, LF line endings, and one final newline. +Every hex fixture is one lowercase hexadecimal line with one final newline. +In `artifacts.tsv`, `bound_digest_hex` is the marker content digest for +`format-marker`, the intent digest for `migration-intent`, the referenced +intent digest for `migration-receipt`, the canonical record digest for +`retention-root` and `retention-manifest`, and the referenced manifest digest +for `retention-head`. + +## Frozen identities + +The realization-profile digest is +`db1c1c1a50613ef11f7c0ee0882e37b6d24e2db2ca57783d01197ba51b61ce59`. +It hashes the exact `retention-profile.tsv` bytes under the registered profile +domain. + +The format-definition digest is +`32381f1ac332d1277a7e1faf8f11576993cb55b7e85d2a110b74dc9c3b873427`. +It hashes the exact `definition.tsv` bytes under the registered format domain. +The definition binds the profile digest, every named domain, magic, version, +field order, record width, format limit, and migration synchronization mask. + +The migration fixture preserves the version-1 one-zero segment and generation-1 +catalog. Its canonical two-entry inventory digest is +`40bf5d49c34847ac9cf46a256f343cee80cd980d1405d2dd02ceff8f58d674f9`. +The derived logical store identifier is +`0cd9d3dfbec9b349fe42d21475271b0e8de23c043440d6427a1c37898ad1dd79`. +Fixture-only root device, mount, and file coordinates are `1`, `2`, and `3`; +they bind in-place recovery but do not enter the logical store identifier. + +The retention fixture uses namespace bytes `00 2f ff`, proving the namespace is +opaque and not a path or Unicode string. Its one anchor combines the canonical +one-zero `BlobId` and `LayoutId` values from the existing layout corpus. + +## Verification + +Run: + +```bash +cargo test --manifest-path xtask/Cargo.toml \ + --test retention_store_v2_format_oracle +``` + +The test-only oracle constructs every record from handwritten offsets and +domain preimages, compares exact fixture bytes and tables, and imports no +production version-2 codec. The repository protocol and documentation gates +route this corpus separately. + +Passing this corpus is necessary but insufficient for issue #19. Production +code still needs parser, corruption, property, model, crash, recovery, +concurrency, fuzz, and public API evidence. diff --git a/conformance/segment-store/v2/artifacts.tsv b/conformance/segment-store/v2/artifacts.tsv new file mode 100644 index 0000000..dace87c --- /dev/null +++ b/conformance/segment-store/v2/artifacts.tsv @@ -0,0 +1,8 @@ +keep.segment-store-v2.artifacts/v1 +case kind byte_length generation entry_count bound_digest_hex final_checksum_hex fixture +format-marker format-marker 96 - - 4b063c329085abdebe86b256d531b112c7ea33cb2f545caa40a7a869ff3337ce 06384cbaf2b69e0a12eeb2bf62df4c49e193d56f2bde940b3c5637320458abc1 format-marker.hex +migration-intent migration-intent 256 1 2 a15a00000219df20979da36419046eae9a0ba998645fbfe308ea4335a8326b44 7bec10cc8c1eef5ab0e8e8b6a33240bba291252d4263147df134062eb70d3f1f migration-intent.hex +migration-receipt migration-receipt 256 1 2 a15a00000219df20979da36419046eae9a0ba998645fbfe308ea4335a8326b44 3a6a5f29bfafeffb9401de5ba814c09c345adbad69e8ba0531e3eb1ebb0b681d migration-receipt.hex +one-anchor-root retention-root 378 1 1 ca4c11f265c3bed07073bdc3b6aef003e964ac8cb36fcfcc92f20fa6f0b60085 28c52ff0f8d6533234be083f425e921d699639e204e2c66dec0cae2ff0a2dc34 one-anchor-root.hex +one-root-manifest retention-manifest 296 1 1 f46b96a2bf3379320cf59e8af15b9d108de06025c415307b28953714bd7a80eb 10597643c3fc9485c7ecd3bb511d6726e726fd92f0f769a204b899c5fdc77d2c one-root-manifest.hex +one-root-head retention-head 144 1 1 f46b96a2bf3379320cf59e8af15b9d108de06025c415307b28953714bd7a80eb ac049edb33af7e957c6ff11ead7e1bcf9c40fa9793cc84979215ffbba5f630b7 one-root-head.hex diff --git a/conformance/segment-store/v2/definition.tsv b/conformance/segment-store/v2/definition.tsv new file mode 100644 index 0000000..cea0627 --- /dev/null +++ b/conformance/segment-store/v2/definition.tsv @@ -0,0 +1,90 @@ +keep.segment-store.definition/v2 +key value +domain.empty-disposition-set keep.empty-disposition-set/v2\0 +domain.format-definition keep.segment-store-definition/v2\0 +domain.format-marker keep.store-format-marker/v2\0 +domain.format-marker-checksum keep.segment-store-marker-checksum/v2\0 +domain.gc-candidate-set keep.gc-candidate-set/v2\0 +domain.gc-intent keep.gc-retirement-intent/v2\0 +domain.gc-intent-checksum keep.gc-retirement-intent-checksum/v2\0 +domain.gc-receipt-checksum keep.gc-retirement-receipt-checksum/v2\0 +domain.initial-gc-state keep.initial-gc-state/v2\0 +domain.initial-retention-state keep.initial-retention-state/v2\0 +domain.migration-intent keep.store-migration-intent/v2\0 +domain.migration-intent-checksum keep.store-migration-intent-checksum/v2\0 +domain.migration-inventory keep.store-v1-pool-inventory/v2\0 +domain.migration-receipt-checksum keep.store-migration-receipt-checksum/v2\0 +domain.recovery-disposition-checksum keep.recovery-disposition-receipt-checksum/v2\0 +domain.retention-anchor-set keep.retention-anchor-set/v2\0 +domain.retention-head-checksum keep.retention-head-checksum/v2\0 +domain.retention-manifest keep.retention-manifest/v2\0 +domain.retention-manifest-checksum keep.retention-manifest-checksum/v2\0 +domain.retention-manifest-entries keep.retention-manifest-entries/v2\0 +domain.retention-namespace keep.retention-namespace/v1\0 +domain.retention-profile keep.retention-realization-profile/v1\0 +domain.retention-root keep.retention-root/v2\0 +domain.retention-root-checksum keep.retention-root-checksum/v2\0 +domain.store-identifier keep.store-identifier/v2\0 +format.coordinate keep.segment-store/v2 +format.marker.fields magic:16,version:u16,record_length:u16,flags:u32,definition_digest:32,maximum_namespace_count:u32,reserved:u32,checksum:32 +format.marker.length 96 +format.marker.magic KEEP:STORE:V2\0\0\0 +format.marker.version 2 +gc.intent.candidate-width 72 +gc.intent.fields magic:16,version:u16,header_length:u16,flags:u32,total_length:u64,generation:u64,candidate_width:u16,reserved:u16,candidate_count:u32,liveness_generation:u64,manifest_digest:32,catalog_generation:u64,catalog_digest:32,profile_identity:u32,profile_version:u32,profile_digest:32,catalog_proof_digest:32,pool_digest:32,disposition_set_digest:32,reader_device:u64,reader_mount:u64,reader_file:u64,candidate_set_digest:32,candidates:count*72,intent_digest:32,checksum:32 +gc.intent.header-length 320 +gc.intent.magic KEEP:GC:INTENT2\0 +gc.intent.maximum-candidates 65536 +gc.intent.maximum-length 4718976 +gc.intent.version 2 +gc.receipt.fields magic:16,version:u16,record_length:u16,flags:u32,generation:u64,intent_digest:32,retired_set_digest:32,pool_state_digest:32,liveness_generation:u64,manifest_digest:32,catalog_generation:u64,catalog_digest:32,reader_device:u64,reader_mount:u64,reader_file:u64,synchronization_count:u64,reserved:48,checksum:32 +gc.receipt.length 320 +gc.receipt.magic KEEP:GC:RECEIPT2 +gc.receipt.version 2 +migration.intent.fields magic:16,version:u16,record_length:u16,flags:u32,catalog_generation:u64,catalog_length:u64,catalog_digest:32,predecessor_digest:32,inventory_digest:32,root_device:u64,root_mount:u64,root_file:u64,definition_digest:32,store_id:32,checksum:32 +migration.intent.length 256 +migration.intent.magic KEEP:MIG:INT2\0\0\0 +migration.intent.version 2 +migration.inventory.entry-fields kind:u8,reserved:7,catalog_generation:u64,artifact_length:u64,artifact_digest:32 +migration.inventory.entry-width 56 +migration.inventory.maximum-entries 2097152 +migration.receipt.fields magic:16,version:u16,record_length:u16,flags:u32,intent_digest:32,store_id:32,format_marker_digest:32,initial_retention_digest:32,initial_gc_digest:32,disposition_set_digest:32,synchronization_mask:u64,checksum:32 +migration.receipt.length 256 +migration.receipt.magic KEEP:MIG:REC2\0\0\0 +migration.receipt.synchronization-mask 0x00000000000003ff +migration.receipt.version 2 +recovery.disposition.fields magic:16,version:u16,record_length:u16,flags:u32,artifact_kind:u16,decision:u16,classification:u16,reserved:u16,artifact_length:u64,artifact_identity_digest:32,artifact_content_digest:32,publication_generation:u64,publication_checksum:32,catalog_generation:u64,catalog_digest:32,liveness_generation:u64,manifest_digest:32,reader_device:u64,reader_mount:u64,reader_file:u64,decision_evidence_digest:32,reserved:8,checksum:32 +recovery.disposition.length 320 +recovery.disposition.magic KEEP:REC:DISP2\0\0 +recovery.disposition.maximum-receipts 65536 +recovery.disposition.version 2 +retention.anchor.fields blob_id:59,layout_id:60 +retention.anchor.width 119 +retention.closure.maximum-depth 8 +retention.closure.maximum-encoded-bytes 16777216 +retention.closure.maximum-nodes 1048576 +retention.closure.maximum-physical-bytes 1073741824 +retention.head.fields magic:16,version:u16,record_length:u16,flags:u32,liveness_generation:u64,manifest_length:u64,manifest_digest:32,predecessor_manifest_digest:32,reserved:u64,checksum:32 +retention.head.length 144 +retention.head.magic KEEP:RET:HEAD2\0\0 +retention.head.version 2 +retention.manifest.entry-fields namespace_digest:32,root_generation:u64,root_digest:32 +retention.manifest.entry-width 72 +retention.manifest.fields magic:16,version:u16,header_length:u16,flags:u32,total_length:u64,liveness_generation:u64,entry_width:u16,reserved:u16,entry_count:u32,predecessor_digest:32,entry_set_digest:32,reserved:48,entries:count*72,manifest_digest:32,checksum:32 +retention.manifest.header-length 160 +retention.manifest.magic KEEP:RET:LIVE2\0\0 +retention.manifest.maximum-entries 4096 +retention.manifest.maximum-length 295136 +retention.manifest.version 2 +retention.maximum-namespaces 4096 +retention.namespace.maximum-length 255 +retention.namespace.minimum-length 1 +retention.profile.digest db1c1c1a50613ef11f7c0ee0882e37b6d24e2db2ca57783d01197ba51b61ce59 +retention.profile.identity 1 +retention.profile.version 1 +retention.root.fields magic:16,version:u16,header_length:u16,flags:u32,total_length:u64,root_generation:u64,namespace_length:u16,anchor_width:u16,anchor_count:u32,profile_identity:u32,profile_version:u32,profile_digest:32,closure_node_limit:u64,closure_depth_limit:u16,reserved:u16,encoded_byte_limit:u64,physical_byte_limit:u64,predecessor_digest:32,anchor_set_digest:32,reserved:12,namespace:namespace_length,anchors:count*119,root_digest:32,checksum:32 +retention.root.header-length 192 +retention.root.magic KEEP:RET:ROOT2\0\0 +retention.root.maximum-anchors 65536 +retention.root.maximum-length 7799295 +retention.root.version 2 diff --git a/conformance/segment-store/v2/format-marker.hex b/conformance/segment-store/v2/format-marker.hex new file mode 100644 index 0000000..30640a9 --- /dev/null +++ b/conformance/segment-store/v2/format-marker.hex @@ -0,0 +1 @@ +4b4545503a53544f52453a5632000000000200600000000032381f1ac332d1277a7e1faf8f11576993cb55b7e85d2a110b74dc9c3b873427000010000000000006384cbaf2b69e0a12eeb2bf62df4c49e193d56f2bde940b3c5637320458abc1 diff --git a/conformance/segment-store/v2/inventory.tsv b/conformance/segment-store/v2/inventory.tsv new file mode 100644 index 0000000..cd0443a --- /dev/null +++ b/conformance/segment-store/v2/inventory.tsv @@ -0,0 +1,4 @@ +keep.segment-store-v2.inventory/v1 +kind generation byte_length artifact_digest_hex source_fixture +segment 0 337 b7542dced2ab770894a14d1d04b066e3a899942602c5986d35ba6df6c1a35cfc one-zero-segment.hex +catalog 1 352 04b82519b0399baefd0b9c0f32a871052e4c47e3a00226ab03b21661470f7320 one-zero-catalog.hex diff --git a/conformance/segment-store/v2/migration-intent.hex b/conformance/segment-store/v2/migration-intent.hex new file mode 100644 index 0000000..5ce426b --- /dev/null +++ b/conformance/segment-store/v2/migration-intent.hex @@ -0,0 +1 @@ +4b4545503a4d49473a494e543200000000020100000000000000000000000001000000000000016004b82519b0399baefd0b9c0f32a871052e4c47e3a00226ab03b21661470f7320000000000000000000000000000000000000000000000000000000000000000040bf5d49c34847ac9cf46a256f343cee80cd980d1405d2dd02ceff8f58d674f900000000000000010000000000000002000000000000000332381f1ac332d1277a7e1faf8f11576993cb55b7e85d2a110b74dc9c3b8734270cd9d3dfbec9b349fe42d21475271b0e8de23c043440d6427a1c37898ad1dd797bec10cc8c1eef5ab0e8e8b6a33240bba291252d4263147df134062eb70d3f1f diff --git a/conformance/segment-store/v2/migration-receipt.hex b/conformance/segment-store/v2/migration-receipt.hex new file mode 100644 index 0000000..66b524e --- /dev/null +++ b/conformance/segment-store/v2/migration-receipt.hex @@ -0,0 +1 @@ +4b4545503a4d49473a524543320000000002010000000000a15a00000219df20979da36419046eae9a0ba998645fbfe308ea4335a8326b440cd9d3dfbec9b349fe42d21475271b0e8de23c043440d6427a1c37898ad1dd794b063c329085abdebe86b256d531b112c7ea33cb2f545caa40a7a869ff3337ced52f1f022edb1de7b840c5bf8fb55de7932ca69370ae85e2bee4179143792bc3ba0ea200a5b06741564c43a79a91945bef0b0fac51c960ea4f8207094f3e1e31a80259fcd1237203ea6c6cc5065514abdeb01da603c3194b096a045cf694c95a00000000000003ff3a6a5f29bfafeffb9401de5ba814c09c345adbad69e8ba0531e3eb1ebb0b681d diff --git a/conformance/segment-store/v2/migration-source.tsv b/conformance/segment-store/v2/migration-source.tsv new file mode 100644 index 0000000..5d0204b --- /dev/null +++ b/conformance/segment-store/v2/migration-source.tsv @@ -0,0 +1,3 @@ +keep.segment-store-v2.migration-source/v1 +case catalog_generation catalog_length catalog_digest_hex predecessor_digest_hex inventory_digest_hex definition_digest_hex store_id_hex root_device root_mount root_file +one-zero 1 352 04b82519b0399baefd0b9c0f32a871052e4c47e3a00226ab03b21661470f7320 0000000000000000000000000000000000000000000000000000000000000000 40bf5d49c34847ac9cf46a256f343cee80cd980d1405d2dd02ceff8f58d674f9 32381f1ac332d1277a7e1faf8f11576993cb55b7e85d2a110b74dc9c3b873427 0cd9d3dfbec9b349fe42d21475271b0e8de23c043440d6427a1c37898ad1dd79 1 2 3 diff --git a/conformance/segment-store/v2/one-anchor-root.hex b/conformance/segment-store/v2/one-anchor-root.hex new file mode 100644 index 0000000..caeb194 --- /dev/null +++ b/conformance/segment-store/v2/one-anchor-root.hex @@ -0,0 +1 @@ +4b4545503a5245543a524f4f54320000000200c000000000000000000000017a000000000000000100030077000000010000000100000001db1c1c1a50613ef11f7c0ee0882e37b6d24e2db2ca57783d01197ba51b61ce59000000000000000400020000000000000000100000000000000010000000000000000000000000000000000000000000000000000000000000000000227f6333d1bcc380899ba25903b5d7d2b8804cc828e8f580b5876b0677d024f5000000000000000000000000002fff4b4545503a424c4f423a49440000000000010100000000000000011cfb8fa9e917aba15a1f592095f377ff180755fe1212b0d7d2ec750bd128b6064b4545503a4c41594f55543a494400000001000100000000000000dc887da23f1a7483359a78fc9a7fde80030ec2c4690603803f0ab7d0edb56575b8ca4c11f265c3bed07073bdc3b6aef003e964ac8cb36fcfcc92f20fa6f0b6008528c52ff0f8d6533234be083f425e921d699639e204e2c66dec0cae2ff0a2dc34 diff --git a/conformance/segment-store/v2/one-root-head.hex b/conformance/segment-store/v2/one-root-head.hex new file mode 100644 index 0000000..47df02a --- /dev/null +++ b/conformance/segment-store/v2/one-root-head.hex @@ -0,0 +1 @@ +4b4545503a5245543a48454144320000000200900000000000000000000000010000000000000128f46b96a2bf3379320cf59e8af15b9d108de06025c415307b28953714bd7a80eb00000000000000000000000000000000000000000000000000000000000000000000000000000000ac049edb33af7e957c6ff11ead7e1bcf9c40fa9793cc84979215ffbba5f630b7 diff --git a/conformance/segment-store/v2/one-root-manifest.hex b/conformance/segment-store/v2/one-root-manifest.hex new file mode 100644 index 0000000..c2fed0b --- /dev/null +++ b/conformance/segment-store/v2/one-root-manifest.hex @@ -0,0 +1 @@ +4b4545503a5245543a4c495645320000000200a0000000000000000000000128000000000000000100480000000000010000000000000000000000000000000000000000000000000000000000000000e763e4d12e1ed333daeb84cc6336d9c3262f639b9d410c0a69c8d23680e9049a000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000ddde2ac65c5ba3829bf0fbd6f36e90272d69a0459fade92272b728a80d7ae6e20000000000000001ca4c11f265c3bed07073bdc3b6aef003e964ac8cb36fcfcc92f20fa6f0b60085f46b96a2bf3379320cf59e8af15b9d108de06025c415307b28953714bd7a80eb10597643c3fc9485c7ecd3bb511d6726e726fd92f0f769a204b899c5fdc77d2c diff --git a/conformance/segment-store/v2/retention-profile.tsv b/conformance/segment-store/v2/retention-profile.tsv new file mode 100644 index 0000000..69c31b9 --- /dev/null +++ b/conformance/segment-store/v2/retention-profile.tsv @@ -0,0 +1,3 @@ +keep.retention-realization-profiles/v1 +identity version canonical_name witness_count selection +1 1 keep.retention-single-canonical-witness/v1 1 canonical-physical-catalog-coordinate diff --git a/docs/formats/README.md b/docs/formats/README.md index 6b8ce93..6952726 100644 --- a/docs/formats/README.md +++ b/docs/formats/README.md @@ -9,7 +9,7 @@ admitted merely because one Rust type can serialize and deserialize it. | --- | --- | --- | --- | | [Flat Chunk Layout v1](flat-chunk-layout-v1/README.md) | `keep.flat-chunks/v1` | Implemented through verified reconstruction in issues #10 and #13 | [Golden corpus](../../conformance/layout/v1/README.md) | | [Durable Segment Store v1](segment-store-v1/README.md) | `keep.segment-store/v1` | Implemented through initialization, publication, restart, and recovery in issues #14–#17 | [Golden corpus](../../conformance/segment-store/v1/README.md) | -| [Durable Segment Store v2](segment-store-v2/README.md) | `keep.segment-store/v2` | Retention transition implementation and executable evidence planned in issue #19 | Golden corpus planned in issue #19 | +| [Durable Segment Store v2](segment-store-v2/README.md) | `keep.segment-store/v2` | Retention transition implementation planned in issue #19 | [Golden corpus](../../conformance/segment-store/v2/README.md) | The registry records protocol specifications, including formats whose implementation is still planned. Each format page states its exact proof diff --git a/docs/formats/segment-store-v2/README.md b/docs/formats/segment-store-v2/README.md index c908aab..4d7ea1b 100644 --- a/docs/formats/segment-store-v2/README.md +++ b/docs/formats/segment-store-v2/README.md @@ -47,11 +47,17 @@ The following pages form one protocol: recovery-disposition reservation, and restart behavior. - [Migration crash points](migration-crash.md) owns fixed-stage publication and the exact process-death boundaries for migration. +- [Migration inventory](migration-inventory.md) owns the bounded canonical + digest over preserved version-1 immutable pools. - [Requirements and evidence](requirements.md) owns stable requirement and crash identifiers, evidence status, compatibility, and nonclaims. - [Format rationale](rationale.md) records format-local choices and rejected alternatives. +The [version-2 golden corpus](../../../conformance/segment-store/v2/README.md) +freezes independent definition, profile, inventory, and record bytes. It is +format evidence, not production-writer evidence. + The version-1 [segment](../segment-store-v1/segment.md), [catalog](../segment-store-v1/catalog.md), and [publication-head](../segment-store-v1/catalog.md#publication-head) diff --git a/docs/formats/segment-store-v2/migration-crash.md b/docs/formats/segment-store-v2/migration-crash.md index c5d7041..16127b0 100644 --- a/docs/formats/segment-store-v2/migration-crash.md +++ b/docs/formats/segment-store-v2/migration-crash.md @@ -38,6 +38,29 @@ The fixed stage is not authority. `migration.intent` becomes migration authority only after its canonical link and store-root synchronization. `migration.receipt` becomes completion evidence at the equivalent boundary. +## Receipt synchronization mask + +`migration.receipt` records the exact pre-receipt mask +`0x00000000000003ff`. Bits are: + +| Bit | Completed evidence | +| ---: | --- | +| 0 | canonical migration intent and store root synchronized | +| 1 | `reader.lock` verified, synchronized, and root-synchronized | +| 2 | `retention` created and parent synchronized | +| 3 | `retention/roots` created and parent synchronized | +| 4 | `retention/manifests` created and parent synchronized | +| 5 | `gc` created and parent synchronized | +| 6 | `recovery` created and parent synchronized | +| 7 | `recovery/dispositions` created and parent synchronized | +| 8 | canonical format marker and store root synchronized | +| 9 | complete version-2 view reopened and verified | + +Bits 10 through 63 are zero and refuse when set. The mask records only +evidence completed before receipt construction; receipt-stage publication and +its final root synchronizations are established by admission of the canonical +receipt, not claimed by its own bytes. + ## Namespace prefix After durable intent publication, migration creates persistent `reader.lock` diff --git a/docs/formats/segment-store-v2/migration-inventory.md b/docs/formats/segment-store-v2/migration-inventory.md new file mode 100644 index 0000000..b50501e --- /dev/null +++ b/docs/formats/segment-store-v2/migration-inventory.md @@ -0,0 +1,49 @@ +# Migration Inventory + +This page owns the bounded canonical digest over version-1 immutable segment +and catalog pools used by `migration.intent`. + +## Entry grammar + +One migration inventory entry is exactly 56 bytes: + +| Offset | Width | Field | Canonical value | +| ---: | ---: | --- | --- | +| 0 | 1 | artifact kind | `1` segment or `2` catalog | +| 1 | 7 | reserved | zero | +| 8 | 8 | catalog generation | zero for segment; positive for catalog | +| 16 | 8 | artifact length | exact positive length | +| 24 | 32 | artifact digest | exact admitted segment or catalog digest | + +Entries are sorted by their complete 56-byte canonical encoding and are +duplicate-free. The maximum is 2,097,152 entries across both pools. Count and +length arithmetic is checked before bytes are retained or allocated. + +The inventory digest is: + +```text +BLAKE3-256("keep.store-v1-pool-inventory/v2\0" || + entry-count-u32 || + canonical-entry-bytes) +``` + +The digest is streamed; the complete encoded inventory is never required in +memory. + +## Admission + +Migration inventories the exact pinned version-1 `segments` and `catalogs` +directories under writer authority. Every regular entry must have the one +canonical physical name derived from its verified semantic digest and, for a +catalog, generation. Each artifact is reopened without following links and +completely admitted before its semantic coordinate enters the digest. + +An unknown name, alternate case or width, alias, duplicate semantic coordinate, +wrong kind, link, changed directory, changed artifact, corrupt bytes, count +overflow, or entry above the fixed maximum refuses migration. File existence, +iteration order, path spelling, modification time, and physical file identity +do not enter the digest. + +The exact one-segment, one-catalog input and its canonical entries are frozen in +the version-2 corpus +[`inventory.tsv`](../../../conformance/segment-store/v2/inventory.tsv). diff --git a/docs/formats/segment-store-v2/recovery.md b/docs/formats/segment-store-v2/recovery.md index 871e675..dfd29dc 100644 --- a/docs/formats/segment-store-v2/recovery.md +++ b/docs/formats/segment-store-v2/recovery.md @@ -56,6 +56,10 @@ The definition and checksum domains are when the exact version-1 namespace admits. An unsupported, corrupt, substituted, or same-name/different-digest marker refuses. +The format-definition digest is BLAKE3-256 of its domain followed by the exact +corpus `definition.tsv` bytes. The format-marker digest is BLAKE3-256 of +`keep.store-format-marker/v2\0` followed by all 96 marker bytes. + ## Reader fence `reader.lock` is a persistent regular zero-length file. Its contents and @@ -101,10 +105,13 @@ Version 1 is never extended in place without durable migration evidence. -The checksum domain is `keep.store-migration-intent-checksum/v2\0`. The pool -inventory digest uses `keep.store-v1-pool-inventory/v2\0` over the sorted, -duplicate-free canonical names, lengths, and verified content digests from -both immutable pools. The intent therefore binds the exact catalog generation, +The checksum domain is `keep.store-migration-intent-checksum/v2\0`. The +receipt's intent digest is BLAKE3-256 of +`keep.store-migration-intent/v2\0` followed by all 256 intent bytes. + +The [migration inventory](migration-inventory.md) defines its domain and law: +each migration inventory entry is exactly 56 bytes, and the fixed maximum is +2,097,152 entries. The intent therefore binds the exact catalog generation, length, and digest named by the admitted version-1 `HEAD`. The deterministically derived store identifier is: diff --git a/docs/formats/segment-store-v2/retention.md b/docs/formats/segment-store-v2/retention.md index ca4811b..6462ce4 100644 --- a/docs/formats/segment-store-v2/retention.md +++ b/docs/formats/segment-store-v2/retention.md @@ -71,9 +71,10 @@ Version 2 admits one realization profile: - exact witness count `1` for each layout and chunk identity; and - selection by canonical physical catalog coordinate. -The stored profile coordinate is the `u32` identity, `u32` version, and -BLAKE3-256 digest of its canonical definition bytes. Any unknown or mismatched -coordinate refuses. A future profile requires a successor specification. +The stored coordinate is its `u32` identity, `u32` version, and BLAKE3-256 of +`keep.retention-realization-profile/v1\0` followed by the exact corpus +`retention-profile.tsv` bytes. Any mismatch refuses. A future profile requires +a successor specification. Each root generation stores caller-selected limits no greater than these implementation ceilings: @@ -202,13 +203,10 @@ count is 4,096 and the maximum manifest length is 295,136 bytes. -The entry-set, manifest, and checksum domains are respectively: - -```text -keep.retention-manifest-entries/v2\0 -keep.retention-manifest/v2\0 -keep.retention-manifest-checksum/v2\0 -``` +The `keep.retention-manifest-entries/v2\0` preimage is +`entry-count-u32 || entries`. The `keep.retention-manifest/v2\0` preimage is +`header || entries`. The `keep.retention-manifest-checksum/v2\0` preimage is +`header || entries || manifest-digest`. Each operation is BLAKE3-256. The manifest pool coordinate is: diff --git a/xtask/tests/retention_store_v2_conformance_contract.rs b/xtask/tests/retention_store_v2_conformance_contract.rs new file mode 100644 index 0000000..38f7a60 --- /dev/null +++ b/xtask/tests/retention_store_v2_conformance_contract.rs @@ -0,0 +1,61 @@ +//! Repository-shape evidence for the version-2 segment-store corpus. + +#![cfg(feature = "repository-tasks")] + +use std::collections::BTreeSet; +use std::ffi::OsString; +use std::fs; +use std::io; +use std::path::{Path, PathBuf}; + +const CORPUS_ROOT: &str = "conformance/segment-store/v2"; +const REQUIRED_PATHS: &[&str] = &[ + "README.md", + "ORIGIN.md", + "definition.tsv", + "retention-profile.tsv", + "inventory.tsv", + "migration-source.tsv", + "artifacts.tsv", + "format-marker.hex", + "migration-intent.hex", + "migration-receipt.hex", + "one-anchor-root.hex", + "one-root-manifest.hex", + "one-root-head.hex", +]; + +fn repository_root() -> Result { + Path::new(env!("CARGO_MANIFEST_DIR")) + .parent() + .map(Path::to_path_buf) + .ok_or_else(|| io::Error::other("xtask manifest directory has no parent")) +} + +#[test] +fn version_two_corpus_has_one_complete_regular_file_shape() -> Result<(), io::Error> { + let root = repository_root()?.join(CORPUS_ROOT); + let expected: BTreeSet = REQUIRED_PATHS.iter().map(OsString::from).collect(); + let mut observed = BTreeSet::new(); + for entry in fs::read_dir(&root)? { + let entry = entry?; + assert!( + entry.file_type()?.is_file(), + "{} is not a regular file", + entry.path().display() + ); + observed.insert(entry.file_name()); + } + assert_eq!(observed, expected, "version-2 corpus shape drifted"); + Ok(()) +} + +#[test] +fn format_registry_routes_to_executable_version_two_evidence() -> Result<(), io::Error> { + let format_index = fs::read_to_string(repository_root()?.join("docs/formats/README.md"))?; + assert!( + format_index.contains("../../conformance/segment-store/v2/README.md"), + "format registry does not route to the version-2 corpus" + ); + Ok(()) +} diff --git a/xtask/tests/retention_store_v2_format_oracle.rs b/xtask/tests/retention_store_v2_format_oracle.rs new file mode 100644 index 0000000..86081ec --- /dev/null +++ b/xtask/tests/retention_store_v2_format_oracle.rs @@ -0,0 +1,75 @@ +//! Independent construction oracle for version-2 segment-store golden bytes. + +#![cfg(feature = "repository-tasks")] + +const CORPUS_ROOT: &str = "conformance/segment-store/v2"; +const PROFILE_DEFINITION: &str = + include_str!("../../conformance/segment-store/v2/retention-profile.tsv"); +const FORMAT_DEFINITION: &str = include_str!("../../conformance/segment-store/v2/definition.tsv"); +const LAYOUTS: &str = include_str!("../../conformance/layout/v1/layouts.tsv"); +const V1_SEGMENT: &str = include_str!("../../conformance/segment-store/v1/one-zero-segment.hex"); +const V1_CATALOG: &str = include_str!("../../conformance/segment-store/v1/one-zero-catalog.hex"); +const V1_HEAD: &str = include_str!("../../conformance/segment-store/v1/one-zero-head.hex"); + +struct Artifact { + case_name: &'static str, + kind: &'static str, + generation: &'static str, + entry_count: &'static str, + bound_digest: [u8; 32], + final_checksum: [u8; 32], + fixture: &'static str, + bytes: Vec, +} + +struct Corpus { + profile_digest: [u8; 32], + definition_digest: [u8; 32], + inventory: Inventory, + migration: MigrationSource, + artifacts: Vec, +} + +struct Inventory { + rows: Vec, + digest: [u8; 32], +} + +struct InventoryRow { + kind: &'static str, + generation: u64, + byte_length: u64, + artifact_digest: [u8; 32], + source_fixture: &'static str, + bytes: [u8; 56], +} + +struct MigrationSource { + catalog_generation: u64, + catalog_length: u64, + catalog_digest: [u8; 32], + predecessor_digest: [u8; 32], + inventory_digest: [u8; 32], + definition_digest: [u8; 32], + store_id: [u8; 32], + root_device: u64, + root_mount: u64, + root_file: u64, +} + +struct RootArtifact { + bytes: Vec, + digest: [u8; 32], + checksum: [u8; 32], + namespace_digest: [u8; 32], +} + +struct ManifestArtifact { + bytes: Vec, + digest: [u8; 32], + checksum: [u8; 32], +} + +include!("retention_store_v2_format_oracle/encoding.rs"); +include!("retention_store_v2_format_oracle/artifacts.rs"); +include!("retention_store_v2_format_oracle/fixture_assertion.rs"); diff --git a/xtask/tests/retention_store_v2_format_oracle/artifacts.rs b/xtask/tests/retention_store_v2_format_oracle/artifacts.rs new file mode 100644 index 0000000..06b2f50 --- /dev/null +++ b/xtask/tests/retention_store_v2_format_oracle/artifacts.rs @@ -0,0 +1,145 @@ +// This included source owns handwritten construction of every golden artifact. + +const PROFILE_DOMAIN: &[u8] = b"keep.retention-realization-profile/v1\0"; +const DEFINITION_DOMAIN: &[u8] = b"keep.segment-store-definition/v2\0"; +const INVENTORY_DOMAIN: &[u8] = b"keep.store-v1-pool-inventory/v2\0"; +const STORE_ID_DOMAIN: &[u8] = b"keep.store-identifier/v2\0"; + +fn build_corpus() -> Result { + let profile_digest = hash(PROFILE_DOMAIN, &[PROFILE_DEFINITION.as_bytes()]); + let definition_digest = hash(DEFINITION_DOMAIN, &[FORMAT_DEFINITION.as_bytes()]); + let inventory = build_inventory()?; + let migration = migration_source(definition_digest, inventory.digest)?; + let format = build_format_marker(definition_digest)?; + let intent = build_migration_intent(&migration)?; + let receipt = build_migration_receipt(&migration, &format, &intent)?; + let root = build_retention_root(profile_digest)?; + let manifest = build_retention_manifest(&root)?; + let head = build_retention_head(&manifest)?; + let artifacts = vec![ + format, + intent, + receipt, + Artifact { + case_name: "one-anchor-root", + kind: "retention-root", + generation: "1", + entry_count: "1", + bound_digest: root.digest, + final_checksum: root.checksum, + fixture: "one-anchor-root.hex", + bytes: root.bytes, + }, + Artifact { + case_name: "one-root-manifest", + kind: "retention-manifest", + generation: "1", + entry_count: "1", + bound_digest: manifest.digest, + final_checksum: manifest.checksum, + fixture: "one-root-manifest.hex", + bytes: manifest.bytes, + }, + head, + ]; + Ok(Corpus { + profile_digest, + definition_digest, + inventory, + migration, + artifacts, + }) +} + +fn build_inventory() -> Result { + let segment = decode_hex(V1_SEGMENT)?; + let catalog = decode_hex(V1_CATALOG)?; + require_length(&segment, 337, "version-1 source segment")?; + require_length(&catalog, 352, "version-1 source catalog")?; + let segment_digest = array_32(&segment, 273)?; + let catalog_digest = array_32(&catalog, 320)?; + let mut rows = vec![ + inventory_row(1, 0, &segment, segment_digest, "one-zero-segment.hex")?, + inventory_row(2, 1, &catalog, catalog_digest, "one-zero-catalog.hex")?, + ]; + rows.sort_by_key(|row| row.bytes); + let entry_count = + u32::try_from(rows.len()).map_err(|_| "inventory entry count overflow".to_owned())?; + let mut count = entry_count.to_be_bytes().to_vec(); + for row in &rows { + count.extend_from_slice(&row.bytes); + } + let digest = hash(INVENTORY_DOMAIN, &[&count]); + Ok(Inventory { rows, digest }) +} + +fn inventory_row( + kind: u8, + generation: u64, + artifact: &[u8], + digest: [u8; 32], + source_fixture: &'static str, +) -> Result { + let byte_length = + u64::try_from(artifact.len()).map_err(|_| "inventory length overflow".to_owned())?; + let mut bytes = Vec::with_capacity(56); + bytes.push(kind); + bytes.extend_from_slice(&[0; 7]); + push_u64(&mut bytes, generation); + push_u64(&mut bytes, byte_length); + bytes.extend_from_slice(&digest); + require_length(&bytes, 56, "migration inventory entry")?; + let kind_name = match kind { + 1 => "segment", + 2 => "catalog", + _ => return Err("unregistered inventory artifact kind".to_owned()), + }; + Ok(InventoryRow { + kind: kind_name, + generation, + byte_length, + artifact_digest: digest, + source_fixture, + bytes: <[u8; 56]>::try_from(bytes) + .map_err(|_| "inventory entry conversion failed".to_owned())?, + }) +} + +fn migration_source( + definition_digest: [u8; 32], + inventory_digest: [u8; 32], +) -> Result { + let head = decode_hex(V1_HEAD)?; + let catalog = decode_hex(V1_CATALOG)?; + require_length(&head, 128, "version-1 source head")?; + let catalog_generation = u64_at(&head, 24)?; + let catalog_length = u64_at(&head, 32)?; + let catalog_digest = array_32(&head, 40)?; + let predecessor_digest = array_32(&catalog, 32)?; + let store_id = hash( + STORE_ID_DOMAIN, + &[ + &catalog_generation.to_be_bytes(), + &catalog_length.to_be_bytes(), + &catalog_digest, + &predecessor_digest, + &inventory_digest, + &definition_digest, + ], + ); + Ok(MigrationSource { + catalog_generation, + catalog_length, + catalog_digest, + predecessor_digest, + inventory_digest, + definition_digest, + store_id, + root_device: 1, + root_mount: 2, + root_file: 3, + }) +} + +include!("artifacts/migration.rs"); +include!("artifacts/retention.rs"); diff --git a/xtask/tests/retention_store_v2_format_oracle/artifacts/migration.rs b/xtask/tests/retention_store_v2_format_oracle/artifacts/migration.rs new file mode 100644 index 0000000..4120258 --- /dev/null +++ b/xtask/tests/retention_store_v2_format_oracle/artifacts/migration.rs @@ -0,0 +1,96 @@ +// This included source owns construction of the format marker and migration records. + +fn build_format_marker(definition_digest: [u8; 32]) -> Result { + let mut bytes = Vec::with_capacity(96); + bytes.extend_from_slice(b"KEEP:STORE:V2\0\0\0"); + push_u16(&mut bytes, 2); + push_u16(&mut bytes, 96); + push_u32(&mut bytes, 0); + bytes.extend_from_slice(&definition_digest); + push_u32(&mut bytes, 4_096); + push_u32(&mut bytes, 0); + require_length(&bytes, 64, "format-marker checksum preimage")?; + let checksum = hash(b"keep.segment-store-marker-checksum/v2\0", &[&bytes]); + bytes.extend_from_slice(&checksum); + require_length(&bytes, 96, "format marker")?; + let marker_digest = hash(b"keep.store-format-marker/v2\0", &[&bytes]); + Ok(Artifact { + case_name: "format-marker", + kind: "format-marker", + generation: "-", + entry_count: "-", + bound_digest: marker_digest, + final_checksum: checksum, + fixture: "format-marker.hex", + bytes, + }) +} + +fn build_migration_intent(source: &MigrationSource) -> Result { + let mut bytes = Vec::with_capacity(256); + bytes.extend_from_slice(b"KEEP:MIG:INT2\0\0\0"); + push_u16(&mut bytes, 2); + push_u16(&mut bytes, 256); + push_u32(&mut bytes, 0); + push_u64(&mut bytes, source.catalog_generation); + push_u64(&mut bytes, source.catalog_length); + bytes.extend_from_slice(&source.catalog_digest); + bytes.extend_from_slice(&source.predecessor_digest); + bytes.extend_from_slice(&source.inventory_digest); + push_u64(&mut bytes, source.root_device); + push_u64(&mut bytes, source.root_mount); + push_u64(&mut bytes, source.root_file); + bytes.extend_from_slice(&source.definition_digest); + bytes.extend_from_slice(&source.store_id); + require_length(&bytes, 224, "migration-intent checksum preimage")?; + let checksum = hash(b"keep.store-migration-intent-checksum/v2\0", &[&bytes]); + bytes.extend_from_slice(&checksum); + require_length(&bytes, 256, "migration intent")?; + let intent_digest = hash(b"keep.store-migration-intent/v2\0", &[&bytes]); + Ok(Artifact { + case_name: "migration-intent", + kind: "migration-intent", + generation: "1", + entry_count: "2", + bound_digest: intent_digest, + final_checksum: checksum, + fixture: "migration-intent.hex", + bytes, + }) +} + +fn build_migration_receipt( + source: &MigrationSource, + format: &Artifact, + intent: &Artifact, +) -> Result { + let initial_retention = hash(b"keep.initial-retention-state/v2\0", &[]); + let initial_gc = hash(b"keep.initial-gc-state/v2\0", &[]); + let empty_dispositions = hash(b"keep.empty-disposition-set/v2\0", &[]); + let mut bytes = Vec::with_capacity(256); + bytes.extend_from_slice(b"KEEP:MIG:REC2\0\0\0"); + push_u16(&mut bytes, 2); + push_u16(&mut bytes, 256); + push_u32(&mut bytes, 0); + bytes.extend_from_slice(&intent.bound_digest); + bytes.extend_from_slice(&source.store_id); + bytes.extend_from_slice(&format.bound_digest); + bytes.extend_from_slice(&initial_retention); + bytes.extend_from_slice(&initial_gc); + bytes.extend_from_slice(&empty_dispositions); + push_u64(&mut bytes, 0x03ff); + require_length(&bytes, 224, "migration-receipt checksum preimage")?; + let checksum = hash(b"keep.store-migration-receipt-checksum/v2\0", &[&bytes]); + bytes.extend_from_slice(&checksum); + require_length(&bytes, 256, "migration receipt")?; + Ok(Artifact { + case_name: "migration-receipt", + kind: "migration-receipt", + generation: "1", + entry_count: "2", + bound_digest: intent.bound_digest, + final_checksum: checksum, + fixture: "migration-receipt.hex", + bytes, + }) +} diff --git a/xtask/tests/retention_store_v2_format_oracle/artifacts/retention.rs b/xtask/tests/retention_store_v2_format_oracle/artifacts/retention.rs new file mode 100644 index 0000000..44ad067 --- /dev/null +++ b/xtask/tests/retention_store_v2_format_oracle/artifacts/retention.rs @@ -0,0 +1,155 @@ +// This included source owns construction of retention root, manifest, and head records. + +const NAMESPACE: &[u8] = &[0x00, 0x2f, 0xff]; +const BLOB_ID: [u8; 59] = [ + 0x4b, 0x45, 0x45, 0x50, 0x3a, 0x42, 0x4c, 0x4f, 0x42, 0x3a, 0x49, 0x44, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x01, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x1c, + 0xfb, 0x8f, 0xa9, 0xe9, 0x17, 0xab, 0xa1, 0x5a, 0x1f, 0x59, 0x20, 0x95, 0xf3, 0x77, + 0xff, 0x18, 0x07, 0x55, 0xfe, 0x12, 0x12, 0xb0, 0xd7, 0xd2, 0xec, 0x75, 0x0b, 0xd1, + 0x28, 0xb6, 0x06, +]; +const LAYOUT_ID: [u8; 60] = [ + 0x4b, 0x45, 0x45, 0x50, 0x3a, 0x4c, 0x41, 0x59, 0x4f, 0x55, 0x54, 0x3a, 0x49, 0x44, + 0x00, 0x00, 0x00, 0x01, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xdc, + 0x88, 0x7d, 0xa2, 0x3f, 0x1a, 0x74, 0x83, 0x35, 0x9a, 0x78, 0xfc, 0x9a, 0x7f, 0xde, + 0x80, 0x03, 0x0e, 0xc2, 0xc4, 0x69, 0x06, 0x03, 0x80, 0x3f, 0x0a, 0xb7, 0xd0, 0xed, + 0xb5, 0x65, 0x75, 0xb8, +]; + +fn build_retention_root(profile_digest: [u8; 32]) -> Result { + let mut anchor = Vec::with_capacity(119); + anchor.extend_from_slice(&BLOB_ID); + anchor.extend_from_slice(&LAYOUT_ID); + require_length(&anchor, 119, "retention anchor")?; + let anchor_count = 1u32.to_be_bytes(); + let anchor_set_digest = hash( + b"keep.retention-anchor-set/v2\0", + &[&anchor_count, &anchor], + ); + let mut header = root_header(profile_digest, anchor_set_digest)?; + let mut body = NAMESPACE.to_vec(); + body.extend_from_slice(&anchor); + let digest = hash(b"keep.retention-root/v2\0", &[&header, &body]); + let checksum = hash( + b"keep.retention-root-checksum/v2\0", + &[&header, &body, &digest], + ); + header.extend_from_slice(&body); + header.extend_from_slice(&digest); + header.extend_from_slice(&checksum); + require_length(&header, 378, "retention root")?; + let namespace_length = + u16::try_from(NAMESPACE.len()).map_err(|_| "namespace length overflow".to_owned())?; + let namespace_digest = hash( + b"keep.retention-namespace/v1\0", + &[&namespace_length.to_be_bytes(), NAMESPACE], + ); + Ok(RootArtifact { + bytes: header, + digest, + checksum, + namespace_digest, + }) +} + +fn root_header( + profile_digest: [u8; 32], + anchor_set_digest: [u8; 32], +) -> Result, String> { + let mut bytes = Vec::with_capacity(192); + bytes.extend_from_slice(b"KEEP:RET:ROOT2\0\0"); + push_u16(&mut bytes, 2); + push_u16(&mut bytes, 192); + push_u32(&mut bytes, 0); + push_u64(&mut bytes, 378); + push_u64(&mut bytes, 1); + push_u16(&mut bytes, 3); + push_u16(&mut bytes, 119); + push_u32(&mut bytes, 1); + push_u32(&mut bytes, 1); + push_u32(&mut bytes, 1); + bytes.extend_from_slice(&profile_digest); + push_u64(&mut bytes, 4); + push_u16(&mut bytes, 2); + push_u16(&mut bytes, 0); + push_u64(&mut bytes, 4_096); + push_u64(&mut bytes, 4_096); + bytes.extend_from_slice(&[0; 32]); + bytes.extend_from_slice(&anchor_set_digest); + bytes.extend_from_slice(&[0; 12]); + require_length(&bytes, 192, "retention-root header")?; + Ok(bytes) +} + +fn build_retention_manifest(root: &RootArtifact) -> Result { + let mut entry = Vec::with_capacity(72); + entry.extend_from_slice(&root.namespace_digest); + push_u64(&mut entry, 1); + entry.extend_from_slice(&root.digest); + require_length(&entry, 72, "retention manifest entry")?; + let entry_count = 1u32.to_be_bytes(); + let entry_set_digest = hash( + b"keep.retention-manifest-entries/v2\0", + &[&entry_count, &entry], + ); + let mut header = manifest_header(entry_set_digest)?; + let digest = hash(b"keep.retention-manifest/v2\0", &[&header, &entry]); + let checksum = hash( + b"keep.retention-manifest-checksum/v2\0", + &[&header, &entry, &digest], + ); + header.extend_from_slice(&entry); + header.extend_from_slice(&digest); + header.extend_from_slice(&checksum); + require_length(&header, 296, "retention manifest")?; + Ok(ManifestArtifact { + bytes: header, + digest, + checksum, + }) +} + +fn manifest_header(entry_set_digest: [u8; 32]) -> Result, String> { + let mut bytes = Vec::with_capacity(160); + bytes.extend_from_slice(b"KEEP:RET:LIVE2\0\0"); + push_u16(&mut bytes, 2); + push_u16(&mut bytes, 160); + push_u32(&mut bytes, 0); + push_u64(&mut bytes, 296); + push_u64(&mut bytes, 1); + push_u16(&mut bytes, 72); + push_u16(&mut bytes, 0); + push_u32(&mut bytes, 1); + bytes.extend_from_slice(&[0; 32]); + bytes.extend_from_slice(&entry_set_digest); + bytes.extend_from_slice(&[0; 48]); + require_length(&bytes, 160, "retention-manifest header")?; + Ok(bytes) +} + +fn build_retention_head(manifest: &ManifestArtifact) -> Result { + let mut bytes = Vec::with_capacity(144); + bytes.extend_from_slice(b"KEEP:RET:HEAD2\0\0"); + push_u16(&mut bytes, 2); + push_u16(&mut bytes, 144); + push_u32(&mut bytes, 0); + push_u64(&mut bytes, 1); + push_u64(&mut bytes, 296); + bytes.extend_from_slice(&manifest.digest); + bytes.extend_from_slice(&[0; 32]); + push_u64(&mut bytes, 0); + require_length(&bytes, 112, "retention-head checksum preimage")?; + let checksum = hash(b"keep.retention-head-checksum/v2\0", &[&bytes]); + bytes.extend_from_slice(&checksum); + require_length(&bytes, 144, "retention head")?; + Ok(Artifact { + case_name: "one-root-head", + kind: "retention-head", + generation: "1", + entry_count: "1", + bound_digest: manifest.digest, + final_checksum: checksum, + fixture: "one-root-head.hex", + bytes, + }) +} diff --git a/xtask/tests/retention_store_v2_format_oracle/encoding.rs b/xtask/tests/retention_store_v2_format_oracle/encoding.rs new file mode 100644 index 0000000..2275b17 --- /dev/null +++ b/xtask/tests/retention_store_v2_format_oracle/encoding.rs @@ -0,0 +1,126 @@ +// This included source owns primitive binary construction and fixture transport. + +use std::fmt::Write as _; +use std::fs; +use std::io; +use std::path::{Path, PathBuf}; + +fn hash(domain: &[u8], parts: &[&[u8]]) -> [u8; 32] { + let mut hasher = blake3::Hasher::new(); + hasher.update(domain); + for part in parts { + hasher.update(part); + } + *hasher.finalize().as_bytes() +} + +fn push_u16(bytes: &mut Vec, value: u16) { + bytes.extend_from_slice(&value.to_be_bytes()); +} + +fn push_u32(bytes: &mut Vec, value: u32) { + bytes.extend_from_slice(&value.to_be_bytes()); +} + +fn push_u64(bytes: &mut Vec, value: u64) { + bytes.extend_from_slice(&value.to_be_bytes()); +} + +fn array_32(bytes: &[u8], offset: usize) -> Result<[u8; 32], String> { + let end = offset + .checked_add(32) + .ok_or_else(|| "32-byte field offset overflow".to_owned())?; + let field = bytes + .get(offset..end) + .ok_or_else(|| format!("missing 32-byte field at offset {offset}"))?; + <[u8; 32]>::try_from(field).map_err(|_| "32-byte field conversion failed".to_owned()) +} + +fn u64_at(bytes: &[u8], offset: usize) -> Result { + let end = offset + .checked_add(8) + .ok_or_else(|| "u64 field offset overflow".to_owned())?; + let field = bytes + .get(offset..end) + .ok_or_else(|| format!("missing u64 field at offset {offset}"))?; + let encoded = + <[u8; 8]>::try_from(field).map_err(|_| "u64 field conversion failed".to_owned())?; + Ok(u64::from_be_bytes(encoded)) +} + +fn decode_hex(source: &str) -> Result, String> { + let encoded = source + .strip_suffix('\n') + .ok_or_else(|| "hex fixture lacks one final newline".to_owned())?; + if encoded.contains('\n') || encoded.contains('\r') { + return Err("hex fixture contains embedded line ending".to_owned()); + } + if encoded.len() % 2 != 0 { + return Err("hex fixture has odd encoded length".to_owned()); + } + encoded + .as_bytes() + .chunks_exact(2) + .map(|pair| { + let [high_byte, low_byte] = <[u8; 2]>::try_from(pair) + .map_err(|_| "hex pair conversion failed".to_owned())?; + let high = hex_nibble(high_byte)?; + let low = hex_nibble(low_byte)?; + Ok((high << 4) | low) + }) + .collect() +} + +fn hex_nibble(byte: u8) -> Result { + match byte { + b'0'..=b'9' => byte + .checked_sub(b'0') + .ok_or_else(|| "decimal hex nibble underflow".to_owned()), + b'a'..=b'f' => byte + .checked_sub(b'a') + .and_then(|value| value.checked_add(10)) + .ok_or_else(|| "alphabetic hex nibble overflow".to_owned()), + _ => Err("hex fixture contains a non-lowercase hexadecimal byte".to_owned()), + } +} + +fn encode_hex(bytes: &[u8]) -> Result { + let capacity = bytes + .len() + .checked_mul(2) + .and_then(|length| length.checked_add(1)) + .ok_or_else(|| "hex output length overflow".to_owned())?; + let mut encoded = String::with_capacity(capacity); + for byte in bytes { + write!(&mut encoded, "{byte:02x}") + .map_err(|_| "hex output formatting failed".to_owned())?; + } + encoded.push('\n'); + Ok(encoded) +} + +fn repository_root() -> Result { + Path::new(env!("CARGO_MANIFEST_DIR")) + .parent() + .map(Path::to_path_buf) + .ok_or_else(|| io::Error::other("xtask manifest directory has no parent")) +} + +fn corpus_path(relative: &str) -> Result { + Ok(repository_root()?.join(CORPUS_ROOT).join(relative)) +} + +fn read_corpus_file(relative: &str) -> Result { + fs::read_to_string(corpus_path(relative)?) +} + +fn require_length(bytes: &[u8], expected: usize, name: &str) -> Result<(), String> { + if bytes.len() == expected { + Ok(()) + } else { + Err(format!( + "{name} has {} bytes; expected {expected}", + bytes.len() + )) + } +} diff --git a/xtask/tests/retention_store_v2_format_oracle/fixture_assertion.rs b/xtask/tests/retention_store_v2_format_oracle/fixture_assertion.rs new file mode 100644 index 0000000..88b3570 --- /dev/null +++ b/xtask/tests/retention_store_v2_format_oracle/fixture_assertion.rs @@ -0,0 +1,198 @@ +// This included source owns assertions over constructed bytes and canonical tables. + +#[test] +fn golden_artifacts_match_the_independent_oracle() -> Result<(), String> { + let corpus = build_corpus()?; + let expected_manifest = artifacts_table(&corpus.artifacts)?; + assert_eq!( + read_corpus_file("artifacts.tsv").map_err(|error| error.to_string())?, + expected_manifest + ); + for artifact in &corpus.artifacts { + let fixture = + read_corpus_file(artifact.fixture).map_err(|error| error.to_string())?; + assert_eq!( + fixture, + encode_hex(&artifact.bytes)?, + "golden fixture drifted: {}", + artifact.fixture + ); + } + Ok(()) +} + +#[test] +fn definition_profile_and_migration_sources_match_the_oracle() -> Result<(), String> { + let corpus = build_corpus()?; + let profile_hex = encode_digest(&corpus.profile_digest)?; + let definition_hex = encode_digest(&corpus.definition_digest)?; + assert!( + FORMAT_DEFINITION.contains(&format!("retention.profile.digest\t{profile_hex}")), + "format definition does not bind the exact profile digest" + ); + assert_eq!( + corpus.definition_digest, corpus.migration.definition_digest, + "migration source does not bind the exact format definition" + ); + assert_eq!( + read_corpus_file("inventory.tsv").map_err(|error| error.to_string())?, + inventory_table(&corpus.inventory)? + ); + assert_eq!( + read_corpus_file("migration-source.tsv").map_err(|error| error.to_string())?, + migration_source_table(&corpus.migration)? + ); + for documentation in ["README.md", "ORIGIN.md"] { + assert!( + read_corpus_file(documentation) + .map_err(|error| error.to_string())? + .contains(&definition_hex), + "{documentation} does not name the exact format-definition digest" + ); + } + Ok(()) +} + +#[test] +fn definition_and_profile_tables_are_exact_and_canonical() -> Result<(), String> { + assert_eq!( + PROFILE_DEFINITION, + "keep.retention-realization-profiles/v1\n\ + identity\tversion\tcanonical_name\twitness_count\tselection\n\ + 1\t1\tkeep.retention-single-canonical-witness/v1\t1\t\ + canonical-physical-catalog-coordinate\n" + ); + assert!(FORMAT_DEFINITION.ends_with('\n')); + assert!(!FORMAT_DEFINITION.contains('\r')); + let mut rows = FORMAT_DEFINITION.lines(); + assert_eq!(rows.next(), Some("keep.segment-store.definition/v2")); + assert_eq!(rows.next(), Some("key\tvalue")); + let mut previous: Option<&str> = None; + for row in rows { + let (key, value) = row + .split_once('\t') + .ok_or_else(|| format!("definition row lacks one key/value boundary: {row}"))?; + assert!(!key.is_empty(), "definition row has an empty key"); + assert!(!value.is_empty(), "definition row has an empty value"); + assert!( + !value.contains('\t'), + "definition row has more than one key/value boundary: {row}" + ); + if let Some(prior) = previous { + assert!(prior < key, "definition keys are not strictly sorted"); + } + previous = Some(key); + } + Ok(()) +} + +#[test] +fn retention_anchor_ids_are_derived_from_the_accepted_layout_corpus() -> Result<(), String> { + let row = LAYOUTS + .lines() + .find(|line| line.starts_with("one-zero\t")) + .ok_or_else(|| "layout corpus lacks the one-zero case".to_owned())?; + let fields: Vec<&str> = row.split('\t').collect(); + let blob_id = fields + .get(5) + .ok_or_else(|| "one-zero layout row lacks BlobId text".to_owned())?; + assert_eq!( + *blob_id, + "keep:blob:v1:blake3-256:1:\ + 1cfb8fa9e917aba15a1f592095f377ff180755fe1212b0d7d2ec750bd128b606" + ); + let blob_digest = blob_id + .rsplit(':') + .next() + .ok_or_else(|| "one-zero BlobId lacks a digest".to_owned())?; + assert_eq!( + BLOB_ID.get(..16), + Some(b"KEEP:BLOB:ID\0\0\0\0".as_slice()) + ); + assert_eq!(BLOB_ID.get(16..18), Some([0_u8, 1].as_slice())); + assert_eq!(BLOB_ID.get(18), Some(&1)); + assert_eq!(u64_at(&BLOB_ID, 19)?, 1); + assert_eq!( + BLOB_ID.get(27..), + Some(decode_hex(&format!("{blob_digest}\n"))?.as_slice()) + ); + let layout_id_binary = encode_hex(&LAYOUT_ID)?; + assert_eq!( + fields.get(11).copied(), + Some(layout_id_binary.trim_end()) + ); + Ok(()) +} + +fn artifacts_table(artifacts: &[Artifact]) -> Result { + let mut table = String::from( + "keep.segment-store-v2.artifacts/v1\n\ + case\tkind\tbyte_length\tgeneration\tentry_count\tbound_digest_hex\t\ + final_checksum_hex\tfixture\n", + ); + for artifact in artifacts { + writeln!( + table, + "{}\t{}\t{}\t{}\t{}\t{}\t{}\t{}", + artifact.case_name, + artifact.kind, + artifact.bytes.len(), + artifact.generation, + artifact.entry_count, + encode_digest(&artifact.bound_digest)?, + encode_digest(&artifact.final_checksum)?, + artifact.fixture + ) + .map_err(|_| "artifact table formatting failed".to_owned())?; + } + Ok(table) +} + +fn inventory_table(inventory: &Inventory) -> Result { + let mut table = String::from( + "keep.segment-store-v2.inventory/v1\n\ + kind\tgeneration\tbyte_length\tartifact_digest_hex\tsource_fixture\n", + ); + for row in &inventory.rows { + writeln!( + table, + "{}\t{}\t{}\t{}\t{}", + row.kind, + row.generation, + row.byte_length, + encode_digest(&row.artifact_digest)?, + row.source_fixture + ) + .map_err(|_| "inventory table formatting failed".to_owned())?; + } + Ok(table) +} + +fn migration_source_table(source: &MigrationSource) -> Result { + let mut table = String::from( + "keep.segment-store-v2.migration-source/v1\n\ + case\tcatalog_generation\tcatalog_length\tcatalog_digest_hex\t\ + predecessor_digest_hex\tinventory_digest_hex\tdefinition_digest_hex\t\ + store_id_hex\troot_device\troot_mount\troot_file\n", + ); + writeln!( + table, + "one-zero\t{}\t{}\t{}\t{}\t{}\t{}\t{}\t{}\t{}\t{}", + source.catalog_generation, + source.catalog_length, + encode_digest(&source.catalog_digest)?, + encode_digest(&source.predecessor_digest)?, + encode_digest(&source.inventory_digest)?, + encode_digest(&source.definition_digest)?, + encode_digest(&source.store_id)?, + source.root_device, + source.root_mount, + source.root_file + ) + .map_err(|_| "migration-source table formatting failed".to_owned())?; + Ok(table) +} + +fn encode_digest(digest: &[u8; 32]) -> Result { + encode_hex(digest).map(|encoded| encoded.trim_end().to_owned()) +} diff --git a/xtask/tests/retention_store_v2_protocol_contract.rs b/xtask/tests/retention_store_v2_protocol_contract.rs index 7c80151..34ebdcb 100644 --- a/xtask/tests/retention_store_v2_protocol_contract.rs +++ b/xtask/tests/retention_store_v2_protocol_contract.rs @@ -45,6 +45,7 @@ fn version_two_is_one_routed_protocol() -> Result<(), Box "[GC and disposition records](gc.md)", "[Migration and recovery](recovery.md)", "[Migration crash points](migration-crash.md)", + "[Migration inventory](migration-inventory.md)", "[Requirements and evidence](requirements.md)", "[Format rationale](rationale.md)", ] { @@ -68,6 +69,9 @@ fn retention_records_have_exact_canonical_grammars() -> Result<(), Box Result<(), Box Result<(), Box Result<(), Box Date: Wed, 29 Jul 2026 20:00:18 -0700 Subject: [PATCH 003/111] Add: Validate core retention values --- CHANGELOG.md | 9 ++- docs/formats/segment-store-v2/README.md | 9 ++- docs/formats/segment-store-v2/requirements.md | 2 +- src/lib.rs | 11 ++- src/retention/anchor.rs | 35 ++++++++ src/retention/liveness_generation.rs | 46 +++++++++++ src/retention/liveness_generation_error.rs | 30 +++++++ src/retention/mod.rs | 24 ++++++ src/retention/namespace.rs | 78 ++++++++++++++++++ src/retention/namespace_digest.rs | 21 +++++ src/retention/namespace_error.rs | 32 ++++++++ src/retention/root_generation.rs | 46 +++++++++++ src/retention/root_generation_error.rs | 30 +++++++ tests/retention_values.rs | 79 +++++++++++++++++++ 14 files changed, 442 insertions(+), 10 deletions(-) create mode 100644 src/retention/anchor.rs create mode 100644 src/retention/liveness_generation.rs create mode 100644 src/retention/liveness_generation_error.rs create mode 100644 src/retention/mod.rs create mode 100644 src/retention/namespace.rs create mode 100644 src/retention/namespace_digest.rs create mode 100644 src/retention/namespace_error.rs create mode 100644 src/retention/root_generation.rs create mode 100644 src/retention/root_generation_error.rs create mode 100644 tests/retention_values.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index 0da89d9..dd30ed6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -305,9 +305,12 @@ after its public API and format compatibility policies are established. - Specified `keep.segment-store/v2` retention values, root generations, liveness manifests, reader snapshots, one-way staged migration, exact crash - boundaries, and reserved GC/disposition records. Version-1 immutable bytes - remain authoritative; production version-2 writing remains unavailable until - issue #19's executable evidence is complete. + boundaries, and reserved GC/disposition records. Validated public + `RetentionNamespace`, namespace-digest, `RootGeneration`, + `LivenessGeneration`, and `RetentionAnchor` values now establish the core + boundary. Version-1 immutable bytes remain authoritative; production + version-2 writing remains unavailable until issue #19's executable evidence + is complete. - Accepted ADR-0009 defines caller-supplied retention namespaces, `BlobId`/`LayoutId` reconstruction anchors, fail-closed canonical closure, generation-checked retention publication, immutable liveness snapshots, diff --git a/docs/formats/segment-store-v2/README.md b/docs/formats/segment-store-v2/README.md index 4d7ea1b..3fe3d8b 100644 --- a/docs/formats/segment-store-v2/README.md +++ b/docs/formats/segment-store-v2/README.md @@ -66,8 +66,11 @@ re-encode them. ## Status -The format contract is frozen by ADR-0009 and this specification. Requirements -marked **Planned in #19** or **Planned in #21** are not implementation evidence. -A store must refuse version-2 state until the relevant parser, corruption, +The format contract is frozen by ADR-0009 and this specification. Public core +types now admit exact namespace bytes, namespace digests, root and liveness +generations, and reconstruction anchors. No production version-2 parser, +transition, migration, or writer exists yet. Requirements that remain marked +as planned in issue #19 or issue #21 are not implementation evidence. A store +must refuse version-2 state until the relevant parser, corruption, golden-format, model-based, crash-injection, recovery, and fuzz evidence is implemented. diff --git a/docs/formats/segment-store-v2/requirements.md b/docs/formats/segment-store-v2/requirements.md index b011772..e57e60b 100644 --- a/docs/formats/segment-store-v2/requirements.md +++ b/docs/formats/segment-store-v2/requirements.md @@ -9,7 +9,7 @@ case is not evidence. | ID | Requirement | Evidence | Status | | --- | --- | --- | --- | -| `KEEP-RETENTION-001` | `RetentionNamespace`, `RootGeneration`, `LivenessGeneration`, profile coordinates, limits, anchors, and digests are validated typed values | unit and public API tests | Planned in #19 | +| `KEEP-RETENTION-001` | `RetentionNamespace`, `RootGeneration`, `LivenessGeneration`, profile coordinates, limits, anchors, and digests are validated typed values | `tests/retention_values.rs` covers namespace, digest, generations, and anchors; profile and limit evidence remains | In progress in #19 | | `KEEP-RETENTION-002` | Root, manifest, and head codecs implement the exact canonical grammars and fixed bounds | golden-format fixtures plus independent oracle | Planned in #19 | | `KEEP-RETENTION-003` | Every structural field, truncation boundary, ordering law, duplicate, overflow, flag, reserved byte, digest, checksum, and trailing byte has a precise refusal | corruption matrix and fuzz | Planned in #19 | | `KEEP-RETENTION-004` | Retain and release compare expected and observed generations and publish exact successors only | unit and model-based transition tests | Planned in #19 | diff --git a/src/lib.rs b/src/lib.rs index 84b5b78..88026b5 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -20,9 +20,9 @@ //! explicit. Exact next-head finalization now has a storage-independent //! contract and a pinned writer-authorized filesystem adapter. Reusable-stage //! continuation has a storage-independent planning and execution boundary plus -//! a pinned writer-authorized filesystem adapter. Retention and garbage -//! collection remain intentionally absent until their contracts have -//! executable specifications. +//! a pinned writer-authorized filesystem adapter. Core retention namespaces, +//! generations, and reconstruction anchors are validated. Retention +//! publication, recovery, and garbage collection remain intentionally absent. #[cfg(test)] extern crate self as keep; @@ -34,6 +34,7 @@ mod chunk; mod layout; mod profile; mod reference; +mod retention; #[cfg(feature = "repository-tasks")] #[doc(hidden)] @@ -119,3 +120,7 @@ pub use reference::{ RangeReadError, RangeReadReceipt, ReconstructionError, ReconstructionReceipt, ReferenceStore, ReferenceStoreCapacity, StagedBlob, }; +pub use retention::{ + LivenessGeneration, LivenessGenerationError, RetentionAnchor, RetentionNamespace, + RetentionNamespaceDigest, RetentionNamespaceError, RootGeneration, RootGenerationError, +}; diff --git a/src/retention/anchor.rs b/src/retention/anchor.rs new file mode 100644 index 0000000..f30e799 --- /dev/null +++ b/src/retention/anchor.rs @@ -0,0 +1,35 @@ +//! This module owns one typed logical reconstruction anchor. + +use crate::blob::BlobId; +use crate::layout::LayoutId; + +/// Exact logical blob and canonical layout coordinates retained together. +/// +/// Construction cannot fail because both component identities are already +/// validated. An anchor proves only the requested coordinates; closure +/// admission must separately prove the named layout, chunks, and blob bytes. +#[must_use] +#[derive(Clone, Copy, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)] +pub struct RetentionAnchor { + blob_id: BlobId, + layout_id: LayoutId, +} + +impl RetentionAnchor { + /// Combines one validated logical blob and layout coordinate. + pub const fn new(blob_id: BlobId, layout_id: LayoutId) -> Self { + Self { blob_id, layout_id } + } + + /// Returns the exact retained logical blob coordinate. + #[must_use] + pub const fn blob_id(self) -> BlobId { + self.blob_id + } + + /// Returns the exact retained layout coordinate. + #[must_use] + pub const fn layout_id(self) -> LayoutId { + self.layout_id + } +} diff --git a/src/retention/liveness_generation.rs b/src/retention/liveness_generation.rs new file mode 100644 index 0000000..3936c5d --- /dev/null +++ b/src/retention/liveness_generation.rs @@ -0,0 +1,46 @@ +//! This module owns checked global retention liveness generations. + +use std::num::NonZeroU64; + +use super::LivenessGenerationError; + +/// Positive generation of the global retention manifest. +/// +/// This coordinate is deliberately distinct from every per-namespace +/// [`RootGeneration`](super::RootGeneration). +#[must_use] +#[derive(Clone, Copy, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)] +pub struct LivenessGeneration(NonZeroU64); + +impl LivenessGeneration { + /// Admits one positive liveness generation. + /// + /// # Errors + /// + /// Returns [`LivenessGenerationError::Zero`] when `value` is zero. + pub const fn new(value: u64) -> Result { + match NonZeroU64::new(value) { + Some(value) => Ok(Self(value)), + None => Err(LivenessGenerationError::Zero), + } + } + + /// Returns the exact positive generation. + #[must_use] + pub const fn get(self) -> u64 { + self.0.get() + } + + /// Derives the exact successor through checked addition. + /// + /// # Errors + /// + /// Returns [`LivenessGenerationError::Exhausted`] at `u64::MAX`. + pub const fn successor(self) -> Result { + let current = self.get(); + let Some(next) = current.checked_add(1) else { + return Err(LivenessGenerationError::Exhausted { current }); + }; + Self::new(next) + } +} diff --git a/src/retention/liveness_generation_error.rs b/src/retention/liveness_generation_error.rs new file mode 100644 index 0000000..e48d594 --- /dev/null +++ b/src/retention/liveness_generation_error.rs @@ -0,0 +1,30 @@ +//! This module owns typed retention liveness-generation failures. + +use std::error::Error; +use std::fmt; + +/// Failure to admit or advance a retention liveness generation. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum LivenessGenerationError { + /// Generation zero is outside the version-2 protocol. + Zero, + /// The current generation has no representable successor. + Exhausted { + /// Exact generation that could not advance. + current: u64, + }, +} + +impl fmt::Display for LivenessGenerationError { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::Zero => formatter.write_str("retention liveness generation must be positive"), + Self::Exhausted { current } => write!( + formatter, + "retention liveness generation {current} has no successor" + ), + } + } +} + +impl Error for LivenessGenerationError {} diff --git a/src/retention/mod.rs b/src/retention/mod.rs new file mode 100644 index 0000000..43494e9 --- /dev/null +++ b/src/retention/mod.rs @@ -0,0 +1,24 @@ +//! Semantic retention coordinates and reconstruction anchors. +//! +//! This module owns validated namespace bytes, namespace identity, +//! generation coordinates, and logical reconstruction anchors. It does not own +//! record encoding, filesystem layout, publication, recovery, or garbage +//! collection. + +mod anchor; +mod liveness_generation; +mod liveness_generation_error; +mod namespace; +mod namespace_digest; +mod namespace_error; +mod root_generation; +mod root_generation_error; + +pub use anchor::RetentionAnchor; +pub use liveness_generation::LivenessGeneration; +pub use liveness_generation_error::LivenessGenerationError; +pub use namespace::RetentionNamespace; +pub use namespace_digest::RetentionNamespaceDigest; +pub use namespace_error::RetentionNamespaceError; +pub use root_generation::RootGeneration; +pub use root_generation_error::RootGenerationError; diff --git a/src/retention/namespace.rs b/src/retention/namespace.rs new file mode 100644 index 0000000..01063b6 --- /dev/null +++ b/src/retention/namespace.rs @@ -0,0 +1,78 @@ +//! This module owns admission and identity of opaque retention namespace bytes. + +use std::num::NonZeroU8; + +use super::{RetentionNamespaceDigest, RetentionNamespaceError}; + +const DIGEST_DOMAIN: &[u8] = b"keep.retention-namespace/v1\0"; + +/// One validated opaque retention authority namespace. +/// +/// Every nonempty byte string through 255 bytes is canonical as-is. Admission +/// performs no Unicode, path, case, or application-level interpretation. +/// +/// Constructing from a borrowed slice allocates one owned copy. Constructing +/// from a `Vec` consumes it; boxed-slice conversion may discard excess +/// capacity. +#[must_use] +#[derive(Clone, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)] +pub struct RetentionNamespace { + bytes: Box<[u8]>, + length: NonZeroU8, +} + +impl RetentionNamespace { + /// Maximum admitted namespace length in bytes. + pub const MAXIMUM_BYTE_LENGTH: u8 = u8::MAX; + + /// Returns the exact opaque namespace bytes. + #[must_use] + pub fn as_bytes(&self) -> &[u8] { + &self.bytes + } + + /// Derives the canonical physical namespace-directory identity. + /// + /// The digest binds the domain, the fixed-width big-endian byte length, + /// and the exact namespace bytes. This operation does not allocate. + pub fn digest(&self) -> RetentionNamespaceDigest { + let length = u16::from(self.length.get()).to_be_bytes(); + let mut hasher = blake3::Hasher::new(); + hasher.update(DIGEST_DOMAIN); + hasher.update(&length); + hasher.update(&self.bytes); + RetentionNamespaceDigest::from_hash(*hasher.finalize().as_bytes()) + } + + fn admit_length(observed: usize) -> Result { + let length = u8::try_from(observed).map_err(|_| RetentionNamespaceError::TooLong { + maximum: Self::MAXIMUM_BYTE_LENGTH, + observed, + })?; + NonZeroU8::new(length).ok_or(RetentionNamespaceError::Empty) + } +} + +impl TryFrom> for RetentionNamespace { + type Error = RetentionNamespaceError; + + fn try_from(bytes: Vec) -> Result { + let length = Self::admit_length(bytes.len())?; + Ok(Self { + bytes: bytes.into_boxed_slice(), + length, + }) + } +} + +impl TryFrom<&[u8]> for RetentionNamespace { + type Error = RetentionNamespaceError; + + fn try_from(bytes: &[u8]) -> Result { + let length = Self::admit_length(bytes.len())?; + Ok(Self { + bytes: Box::from(bytes), + length, + }) + } +} diff --git a/src/retention/namespace_digest.rs b/src/retention/namespace_digest.rs new file mode 100644 index 0000000..d68dda4 --- /dev/null +++ b/src/retention/namespace_digest.rs @@ -0,0 +1,21 @@ +//! This module owns the canonical retention namespace digest coordinate. + +/// Canonical BLAKE3-256 identity of one exact retention namespace. +/// +/// This coordinate selects a physical namespace directory. Authority still +/// requires the matching root record to contain the exact namespace bytes. +#[must_use] +#[derive(Clone, Copy, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)] +pub struct RetentionNamespaceDigest([u8; 32]); + +impl RetentionNamespaceDigest { + pub(super) const fn from_hash(bytes: [u8; 32]) -> Self { + Self(bytes) + } + + /// Returns the exact 32 digest bytes. + #[must_use] + pub const fn as_bytes(&self) -> &[u8; 32] { + &self.0 + } +} diff --git a/src/retention/namespace_error.rs b/src/retention/namespace_error.rs new file mode 100644 index 0000000..f266ec2 --- /dev/null +++ b/src/retention/namespace_error.rs @@ -0,0 +1,32 @@ +//! This module owns typed retention namespace admission failures. + +use std::error::Error; +use std::fmt; + +/// Failure to admit opaque retention namespace bytes. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum RetentionNamespaceError { + /// The namespace was empty. + Empty, + /// The namespace exceeded the version-2 byte ceiling. + TooLong { + /// Maximum admitted length. + maximum: u8, + /// Observed byte length. + observed: usize, + }, +} + +impl fmt::Display for RetentionNamespaceError { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::Empty => formatter.write_str("retention namespace must not be empty"), + Self::TooLong { maximum, observed } => write!( + formatter, + "retention namespace has {observed} bytes; maximum is {maximum}" + ), + } + } +} + +impl Error for RetentionNamespaceError {} diff --git a/src/retention/root_generation.rs b/src/retention/root_generation.rs new file mode 100644 index 0000000..488565c --- /dev/null +++ b/src/retention/root_generation.rs @@ -0,0 +1,46 @@ +//! This module owns checked retention root-generation coordinates. + +use std::num::NonZeroU64; + +use super::RootGenerationError; + +/// Positive generation of one retention namespace root. +/// +/// Generation `1` is initial. Empty retained sets still publish a successor; +/// generations are never reused. +#[must_use] +#[derive(Clone, Copy, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)] +pub struct RootGeneration(NonZeroU64); + +impl RootGeneration { + /// Admits one positive root generation. + /// + /// # Errors + /// + /// Returns [`RootGenerationError::Zero`] when `value` is zero. + pub const fn new(value: u64) -> Result { + match NonZeroU64::new(value) { + Some(value) => Ok(Self(value)), + None => Err(RootGenerationError::Zero), + } + } + + /// Returns the exact positive generation. + #[must_use] + pub const fn get(self) -> u64 { + self.0.get() + } + + /// Derives the exact successor through checked addition. + /// + /// # Errors + /// + /// Returns [`RootGenerationError::Exhausted`] at `u64::MAX`. + pub const fn successor(self) -> Result { + let current = self.get(); + let Some(next) = current.checked_add(1) else { + return Err(RootGenerationError::Exhausted { current }); + }; + Self::new(next) + } +} diff --git a/src/retention/root_generation_error.rs b/src/retention/root_generation_error.rs new file mode 100644 index 0000000..2c92989 --- /dev/null +++ b/src/retention/root_generation_error.rs @@ -0,0 +1,30 @@ +//! This module owns typed retention root-generation failures. + +use std::error::Error; +use std::fmt; + +/// Failure to admit or advance a retention root generation. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum RootGenerationError { + /// Generation zero is outside the version-2 protocol. + Zero, + /// The current generation has no representable successor. + Exhausted { + /// Exact generation that could not advance. + current: u64, + }, +} + +impl fmt::Display for RootGenerationError { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::Zero => formatter.write_str("retention root generation must be positive"), + Self::Exhausted { current } => write!( + formatter, + "retention root generation {current} has no successor" + ), + } + } +} + +impl Error for RootGenerationError {} diff --git a/tests/retention_values.rs b/tests/retention_values.rs new file mode 100644 index 0000000..8eea6a0 --- /dev/null +++ b/tests/retention_values.rs @@ -0,0 +1,79 @@ +//! Public laws for version-2 retention values. + +use keep::{ + BlobId, LayoutId, LivenessGeneration, LivenessGenerationError, RetentionAnchor, + RetentionNamespace, RetentionNamespaceError, RootGeneration, RootGenerationError, +}; + +const ONE_ZERO_BLOB: &str = concat!( + "keep:blob:v1:blake3-256:1:", + "1cfb8fa9e917aba15a1f592095f377ff180755fe1212b0d7d2ec750bd128b606" +); +const ONE_ZERO_LAYOUT: &str = concat!( + "keep:layout:v1:flat-chunks-v1:blake3-256:220:", + "887da23f1a7483359a78fc9a7fde80030ec2c4690603803f0ab7d0edb56575b8" +); + +#[test] +fn retention_namespaces_preserve_every_admitted_byte_and_bind_length() +-> Result<(), Box> { + assert_eq!( + RetentionNamespace::try_from(Vec::new()), + Err(RetentionNamespaceError::Empty) + ); + assert_eq!( + RetentionNamespace::try_from(vec![0_u8; 256]), + Err(RetentionNamespaceError::TooLong { + maximum: 255, + observed: 256, + }) + ); + + let bytes = [0x00, 0x2f, 0xff]; + let namespace = RetentionNamespace::try_from(bytes.as_slice())?; + assert_eq!(namespace.as_bytes(), bytes.as_slice()); + assert_eq!( + namespace.digest().as_bytes(), + &[ + 0xdd, 0xde, 0x2a, 0xc6, 0x5c, 0x5b, 0xa3, 0x82, 0x9b, 0xf0, 0xfb, 0xd6, 0xf3, 0x6e, + 0x90, 0x27, 0x2d, 0x69, 0xa0, 0x45, 0x9f, 0xad, 0xe9, 0x22, 0x72, 0xb7, 0x28, 0xa8, + 0x0d, 0x7a, 0xe6, 0xe2, + ] + ); + Ok(()) +} + +#[test] +fn retention_generations_are_positive_checked_and_semantically_distinct() +-> Result<(), Box> { + assert_eq!(RootGeneration::new(0), Err(RootGenerationError::Zero)); + assert_eq!( + LivenessGeneration::new(0), + Err(LivenessGenerationError::Zero) + ); + + let root = RootGeneration::new(1)?; + let liveness = LivenessGeneration::new(1)?; + assert_eq!(root.successor().map(RootGeneration::get), Ok(2)); + assert_eq!(liveness.successor().map(LivenessGeneration::get), Ok(2)); + assert_eq!( + RootGeneration::new(u64::MAX).and_then(RootGeneration::successor), + Err(RootGenerationError::Exhausted { current: u64::MAX }) + ); + assert_eq!( + LivenessGeneration::new(u64::MAX).and_then(LivenessGeneration::successor), + Err(LivenessGenerationError::Exhausted { current: u64::MAX }) + ); + Ok(()) +} + +#[test] +fn retention_anchors_preserve_exact_logical_and_layout_coordinates() +-> Result<(), Box> { + let blob: BlobId = ONE_ZERO_BLOB.parse()?; + let layout: LayoutId = ONE_ZERO_LAYOUT.parse()?; + let anchor = RetentionAnchor::new(blob, layout); + assert_eq!(anchor.blob_id(), blob); + assert_eq!(anchor.layout_id(), layout); + Ok(()) +} From 8200c5393ae48b604c7c18ef688a156ccd381d0a Mon Sep 17 00:00:00 2001 From: James Ross Date: Wed, 29 Jul 2026 20:21:06 -0700 Subject: [PATCH 004/111] Add: Encode canonical retention roots --- CHANGELOG.md | 9 +- docs/formats/segment-store-v2/README.md | 10 +- docs/formats/segment-store-v2/requirements.md | 4 +- src/adapters/mod.rs | 5 +- src/adapters/retention.rs | 8 + src/adapters/retention/canonical_root.rs | 42 ++++ src/adapters/retention/root_encode_error.rs | 48 +++++ src/adapters/retention/root_encoder.rs | 150 +++++++++++++++ src/lib.rs | 55 +++--- src/retention/closure_limit.rs | 27 +++ src/retention/closure_limit_error.rs | 43 +++++ src/retention/closure_limits.rs | 110 +++++++++++ src/retention/mod.rs | 18 ++ src/retention/policy.rs | 28 +++ src/retention/profile.rs | 75 ++++++++ src/retention/profile_admission_error.rs | 51 +++++ src/retention/root.rs | 128 +++++++++++++ src/retention/root_digest.rs | 18 ++ src/retention/root_error.rs | 59 ++++++ tests/retention_root_encoding.rs | 180 ++++++++++++++++++ 20 files changed, 1032 insertions(+), 36 deletions(-) create mode 100644 src/adapters/retention.rs create mode 100644 src/adapters/retention/canonical_root.rs create mode 100644 src/adapters/retention/root_encode_error.rs create mode 100644 src/adapters/retention/root_encoder.rs create mode 100644 src/retention/closure_limit.rs create mode 100644 src/retention/closure_limit_error.rs create mode 100644 src/retention/closure_limits.rs create mode 100644 src/retention/policy.rs create mode 100644 src/retention/profile.rs create mode 100644 src/retention/profile_admission_error.rs create mode 100644 src/retention/root.rs create mode 100644 src/retention/root_digest.rs create mode 100644 src/retention/root_error.rs create mode 100644 tests/retention_root_encoding.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index dd30ed6..7ea1ccb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -307,10 +307,11 @@ after its public API and format compatibility policies are established. liveness manifests, reader snapshots, one-way staged migration, exact crash boundaries, and reserved GC/disposition records. Validated public `RetentionNamespace`, namespace-digest, `RootGeneration`, - `LivenessGeneration`, and `RetentionAnchor` values now establish the core - boundary. Version-1 immutable bytes remain authoritative; production - version-2 writing remains unavailable until issue #19's executable evidence - is complete. + `LivenessGeneration`, `RetentionAnchor`, realization profile, closure limits, + and semantic root values now establish the core boundary. The canonical root + encoder reproduces the independent version-2 golden bytes. Version-1 + immutable bytes remain authoritative; production version-2 writing remains + unavailable until issue #19's executable evidence is complete. - Accepted ADR-0009 defines caller-supplied retention namespaces, `BlobId`/`LayoutId` reconstruction anchors, fail-closed canonical closure, generation-checked retention publication, immutable liveness snapshots, diff --git a/docs/formats/segment-store-v2/README.md b/docs/formats/segment-store-v2/README.md index 3fe3d8b..63e3731 100644 --- a/docs/formats/segment-store-v2/README.md +++ b/docs/formats/segment-store-v2/README.md @@ -68,9 +68,11 @@ re-encode them. The format contract is frozen by ADR-0009 and this specification. Public core types now admit exact namespace bytes, namespace digests, root and liveness -generations, and reconstruction anchors. No production version-2 parser, -transition, migration, or writer exists yet. Requirements that remain marked -as planned in issue #19 or issue #21 are not implementation evidence. A store -must refuse version-2 state until the relevant parser, corruption, +generations, registered realization profiles, bounded closure policies, +reconstruction anchors, and semantic roots. The canonical root encoder matches +the independent golden record. No production version-2 decoder, transition, +migration, or writer exists yet. Requirements that remain marked as planned or +in progress in issue #19 or issue #21 are not complete implementation evidence. +A store must refuse version-2 state until the relevant parser, corruption, golden-format, model-based, crash-injection, recovery, and fuzz evidence is implemented. diff --git a/docs/formats/segment-store-v2/requirements.md b/docs/formats/segment-store-v2/requirements.md index e57e60b..c0ffe91 100644 --- a/docs/formats/segment-store-v2/requirements.md +++ b/docs/formats/segment-store-v2/requirements.md @@ -9,8 +9,8 @@ case is not evidence. | ID | Requirement | Evidence | Status | | --- | --- | --- | --- | -| `KEEP-RETENTION-001` | `RetentionNamespace`, `RootGeneration`, `LivenessGeneration`, profile coordinates, limits, anchors, and digests are validated typed values | `tests/retention_values.rs` covers namespace, digest, generations, and anchors; profile and limit evidence remains | In progress in #19 | -| `KEEP-RETENTION-002` | Root, manifest, and head codecs implement the exact canonical grammars and fixed bounds | golden-format fixtures plus independent oracle | Planned in #19 | +| `KEEP-RETENTION-001` | `RetentionNamespace`, `RootGeneration`, `LivenessGeneration`, profile coordinates, limits, anchors, and digests are validated typed values | `tests/retention_values.rs` and `tests/retention_root_encoding.rs` | Implemented | +| `KEEP-RETENTION-002` | Root, manifest, and head codecs implement the exact canonical grammars and fixed bounds | independent golden corpus plus root encoder evidence in `tests/retention_root_encoding.rs`; root decoder and manifest/head codecs remain | In progress in #19 | | `KEEP-RETENTION-003` | Every structural field, truncation boundary, ordering law, duplicate, overflow, flag, reserved byte, digest, checksum, and trailing byte has a precise refusal | corruption matrix and fuzz | Planned in #19 | | `KEEP-RETENTION-004` | Retain and release compare expected and observed generations and publish exact successors only | unit and model-based transition tests | Planned in #19 | | `KEEP-RETENTION-005` | Closure derivation is deterministic, bounded, cycle-safe, fail-closed, and verifies complete blob reconstruction | property, corruption, and adversarial catalog tests | Planned in #19 | diff --git a/src/adapters/mod.rs b/src/adapters/mod.rs index ead1ffd..77c463a 100644 --- a/src/adapters/mod.rs +++ b/src/adapters/mod.rs @@ -3,7 +3,8 @@ //! This module owns decoding raw input into validated domain types, encoding //! validated domain types into canonical bytes, and exact immutable-segment //! ingress and egress. It does not own identity calculation, logical layout -//! policy, physical location, namespace publication, recovery, or retention. +//! policy, physical location, namespace publication, recovery, or retention +//! policy. mod admitted_catalog; mod admitted_recovery_stage_bytes; @@ -233,6 +234,7 @@ mod recovery_stage_pool_outcome; mod recovery_stage_synchronization_outcome; #[cfg(feature = "repository-tasks")] mod repository_initialization_storage; +mod retention; mod sealed_segment; mod segment_digest; mod segment_digest_builder; @@ -447,6 +449,7 @@ pub use recovery_stage_pool_outcome::RecoveryStagePoolOutcome; pub use recovery_stage_synchronization_outcome::RecoveryStageSynchronizationOutcome; #[cfg(feature = "repository-tasks")] pub use repository_initialization_storage::RepositoryInitializationStorage; +pub use retention::{CanonicalRetentionRoot, RetentionRootEncodeError}; pub use sealed_segment::SealedSegment; pub use segment_digest::SegmentDigest; pub use segment_header::SegmentHeader; diff --git a/src/adapters/retention.rs b/src/adapters/retention.rs new file mode 100644 index 0000000..d2a8653 --- /dev/null +++ b/src/adapters/retention.rs @@ -0,0 +1,8 @@ +//! This module owns canonical retention record boundary adapters. + +mod canonical_root; +mod root_encode_error; +mod root_encoder; + +pub use canonical_root::CanonicalRetentionRoot; +pub use root_encode_error::RetentionRootEncodeError; diff --git a/src/adapters/retention/canonical_root.rs b/src/adapters/retention/canonical_root.rs new file mode 100644 index 0000000..dfdaf64 --- /dev/null +++ b/src/adapters/retention/canonical_root.rs @@ -0,0 +1,42 @@ +//! This boundary module owns materialized canonical retention root bytes. + +use super::{RetentionRootEncodeError, root_encoder}; +use crate::{RetentionRoot, RetentionRootDigest}; + +/// Owned canonical version-2 retention root record. +/// +/// The complete record is materialized in memory after semantic bounds are +/// admitted and exact checked length calculation succeeds. +#[must_use] +#[derive(Debug, Eq, PartialEq)] +pub struct CanonicalRetentionRoot { + encoded: Vec, + digest: RetentionRootDigest, +} + +impl CanonicalRetentionRoot { + /// Encodes one validated semantic retention root. + /// + /// # Errors + /// + /// Returns [`RetentionRootEncodeError`] for checked length overflow, + /// allocation refusal, or an internal construction-length mismatch. + pub fn from_root(root: &RetentionRoot) -> Result { + root_encoder::encode(root) + } + + /// Returns the complete canonical root bytes. + #[must_use] + pub fn encoded(&self) -> &[u8] { + &self.encoded + } + + /// Returns the canonical root digest embedded in the record. + pub const fn digest(&self) -> RetentionRootDigest { + self.digest + } + + pub(super) const fn admitted(encoded: Vec, digest: RetentionRootDigest) -> Self { + Self { encoded, digest } + } +} diff --git a/src/adapters/retention/root_encode_error.rs b/src/adapters/retention/root_encode_error.rs new file mode 100644 index 0000000..eea750c --- /dev/null +++ b/src/adapters/retention/root_encode_error.rs @@ -0,0 +1,48 @@ +//! This boundary module owns typed retention root encoding failures. + +use std::collections::TryReserveError; +use std::error::Error; +use std::fmt; + +/// Failure to materialize one canonical retention root record. +#[derive(Debug)] +pub enum RetentionRootEncodeError { + /// Checked record-length arithmetic overflowed. + LengthOverflow, + /// Exact record allocation was refused. + Allocation { + /// Preserved allocation failure. + source: TryReserveError, + }, + /// Construction produced a length different from its admitted plan. + ConstructionLength { + /// Planned exact length. + expected: usize, + /// Materialized length. + observed: usize, + }, +} + +impl fmt::Display for RetentionRootEncodeError { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::LengthOverflow => formatter.write_str("retention root record length overflow"), + Self::Allocation { .. } => { + formatter.write_str("retention root record allocation failed") + } + Self::ConstructionLength { expected, observed } => write!( + formatter, + "retention root construction produced {observed} bytes; expected {expected}" + ), + } + } +} + +impl Error for RetentionRootEncodeError { + fn source(&self) -> Option<&(dyn Error + 'static)> { + match self { + Self::Allocation { source } => Some(source), + Self::LengthOverflow | Self::ConstructionLength { .. } => None, + } + } +} diff --git a/src/adapters/retention/root_encoder.rs b/src/adapters/retention/root_encoder.rs new file mode 100644 index 0000000..b80c1d6 --- /dev/null +++ b/src/adapters/retention/root_encoder.rs @@ -0,0 +1,150 @@ +//! This boundary module owns canonical version-2 retention root encoding. + +use super::{CanonicalRetentionRoot, RetentionRootEncodeError}; +use crate::{RetentionRoot, RetentionRootDigest}; + +const HEADER_LENGTH: usize = 192; +const ANCHOR_WIDTH: usize = 119; +const TRAILER_LENGTH: usize = 64; + +struct EncodingPlan { + total_length: usize, + digest_preimage_length: usize, + anchor_set_digest: [u8; 32], +} + +pub(super) fn encode( + root: &RetentionRoot, +) -> Result { + let plan = plan(root)?; + let mut encoded = Vec::new(); + encoded + .try_reserve_exact(plan.total_length) + .map_err(|source| RetentionRootEncodeError::Allocation { source })?; + write_header(&mut encoded, root, &plan)?; + write_body(&mut encoded, root); + require_length(&encoded, plan.digest_preimage_length)?; + let digest = hash(b"keep.retention-root/v2\0", &encoded); + encoded.extend_from_slice(&digest); + let checksum = hash(b"keep.retention-root-checksum/v2\0", &encoded); + encoded.extend_from_slice(&checksum); + require_length(&encoded, plan.total_length)?; + Ok(CanonicalRetentionRoot::admitted( + encoded, + RetentionRootDigest::from_hash(digest), + )) +} + +fn plan(root: &RetentionRoot) -> Result { + let anchor_bytes = usize::try_from(root.anchor_count()) + .map_err(|_| RetentionRootEncodeError::LengthOverflow)? + .checked_mul(ANCHOR_WIDTH) + .ok_or(RetentionRootEncodeError::LengthOverflow)?; + let digest_preimage_length = HEADER_LENGTH + .checked_add(root.namespace().as_bytes().len()) + .and_then(|length| length.checked_add(anchor_bytes)) + .ok_or(RetentionRootEncodeError::LengthOverflow)?; + let total_length = digest_preimage_length + .checked_add(TRAILER_LENGTH) + .ok_or(RetentionRootEncodeError::LengthOverflow)?; + Ok(EncodingPlan { + total_length, + digest_preimage_length, + anchor_set_digest: anchor_set_digest(root), + }) +} + +fn anchor_set_digest(root: &RetentionRoot) -> [u8; 32] { + let mut hasher = blake3::Hasher::new(); + hasher.update(b"keep.retention-anchor-set/v2\0"); + hasher.update(&root.anchor_count().to_be_bytes()); + for anchor in root.anchors() { + hasher.update(&anchor.blob_id().encode_binary()); + hasher.update(&anchor.layout_id().encode_binary()); + } + *hasher.finalize().as_bytes() +} + +fn write_header( + encoded: &mut Vec, + root: &RetentionRoot, + plan: &EncodingPlan, +) -> Result<(), RetentionRootEncodeError> { + encoded.extend_from_slice(b"KEEP:RET:ROOT2\0\0"); + push_u16(encoded, 2); + push_u16(encoded, 192); + push_u32(encoded, 0); + push_u64( + encoded, + u64::try_from(plan.total_length).map_err(|_| RetentionRootEncodeError::LengthOverflow)?, + ); + push_u64(encoded, root.generation().get()); + push_u16( + encoded, + u16::try_from(root.namespace().as_bytes().len()) + .map_err(|_| RetentionRootEncodeError::LengthOverflow)?, + ); + push_u16(encoded, 119); + push_u32(encoded, root.anchor_count()); + write_policy(encoded, root); + encoded.extend_from_slice(&predecessor_bytes(root)); + encoded.extend_from_slice(&plan.anchor_set_digest); + encoded.extend_from_slice(&[0_u8; 12]); + require_length(encoded, HEADER_LENGTH) +} + +fn write_policy(encoded: &mut Vec, root: &RetentionRoot) { + let profile = root.profile(); + let limits = root.limits(); + push_u32(encoded, profile.identity()); + push_u32(encoded, profile.version()); + encoded.extend_from_slice(profile.digest()); + push_u64(encoded, limits.nodes()); + push_u16(encoded, limits.depth()); + push_u16(encoded, 0); + push_u64(encoded, limits.encoded_bytes()); + push_u64(encoded, limits.physical_bytes()); +} + +fn predecessor_bytes(root: &RetentionRoot) -> [u8; 32] { + root.predecessor() + .map_or([0_u8; 32], |digest| *digest.as_bytes()) +} + +fn write_body(encoded: &mut Vec, root: &RetentionRoot) { + encoded.extend_from_slice(root.namespace().as_bytes()); + for anchor in root.anchors() { + encoded.extend_from_slice(&anchor.blob_id().encode_binary()); + encoded.extend_from_slice(&anchor.layout_id().encode_binary()); + } +} + +fn hash(domain: &[u8], bytes: &[u8]) -> [u8; 32] { + let mut hasher = blake3::Hasher::new(); + hasher.update(domain); + hasher.update(bytes); + *hasher.finalize().as_bytes() +} + +fn push_u16(encoded: &mut Vec, value: u16) { + encoded.extend_from_slice(&value.to_be_bytes()); +} + +fn push_u32(encoded: &mut Vec, value: u32) { + encoded.extend_from_slice(&value.to_be_bytes()); +} + +fn push_u64(encoded: &mut Vec, value: u64) { + encoded.extend_from_slice(&value.to_be_bytes()); +} + +const fn require_length(encoded: &[u8], expected: usize) -> Result<(), RetentionRootEncodeError> { + if encoded.len() == expected { + Ok(()) + } else { + Err(RetentionRootEncodeError::ConstructionLength { + expected, + observed: encoded.len(), + }) + } +} diff --git a/src/lib.rs b/src/lib.rs index 88026b5..72dc778 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -21,7 +21,8 @@ //! contract and a pinned writer-authorized filesystem adapter. Reusable-stage //! continuation has a storage-independent planning and execution boundary plus //! a pinned writer-authorized filesystem adapter. Core retention namespaces, -//! generations, and reconstruction anchors are validated. Retention +//! generations, realization policy, reconstruction anchors, and semantic roots +//! are validated; canonical root encoding is available. Retention decoding, //! publication, recovery, and garbage collection remain intentionally absent. #[cfg(test)] @@ -42,14 +43,14 @@ pub use adapters::RepositoryInitializationStorage; pub use adapters::{ AdmittedCatalog, AdmittedRecoveryStageBytes, AdmittedSegment, AdmittedSegmentRecord, BlobIdBinaryParseError, BlobIdTextParseError, CanonicalCatalog, CanonicalLayoutRecord, - CanonicalPublicationHead, CatalogAdmissionError, CatalogAllocationPhase, CatalogDecodeError, - CatalogEncodeError, CatalogEntryDecodeError, CatalogPublicationError, - CatalogPublicationExpectation, CatalogPublicationOutcome, CatalogPublicationPhase, - CatalogPublicationReadiness, CatalogPublicationReceipt, CatalogPublicationStorage, - CatalogRestartArtifact, CatalogRestartByteLimit, CatalogRestartByteLimitError, - CatalogRestartError, CatalogRestartPhase, CatalogRestartPolicy, CatalogSnapshot, - CatalogSnapshotError, CatalogSuccessor, CatalogTransitionError, ChecksummedCatalog, - ChecksummedPublicationHead, ChecksummedSegmentRecord, ClosedSegment, + CanonicalPublicationHead, CanonicalRetentionRoot, CatalogAdmissionError, + CatalogAllocationPhase, CatalogDecodeError, CatalogEncodeError, CatalogEntryDecodeError, + CatalogPublicationError, CatalogPublicationExpectation, CatalogPublicationOutcome, + CatalogPublicationPhase, CatalogPublicationReadiness, CatalogPublicationReceipt, + CatalogPublicationStorage, CatalogRestartArtifact, CatalogRestartByteLimit, + CatalogRestartByteLimitError, CatalogRestartError, CatalogRestartPhase, CatalogRestartPolicy, + CatalogSnapshot, CatalogSnapshotError, CatalogSuccessor, CatalogTransitionError, + ChecksummedCatalog, ChecksummedPublicationHead, ChecksummedSegmentRecord, ClosedSegment, FilesystemCatalogPublicationError, FilesystemCatalogPublisher, FilesystemCatalogSnapshot, FilesystemPlatformAdmission, FilesystemPlatformAdmissionError, FilesystemRecoveryInventoryReader, FilesystemRecoveryNextHeadFinalizationOpenError, @@ -83,20 +84,21 @@ pub use adapters::{ RecoveryStageFingerprintAlgorithm, RecoveryStageFingerprintError, RecoveryStageLength, RecoveryStageMetadata, RecoveryStageMetadataError, RecoveryStageNamespacePhase, RecoveryStageParent, RecoveryStagePoolOutcome, RecoveryStageSynchronizationOutcome, - ReusableRecoverySegment, SealedSegment, SegmentDigest, SegmentDurabilityPhase, SegmentHeader, - SegmentHeaderError, SegmentPublication, SegmentPublicationError, SegmentReadError, - SegmentReadPolicy, SegmentRecordAdmissionError, SegmentRecordChecksum, - SegmentRecordDecodeError, SegmentRecordHeader, SegmentRecordHeaderError, SegmentRecordIdentity, - SegmentRecordLength, SegmentRecordLimit, SegmentRecordLimitError, SegmentRecordPayloadLength, - SegmentRecords, SegmentSeal, SegmentSealError, SegmentStage, SegmentStageCreateError, - SegmentWriteError, SegmentWritePhase, StagedSegment, StorageProfileIdParseError, - StoreInitializationError, StoreInitializationPhase, StoreInitializationReceipt, - StoreInitializationStorage, WriterLockAcquireError, WriterLockAcquirePhase, - admit_recovery_stage_bytes, assess_recovery_stage, classify_recovery_catalog_stage, - classify_recovery_names, classify_recovery_next_head_stage, classify_recovery_segment_stage, - execute_recovery_next_head_finalization, execute_recovery_segment_resume, - execute_recovery_stage_completion, execute_recovery_stage_discard, fingerprint_recovery_stage, - initialize_store, plan_recovery_next_head_finalization, plan_recovery_segment_resume, + RetentionRootEncodeError, ReusableRecoverySegment, SealedSegment, SegmentDigest, + SegmentDurabilityPhase, SegmentHeader, SegmentHeaderError, SegmentPublication, + SegmentPublicationError, SegmentReadError, SegmentReadPolicy, SegmentRecordAdmissionError, + SegmentRecordChecksum, SegmentRecordDecodeError, SegmentRecordHeader, SegmentRecordHeaderError, + SegmentRecordIdentity, SegmentRecordLength, SegmentRecordLimit, SegmentRecordLimitError, + SegmentRecordPayloadLength, SegmentRecords, SegmentSeal, SegmentSealError, SegmentStage, + SegmentStageCreateError, SegmentWriteError, SegmentWritePhase, StagedSegment, + StorageProfileIdParseError, StoreInitializationError, StoreInitializationPhase, + StoreInitializationReceipt, StoreInitializationStorage, WriterLockAcquireError, + WriterLockAcquirePhase, admit_recovery_stage_bytes, assess_recovery_stage, + classify_recovery_catalog_stage, classify_recovery_names, classify_recovery_next_head_stage, + classify_recovery_segment_stage, execute_recovery_next_head_finalization, + execute_recovery_segment_resume, execute_recovery_stage_completion, + execute_recovery_stage_discard, fingerprint_recovery_stage, initialize_store, + plan_recovery_next_head_finalization, plan_recovery_segment_resume, plan_recovery_stage_completion, plan_recovery_stage_discard, publish_catalog_generation, read_recovery_inventory, }; @@ -121,6 +123,9 @@ pub use reference::{ ReferenceStoreCapacity, StagedBlob, }; pub use retention::{ - LivenessGeneration, LivenessGenerationError, RetentionAnchor, RetentionNamespace, - RetentionNamespaceDigest, RetentionNamespaceError, RootGeneration, RootGenerationError, + LivenessGeneration, LivenessGenerationError, RegisteredRetentionProfile, RetentionAnchor, + RetentionClosureLimit, RetentionClosureLimitError, RetentionClosureLimits, RetentionNamespace, + RetentionNamespaceDigest, RetentionNamespaceError, RetentionPolicy, + RetentionProfileAdmissionError, RetentionRoot, RetentionRootDigest, RetentionRootError, + RootGeneration, RootGenerationError, }; diff --git a/src/retention/closure_limit.rs b/src/retention/closure_limit.rs new file mode 100644 index 0000000..378b096 --- /dev/null +++ b/src/retention/closure_limit.rs @@ -0,0 +1,27 @@ +//! This module owns semantic names for bounded closure resources. + +use std::fmt; + +/// One independently bounded retention closure resource. +#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)] +pub enum RetentionClosureLimit { + /// Number of logical and physical closure nodes. + Nodes, + /// Maximum traversal depth. + Depth, + /// Total encoded bytes inspected. + EncodedBytes, + /// Total physical bytes inspected. + PhysicalBytes, +} + +impl fmt::Display for RetentionClosureLimit { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter.write_str(match self { + Self::Nodes => "closure nodes", + Self::Depth => "closure depth", + Self::EncodedBytes => "encoded bytes", + Self::PhysicalBytes => "physical bytes", + }) + } +} diff --git a/src/retention/closure_limit_error.rs b/src/retention/closure_limit_error.rs new file mode 100644 index 0000000..156f333 --- /dev/null +++ b/src/retention/closure_limit_error.rs @@ -0,0 +1,43 @@ +//! This module owns typed retention closure-limit failures. + +use std::error::Error; +use std::fmt; + +use super::RetentionClosureLimit; + +/// Failure to admit one bounded closure resource. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum RetentionClosureLimitError { + /// A required positive limit was zero. + Zero { + /// Resource whose limit was zero. + limit: RetentionClosureLimit, + }, + /// A limit exceeded its fixed implementation ceiling. + AboveMaximum { + /// Resource whose limit was excessive. + limit: RetentionClosureLimit, + /// Fixed implementation ceiling. + maximum: u64, + /// Caller-observed limit. + observed: u64, + }, +} + +impl fmt::Display for RetentionClosureLimitError { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::Zero { limit } => write!(formatter, "retention {limit} limit must be positive"), + Self::AboveMaximum { + limit, + maximum, + observed, + } => write!( + formatter, + "retention {limit} limit {observed} exceeds maximum {maximum}" + ), + } + } +} + +impl Error for RetentionClosureLimitError {} diff --git a/src/retention/closure_limits.rs b/src/retention/closure_limits.rs new file mode 100644 index 0000000..ad73135 --- /dev/null +++ b/src/retention/closure_limits.rs @@ -0,0 +1,110 @@ +//! This module owns one fully admitted retention closure resource policy. + +use std::num::{NonZeroU16, NonZeroU64}; + +use super::{RetentionClosureLimit, RetentionClosureLimitError}; + +/// Positive closure limits bounded by the version-2 implementation ceilings. +#[must_use] +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub struct RetentionClosureLimits { + nodes: NonZeroU64, + depth: NonZeroU16, + encoded_bytes: NonZeroU64, + physical_bytes: NonZeroU64, +} + +impl RetentionClosureLimits { + /// Maximum admitted closure node count. + pub const MAXIMUM_NODES: u64 = 1_048_576; + /// Maximum admitted traversal depth. + pub const MAXIMUM_DEPTH: u16 = 8; + /// Maximum admitted encoded bytes. + pub const MAXIMUM_ENCODED_BYTES: u64 = 16_777_216; + /// Maximum admitted physical bytes. + pub const MAXIMUM_PHYSICAL_BYTES: u64 = 1_073_741_824; + + /// Admits one complete positive, ceiling-bounded policy. + /// + /// # Errors + /// + /// Returns the first zero or above-maximum limit in argument order. + pub fn new( + nodes: u64, + depth: u16, + encoded_bytes: u64, + physical_bytes: u64, + ) -> Result { + let nodes = admit_u64(RetentionClosureLimit::Nodes, nodes, Self::MAXIMUM_NODES)?; + let depth = admit_depth(depth)?; + let encoded_bytes = admit_u64( + RetentionClosureLimit::EncodedBytes, + encoded_bytes, + Self::MAXIMUM_ENCODED_BYTES, + )?; + let physical_bytes = admit_u64( + RetentionClosureLimit::PhysicalBytes, + physical_bytes, + Self::MAXIMUM_PHYSICAL_BYTES, + )?; + Ok(Self { + nodes, + depth, + encoded_bytes, + physical_bytes, + }) + } + + /// Returns the positive closure node limit. + #[must_use] + pub const fn nodes(self) -> u64 { + self.nodes.get() + } + + /// Returns the positive closure depth limit. + #[must_use] + pub const fn depth(self) -> u16 { + self.depth.get() + } + + /// Returns the positive encoded-byte limit. + #[must_use] + pub const fn encoded_bytes(self) -> u64 { + self.encoded_bytes.get() + } + + /// Returns the positive physical-byte limit. + #[must_use] + pub const fn physical_bytes(self) -> u64 { + self.physical_bytes.get() + } +} + +fn admit_u64( + limit: RetentionClosureLimit, + observed: u64, + maximum: u64, +) -> Result { + let value = NonZeroU64::new(observed).ok_or(RetentionClosureLimitError::Zero { limit })?; + if observed > maximum { + return Err(RetentionClosureLimitError::AboveMaximum { + limit, + maximum, + observed, + }); + } + Ok(value) +} + +fn admit_depth(observed: u16) -> Result { + let limit = RetentionClosureLimit::Depth; + let value = NonZeroU16::new(observed).ok_or(RetentionClosureLimitError::Zero { limit })?; + if observed > RetentionClosureLimits::MAXIMUM_DEPTH { + return Err(RetentionClosureLimitError::AboveMaximum { + limit, + maximum: u64::from(RetentionClosureLimits::MAXIMUM_DEPTH), + observed: u64::from(observed), + }); + } + Ok(value) +} diff --git a/src/retention/mod.rs b/src/retention/mod.rs index 43494e9..a03e461 100644 --- a/src/retention/mod.rs +++ b/src/retention/mod.rs @@ -6,19 +6,37 @@ //! collection. mod anchor; +mod closure_limit; +mod closure_limit_error; +mod closure_limits; mod liveness_generation; mod liveness_generation_error; mod namespace; mod namespace_digest; mod namespace_error; +mod policy; +mod profile; +mod profile_admission_error; +mod root; +mod root_digest; +mod root_error; mod root_generation; mod root_generation_error; pub use anchor::RetentionAnchor; +pub use closure_limit::RetentionClosureLimit; +pub use closure_limit_error::RetentionClosureLimitError; +pub use closure_limits::RetentionClosureLimits; pub use liveness_generation::LivenessGeneration; pub use liveness_generation_error::LivenessGenerationError; pub use namespace::RetentionNamespace; pub use namespace_digest::RetentionNamespaceDigest; pub use namespace_error::RetentionNamespaceError; +pub use policy::RetentionPolicy; +pub use profile::RegisteredRetentionProfile; +pub use profile_admission_error::RetentionProfileAdmissionError; +pub use root::RetentionRoot; +pub use root_digest::RetentionRootDigest; +pub use root_error::RetentionRootError; pub use root_generation::RootGeneration; pub use root_generation_error::RootGenerationError; diff --git a/src/retention/policy.rs b/src/retention/policy.rs new file mode 100644 index 0000000..04b07a8 --- /dev/null +++ b/src/retention/policy.rs @@ -0,0 +1,28 @@ +//! This module owns one registered, bounded retention realization policy. + +use super::{RegisteredRetentionProfile, RetentionClosureLimits}; + +/// Registered realization semantics paired with caller-selected closure limits. +#[must_use] +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub struct RetentionPolicy { + profile: RegisteredRetentionProfile, + limits: RetentionClosureLimits, +} + +impl RetentionPolicy { + /// Combines one registered profile with already-admitted closure limits. + pub const fn new(profile: RegisteredRetentionProfile, limits: RetentionClosureLimits) -> Self { + Self { profile, limits } + } + + /// Returns the registered realization profile. + pub const fn profile(self) -> RegisteredRetentionProfile { + self.profile + } + + /// Returns the admitted closure limits. + pub const fn limits(self) -> RetentionClosureLimits { + self.limits + } +} diff --git a/src/retention/profile.rs b/src/retention/profile.rs new file mode 100644 index 0000000..3158037 --- /dev/null +++ b/src/retention/profile.rs @@ -0,0 +1,75 @@ +//! This module owns the closed registered retention realization-profile set. + +use super::RetentionProfileAdmissionError; + +const PROFILE_DIGEST: [u8; 32] = [ + 0xdb, 0x1c, 0x1c, 0x1a, 0x50, 0x61, 0x3e, 0xf1, 0x1f, 0x7c, 0x0e, 0xe0, 0x88, 0x2e, 0x37, 0xb6, + 0xd2, 0x4e, 0x2d, 0xb2, 0xca, 0x57, 0x78, 0x3d, 0x01, 0x19, 0x7b, 0xa5, 0x1b, 0x61, 0xce, 0x59, +]; + +/// One deterministic retention realization profile implemented by Keep. +/// +/// The type has private representation so future registered profiles remain +/// an additive registry change rather than an exhaustive-enum break. +#[must_use] +#[derive(Clone, Copy, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)] +pub struct RegisteredRetentionProfile { + identity: u32, + version: u32, + digest: [u8; 32], +} + +impl RegisteredRetentionProfile { + /// The single-canonical-witness version-1 profile. + pub const SINGLE_CANONICAL_WITNESS_V1: Self = Self { + identity: 1, + version: 1, + digest: PROFILE_DIGEST, + }; + + /// Admits an exact registered profile coordinate. + /// + /// # Errors + /// + /// Returns a typed coordinate or definition-digest mismatch. + pub fn admit( + identity: u32, + version: u32, + digest: [u8; 32], + ) -> Result { + let expected = Self::SINGLE_CANONICAL_WITNESS_V1; + if identity != expected.identity || version != expected.version { + return Err(RetentionProfileAdmissionError::UnsupportedCoordinate { + expected_identity: expected.identity, + expected_version: expected.version, + observed_identity: identity, + observed_version: version, + }); + } + if digest != expected.digest { + return Err(RetentionProfileAdmissionError::DefinitionDigestMismatch { + expected: expected.digest, + observed: digest, + }); + } + Ok(expected) + } + + /// Returns the registered integer identity. + #[must_use] + pub const fn identity(self) -> u32 { + self.identity + } + + /// Returns the registered profile version. + #[must_use] + pub const fn version(self) -> u32 { + self.version + } + + /// Returns the exact registered definition digest. + #[must_use] + pub const fn digest(&self) -> &[u8; 32] { + &self.digest + } +} diff --git a/src/retention/profile_admission_error.rs b/src/retention/profile_admission_error.rs new file mode 100644 index 0000000..1be8dc8 --- /dev/null +++ b/src/retention/profile_admission_error.rs @@ -0,0 +1,51 @@ +//! This module owns typed retention-profile admission failures. + +use std::error::Error; +use std::fmt; + +/// Failure to admit a retention realization-profile coordinate. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum RetentionProfileAdmissionError { + /// The identity and version pair is not registered. + UnsupportedCoordinate { + /// Registered identity expected by this Keep version. + expected_identity: u32, + /// Registered version expected by this Keep version. + expected_version: u32, + /// Identity observed at the boundary. + observed_identity: u32, + /// Version observed at the boundary. + observed_version: u32, + }, + /// The registered coordinate carried different definition bytes. + DefinitionDigestMismatch { + /// Exact registered definition digest. + expected: [u8; 32], + /// Digest observed at the boundary. + observed: [u8; 32], + }, +} + +impl fmt::Display for RetentionProfileAdmissionError { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::UnsupportedCoordinate { + expected_identity, + expected_version, + observed_identity, + observed_version, + } => write!( + formatter, + "unsupported retention profile {observed_identity}/{observed_version}; \ + expected {expected_identity}/{expected_version}" + ), + Self::DefinitionDigestMismatch { expected, observed } => write!( + formatter, + "retention profile definition digest mismatch: expected {expected:02x?}, \ + observed {observed:02x?}" + ), + } + } +} + +impl Error for RetentionProfileAdmissionError {} diff --git a/src/retention/root.rs b/src/retention/root.rs new file mode 100644 index 0000000..964876c --- /dev/null +++ b/src/retention/root.rs @@ -0,0 +1,128 @@ +//! This module owns one validated semantic retention root generation. + +use super::{ + RegisteredRetentionProfile, RetentionAnchor, RetentionClosureLimits, RetentionNamespace, + RetentionPolicy, RetentionRootDigest, RetentionRootError, RootGeneration, +}; + +/// One canonical namespace root generation before durable byte encoding. +/// +/// Construction canonicalizes the caller's `Vec` in place, rejects duplicate +/// or excessive anchors, and consumes it into an immutable boxed slice. The +/// boxed-slice conversion may discard excess capacity. +#[must_use] +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct RetentionRoot { + namespace: RetentionNamespace, + generation: RootGeneration, + policy: RetentionPolicy, + predecessor: Option, + anchors: Box<[RetentionAnchor]>, + anchor_count: u32, +} + +impl RetentionRoot { + /// Maximum anchors admitted in one namespace generation. + pub const MAXIMUM_ANCHOR_COUNT: u32 = 65_536; + + /// Admits one deterministic semantic root. + /// + /// Anchors are sorted into canonical order. Duplicate anchors refuse + /// instead of being silently removed. + /// + /// # Errors + /// + /// Returns a typed predecessor, anchor-count, or duplicate refusal. + pub fn new( + namespace: RetentionNamespace, + generation: RootGeneration, + policy: RetentionPolicy, + predecessor: Option, + mut anchors: Vec, + ) -> Result { + admit_predecessor(generation, predecessor)?; + let observed = anchors.len(); + let anchor_count = + u32::try_from(observed).map_err(|_| RetentionRootError::AnchorCountExceeded { + maximum: Self::MAXIMUM_ANCHOR_COUNT, + observed, + })?; + if anchor_count > Self::MAXIMUM_ANCHOR_COUNT { + return Err(RetentionRootError::AnchorCountExceeded { + maximum: Self::MAXIMUM_ANCHOR_COUNT, + observed, + }); + } + anchors.sort_unstable(); + refuse_duplicate(&anchors)?; + Ok(Self { + namespace, + generation, + policy, + predecessor, + anchors: anchors.into_boxed_slice(), + anchor_count, + }) + } + + /// Returns the exact opaque namespace. + pub const fn namespace(&self) -> &RetentionNamespace { + &self.namespace + } + + /// Returns the per-namespace root generation. + pub const fn generation(&self) -> RootGeneration { + self.generation + } + + /// Returns the registered realization profile. + pub const fn profile(&self) -> RegisteredRetentionProfile { + self.policy.profile() + } + + /// Returns the admitted closure limits. + pub const fn limits(&self) -> RetentionClosureLimits { + self.policy.limits() + } + + /// Returns the exact predecessor, absent only for generation one. + #[must_use] + pub const fn predecessor(&self) -> Option { + self.predecessor + } + + /// Returns the canonical, duplicate-free anchors. + pub fn anchors(&self) -> &[RetentionAnchor] { + &self.anchors + } + + /// Returns the bounded anchor count. + #[must_use] + pub const fn anchor_count(&self) -> u32 { + self.anchor_count + } +} + +const fn admit_predecessor( + generation: RootGeneration, + predecessor: Option, +) -> Result<(), RetentionRootError> { + match (generation.get(), predecessor) { + (1, Some(observed)) => { + Err(RetentionRootError::InitialGenerationHasPredecessor { observed }) + } + (1, None) | (_, Some(_)) => Ok(()), + (_, None) => Err(RetentionRootError::MissingPredecessor { generation }), + } +} + +fn refuse_duplicate(anchors: &[RetentionAnchor]) -> Result<(), RetentionRootError> { + let mut previous = None; + for anchor in anchors { + if previous == Some(*anchor) { + return Err(RetentionRootError::DuplicateAnchor { anchor: *anchor }); + } + previous = Some(*anchor); + } + Ok(()) +} diff --git a/src/retention/root_digest.rs b/src/retention/root_digest.rs new file mode 100644 index 0000000..c398fb7 --- /dev/null +++ b/src/retention/root_digest.rs @@ -0,0 +1,18 @@ +//! This module owns canonical retention root identity. + +/// Canonical BLAKE3-256 identity of one complete retention root record. +#[must_use] +#[derive(Clone, Copy, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)] +pub struct RetentionRootDigest([u8; 32]); + +impl RetentionRootDigest { + pub(crate) const fn from_hash(bytes: [u8; 32]) -> Self { + Self(bytes) + } + + /// Returns the exact 32 digest bytes. + #[must_use] + pub const fn as_bytes(&self) -> &[u8; 32] { + &self.0 + } +} diff --git a/src/retention/root_error.rs b/src/retention/root_error.rs new file mode 100644 index 0000000..2360303 --- /dev/null +++ b/src/retention/root_error.rs @@ -0,0 +1,59 @@ +//! This module owns typed semantic retention root failures. + +use std::error::Error; +use std::fmt; + +use super::{RetentionAnchor, RetentionRootDigest, RootGeneration}; + +/// Failure to construct one canonical semantic retention root. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum RetentionRootError { + /// Generation one carried an impossible predecessor. + InitialGenerationHasPredecessor { + /// Observed predecessor digest. + observed: RetentionRootDigest, + }, + /// A successor generation omitted its required predecessor. + MissingPredecessor { + /// Successor generation lacking a predecessor. + generation: RootGeneration, + }, + /// The caller supplied too many anchors. + AnchorCountExceeded { + /// Fixed maximum anchor count. + maximum: u32, + /// Observed anchor count. + observed: usize, + }, + /// The caller supplied the same anchor more than once. + DuplicateAnchor { + /// Exact duplicated anchor. + anchor: RetentionAnchor, + }, +} + +impl fmt::Display for RetentionRootError { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::InitialGenerationHasPredecessor { observed } => write!( + formatter, + "initial retention root has predecessor {:?}", + observed.as_bytes() + ), + Self::MissingPredecessor { generation } => write!( + formatter, + "retention root generation {} requires a predecessor", + generation.get() + ), + Self::AnchorCountExceeded { maximum, observed } => write!( + formatter, + "retention root has {observed} anchors; maximum is {maximum}" + ), + Self::DuplicateAnchor { anchor } => { + write!(formatter, "retention root repeats anchor {anchor:?}") + } + } + } +} + +impl Error for RetentionRootError {} diff --git a/tests/retention_root_encoding.rs b/tests/retention_root_encoding.rs new file mode 100644 index 0000000..446ece2 --- /dev/null +++ b/tests/retention_root_encoding.rs @@ -0,0 +1,180 @@ +//! Public construction laws for canonical version-2 retention roots. + +mod support; + +use std::io; + +use keep::{ + BlobId, CanonicalRetentionRoot, LayoutId, RegisteredRetentionProfile, RetentionAnchor, + RetentionClosureLimit, RetentionClosureLimitError, RetentionClosureLimits, RetentionNamespace, + RetentionPolicy, RetentionProfileAdmissionError, RetentionRoot, RetentionRootError, + RootGeneration, +}; + +const ONE_ANCHOR_ROOT: &str = include_str!("../conformance/segment-store/v2/one-anchor-root.hex"); +const EMPTY_BLOB: &str = concat!( + "keep:blob:v1:blake3-256:0:", + "c0074a279c09f9d019dc10e4c821f79f1450cfb8541ab4627132ab9f3c75e33f" +); +const ONE_ZERO_BLOB: &str = concat!( + "keep:blob:v1:blake3-256:1:", + "1cfb8fa9e917aba15a1f592095f377ff180755fe1212b0d7d2ec750bd128b606" +); +const ONE_ZERO_LAYOUT: &str = concat!( + "keep:layout:v1:flat-chunks-v1:blake3-256:220:", + "887da23f1a7483359a78fc9a7fde80030ec2c4690603803f0ab7d0edb56575b8" +); + +#[test] +fn canonical_root_reproduces_the_frozen_one_anchor_record() -> Result<(), Box> +{ + let root = one_anchor_root()?; + let canonical = CanonicalRetentionRoot::from_root(&root)?; + assert_eq!(canonical.encoded(), fixture_bytes()?); + assert_eq!( + canonical.digest().as_bytes(), + &[ + 0xca, 0x4c, 0x11, 0xf2, 0x65, 0xc3, 0xbe, 0xd0, 0x70, 0x73, 0xbd, 0xc3, 0xb6, 0xae, + 0xf0, 0x03, 0xe9, 0x64, 0xac, 0x8c, 0xb3, 0x6f, 0xcf, 0xcc, 0x92, 0xf2, 0x0f, 0xa6, + 0xf0, 0xb6, 0x00, 0x85, + ] + ); + Ok(()) +} + +#[test] +fn closure_limits_refuse_zero_and_excess_before_root_construction() { + assert_eq!( + RetentionClosureLimits::new(0, 2, 4_096, 4_096), + Err(RetentionClosureLimitError::Zero { + limit: RetentionClosureLimit::Nodes, + }) + ); + assert_eq!( + RetentionClosureLimits::new(4, 2, 4_096, 1_073_741_825), + Err(RetentionClosureLimitError::AboveMaximum { + limit: RetentionClosureLimit::PhysicalBytes, + maximum: 1_073_741_824, + observed: 1_073_741_825, + }) + ); +} + +#[test] +fn realization_profile_admits_only_the_exact_registered_definition() { + let expected = RegisteredRetentionProfile::SINGLE_CANONICAL_WITNESS_V1; + let digest = [ + 0xdb, 0x1c, 0x1c, 0x1a, 0x50, 0x61, 0x3e, 0xf1, 0x1f, 0x7c, 0x0e, 0xe0, 0x88, 0x2e, 0x37, + 0xb6, 0xd2, 0x4e, 0x2d, 0xb2, 0xca, 0x57, 0x78, 0x3d, 0x01, 0x19, 0x7b, 0xa5, 0x1b, 0x61, + 0xce, 0x59, + ]; + assert_eq!( + RegisteredRetentionProfile::admit(1, 1, digest), + Ok(expected) + ); + assert_eq!( + RegisteredRetentionProfile::admit(2, 1, digest), + Err(RetentionProfileAdmissionError::UnsupportedCoordinate { + expected_identity: 1, + expected_version: 1, + observed_identity: 2, + observed_version: 1, + }) + ); + assert_eq!( + RegisteredRetentionProfile::admit(1, 1, [0_u8; 32]), + Err(RetentionProfileAdmissionError::DefinitionDigestMismatch { + expected: digest, + observed: [0_u8; 32], + }) + ); +} + +#[test] +fn root_predecessors_and_anchor_sets_have_one_canonical_admission() +-> Result<(), Box> { + let initial = one_anchor_root()?; + let canonical = CanonicalRetentionRoot::from_root(&initial)?; + let predecessor = canonical.digest(); + let namespace = RetentionNamespace::try_from(vec![0x00, 0x2f, 0xff])?; + let limits = RetentionClosureLimits::new(4, 2, 4_096, 4_096)?; + let profile = RegisteredRetentionProfile::SINGLE_CANONICAL_WITNESS_V1; + let policy = RetentionPolicy::new(profile, limits); + let anchor = one_zero_anchor()?; + let earlier_anchor = RetentionAnchor::new(EMPTY_BLOB.parse()?, anchor.layout_id()); + + assert_eq!( + RetentionRoot::new( + namespace.clone(), + RootGeneration::new(1)?, + policy, + Some(predecessor), + vec![anchor], + ), + Err(RetentionRootError::InitialGenerationHasPredecessor { + observed: predecessor, + }) + ); + assert_eq!( + RetentionRoot::new( + namespace.clone(), + RootGeneration::new(2)?, + policy, + None, + vec![anchor], + ), + Err(RetentionRootError::MissingPredecessor { + generation: RootGeneration::new(2)?, + }) + ); + assert_eq!( + RetentionRoot::new( + namespace.clone(), + RootGeneration::new(2)?, + policy, + Some(predecessor), + vec![anchor, anchor], + ), + Err(RetentionRootError::DuplicateAnchor { anchor }) + ); + let sorted = RetentionRoot::new( + namespace, + RootGeneration::new(2)?, + policy, + Some(predecessor), + vec![anchor, earlier_anchor], + )?; + assert_eq!(sorted.anchors(), &[earlier_anchor, anchor]); + Ok(()) +} + +fn one_anchor_root() -> Result> { + Ok(RetentionRoot::new( + RetentionNamespace::try_from(vec![0x00, 0x2f, 0xff])?, + RootGeneration::new(1)?, + RetentionPolicy::new( + RegisteredRetentionProfile::SINGLE_CANONICAL_WITNESS_V1, + RetentionClosureLimits::new(4, 2, 4_096, 4_096)?, + ), + None, + vec![one_zero_anchor()?], + )?) +} + +fn one_zero_anchor() -> Result> { + let blob: BlobId = ONE_ZERO_BLOB.parse()?; + let layout: LayoutId = ONE_ZERO_LAYOUT.parse()?; + Ok(RetentionAnchor::new(blob, layout)) +} + +fn fixture_bytes() -> Result, io::Error> { + let encoded = ONE_ANCHOR_ROOT + .strip_suffix('\n') + .ok_or_else(|| io::Error::other("retention root fixture lacks final newline"))?; + if encoded.contains(['\n', '\r']) { + return Err(io::Error::other( + "retention root fixture contains an embedded line ending", + )); + } + support::decode_hex(encoded) +} From 741068c42870f684832acc040af5881ee4d3f57b Mon Sep 17 00:00:00 2001 From: James Ross Date: Wed, 29 Jul 2026 20:44:17 -0700 Subject: [PATCH 005/111] Add: Decode canonical retention roots --- CHANGELOG.md | 9 +- README.md | 7 +- docs/formats/segment-store-v2/requirements.md | 2 +- docs/formats/segment-store-v2/retention.md | 5 +- src/adapters/mod.rs | 5 +- src/adapters/retention.rs | 11 + src/adapters/retention/admitted_root.rs | 58 ++++++ src/adapters/retention/root_anchor_decoder.rs | 47 +++++ src/adapters/retention/root_decode_error.rs | 150 ++++++++++++++ .../retention/root_decode_error_display.rs | 122 +++++++++++ src/adapters/retention/root_decoder.rs | 50 +++++ src/adapters/retention/root_field_decoder.rs | 106 ++++++++++ src/adapters/retention/root_header_decoder.rs | 111 ++++++++++ src/adapters/retention/root_integrity.rs | 82 ++++++++ .../retention/root_semantic_header.rs | 55 +++++ src/lib.rs | 15 +- tests/retention_root_decoding.rs | 189 ++++++++++++++++++ 17 files changed, 1008 insertions(+), 16 deletions(-) create mode 100644 src/adapters/retention/admitted_root.rs create mode 100644 src/adapters/retention/root_anchor_decoder.rs create mode 100644 src/adapters/retention/root_decode_error.rs create mode 100644 src/adapters/retention/root_decode_error_display.rs create mode 100644 src/adapters/retention/root_decoder.rs create mode 100644 src/adapters/retention/root_field_decoder.rs create mode 100644 src/adapters/retention/root_header_decoder.rs create mode 100644 src/adapters/retention/root_integrity.rs create mode 100644 src/adapters/retention/root_semantic_header.rs create mode 100644 tests/retention_root_decoding.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index 7ea1ccb..d9ecd3e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -309,9 +309,12 @@ after its public API and format compatibility policies are established. `RetentionNamespace`, namespace-digest, `RootGeneration`, `LivenessGeneration`, `RetentionAnchor`, realization profile, closure limits, and semantic root values now establish the core boundary. The canonical root - encoder reproduces the independent version-2 golden bytes. Version-1 - immutable bytes remain authoritative; production version-2 writing remains - unavailable until issue #19's executable evidence is complete. + encoder reproduces the independent version-2 golden bytes, and the decoder + verifies framing, checksum, root digest, anchor-set digest, nested identities, + resource bounds, canonical anchor order, and semantic invariants before + admission. Version-1 immutable bytes remain authoritative; production + version-2 writing remains unavailable until issue #19's executable evidence + is complete. - Accepted ADR-0009 defines caller-supplied retention namespaces, `BlobId`/`LayoutId` reconstruction anchors, fail-closed canonical closure, generation-checked retention publication, immutable liveness snapshots, diff --git a/README.md b/README.md index 3fc5302..6cc0d33 100644 --- a/README.md +++ b/README.md @@ -115,9 +115,10 @@ bytes, checks hard-link identity and writer-lock release, runs the production recovery classifiers and immutable-artifact admission, and reconstructs the exact published generation and visible one-zero chunk when `HEAD` exists. This matrix proves application process-death behavior; it does not simulate host -power loss. Retention, compaction, and garbage collection remain planned. -Presence in the reference CAS does not claim retention, crash recovery, or -durability. +power loss. Version-2 retention values and canonical in-memory root encoding +and decoding are implemented. Retention publication, recovery, compaction, and +garbage collection remain planned. Presence in the reference CAS does not +claim retention, crash recovery, or durability. Run the complete debug-profile matrix: diff --git a/docs/formats/segment-store-v2/requirements.md b/docs/formats/segment-store-v2/requirements.md index c0ffe91..aeebfad 100644 --- a/docs/formats/segment-store-v2/requirements.md +++ b/docs/formats/segment-store-v2/requirements.md @@ -10,7 +10,7 @@ case is not evidence. | ID | Requirement | Evidence | Status | | --- | --- | --- | --- | | `KEEP-RETENTION-001` | `RetentionNamespace`, `RootGeneration`, `LivenessGeneration`, profile coordinates, limits, anchors, and digests are validated typed values | `tests/retention_values.rs` and `tests/retention_root_encoding.rs` | Implemented | -| `KEEP-RETENTION-002` | Root, manifest, and head codecs implement the exact canonical grammars and fixed bounds | independent golden corpus plus root encoder evidence in `tests/retention_root_encoding.rs`; root decoder and manifest/head codecs remain | In progress in #19 | +| `KEEP-RETENTION-002` | Root, manifest, and head codecs implement the exact canonical grammars and fixed bounds | independent golden corpus plus root encoder and decoder evidence in `tests/retention_root_encoding.rs` and `tests/retention_root_decoding.rs`; manifest/head codecs remain | In progress in #19 | | `KEEP-RETENTION-003` | Every structural field, truncation boundary, ordering law, duplicate, overflow, flag, reserved byte, digest, checksum, and trailing byte has a precise refusal | corruption matrix and fuzz | Planned in #19 | | `KEEP-RETENTION-004` | Retain and release compare expected and observed generations and publish exact successors only | unit and model-based transition tests | Planned in #19 | | `KEEP-RETENTION-005` | Closure derivation is deterministic, bounded, cycle-safe, fail-closed, and verifies complete blob reconstruction | property, corruption, and adversarial catalog tests | Planned in #19 | diff --git a/docs/formats/segment-store-v2/retention.md b/docs/formats/segment-store-v2/retention.md index 6462ce4..3f57f19 100644 --- a/docs/formats/segment-store-v2/retention.md +++ b/docs/formats/segment-store-v2/retention.md @@ -159,7 +159,10 @@ retention/roots// -.root ``` -Names with alternate width, case, suffix, generation, or digest refuse. +Names with alternate width, case, suffix, generation, or digest refuse. Keep +implements validated in-memory root encoding and decoding with complete +integrity verification before semantic admission. Filesystem publication, +manifest/head codecs, transitions, recovery, and garbage collection remain absent. ## Global retention manifest diff --git a/src/adapters/mod.rs b/src/adapters/mod.rs index 77c463a..111841a 100644 --- a/src/adapters/mod.rs +++ b/src/adapters/mod.rs @@ -449,7 +449,10 @@ pub use recovery_stage_pool_outcome::RecoveryStagePoolOutcome; pub use recovery_stage_synchronization_outcome::RecoveryStageSynchronizationOutcome; #[cfg(feature = "repository-tasks")] pub use repository_initialization_storage::RepositoryInitializationStorage; -pub use retention::{CanonicalRetentionRoot, RetentionRootEncodeError}; +pub use retention::{ + AdmittedRetentionRoot, CanonicalRetentionRoot, RetentionRootDecodeError, + RetentionRootEncodeError, +}; pub use sealed_segment::SealedSegment; pub use segment_digest::SegmentDigest; pub use segment_header::SegmentHeader; diff --git a/src/adapters/retention.rs b/src/adapters/retention.rs index d2a8653..5fa3118 100644 --- a/src/adapters/retention.rs +++ b/src/adapters/retention.rs @@ -1,8 +1,19 @@ //! This module owns canonical retention record boundary adapters. +mod admitted_root; mod canonical_root; +mod root_anchor_decoder; +mod root_decode_error; +mod root_decode_error_display; +mod root_decoder; mod root_encode_error; mod root_encoder; +mod root_field_decoder; +mod root_header_decoder; +mod root_integrity; +mod root_semantic_header; +pub use admitted_root::AdmittedRetentionRoot; pub use canonical_root::CanonicalRetentionRoot; +pub use root_decode_error::RetentionRootDecodeError; pub use root_encode_error::RetentionRootEncodeError; diff --git a/src/adapters/retention/admitted_root.rs b/src/adapters/retention/admitted_root.rs new file mode 100644 index 0000000..ac0f33f --- /dev/null +++ b/src/adapters/retention/admitted_root.rs @@ -0,0 +1,58 @@ +//! This boundary module owns one decoded and admitted retention root. + +use super::{RetentionRootDecodeError, root_decoder}; +use crate::{RetentionRoot, RetentionRootDigest}; + +/// Borrowed canonical bytes paired with their admitted semantic root. +/// +/// Decoding verifies exact framing, the complete-record checksum, the root and +/// anchor-set digests, every nested identity, canonical anchor order, and all +/// semantic invariants. Anchor and namespace allocation is bounded by fields +/// admitted from the record. Decoding performs no I/O. +#[must_use] +#[derive(Debug, Eq, PartialEq)] +pub struct AdmittedRetentionRoot<'encoded> { + encoded: &'encoded [u8], + root: RetentionRoot, + digest: RetentionRootDigest, +} + +impl<'encoded> AdmittedRetentionRoot<'encoded> { + /// Decodes and admits one exact canonical version-2 root record. + /// + /// # Errors + /// + /// Returns [`RetentionRootDecodeError`] at the first violated framing, + /// integrity, nested-codec, resource-bound, or semantic invariant. + pub fn decode(encoded: &'encoded [u8]) -> Result { + root_decoder::decode(encoded) + } + + /// Returns the complete verified canonical bytes. + #[must_use] + pub const fn encoded(&self) -> &'encoded [u8] { + self.encoded + } + + /// Returns the admitted semantic root. + pub const fn root(&self) -> &RetentionRoot { + &self.root + } + + /// Returns the verified canonical root digest. + pub const fn digest(&self) -> RetentionRootDigest { + self.digest + } + + pub(super) const fn admitted( + encoded: &'encoded [u8], + root: RetentionRoot, + digest: RetentionRootDigest, + ) -> Self { + Self { + encoded, + root, + digest, + } + } +} diff --git a/src/adapters/retention/root_anchor_decoder.rs b/src/adapters/retention/root_anchor_decoder.rs new file mode 100644 index 0000000..44d2625 --- /dev/null +++ b/src/adapters/retention/root_anchor_decoder.rs @@ -0,0 +1,47 @@ +//! This boundary module owns canonical retention anchor body decoding. + +use super::RetentionRootDecodeError; +use crate::{BlobId, LayoutId, RetentionAnchor}; + +const BLOB_ID_WIDTH: usize = 59; +const ANCHOR_WIDTH: usize = 119; + +pub(super) fn decode( + encoded: &[u8], + anchor_count: u32, +) -> Result, RetentionRootDecodeError> { + let capacity = + usize::try_from(anchor_count).map_err(|_| RetentionRootDecodeError::LengthOverflow)?; + let mut anchors = Vec::new(); + anchors + .try_reserve_exact(capacity) + .map_err(|source| RetentionRootDecodeError::Allocation { source })?; + let mut previous = None; + for (position, bytes) in encoded.chunks_exact(ANCHOR_WIDTH).enumerate() { + let index = + u32::try_from(position).map_err(|_| RetentionRootDecodeError::LengthOverflow)?; + let (blob_bytes, layout_bytes) = bytes.split_at(BLOB_ID_WIDTH); + let blob_id = BlobId::parse_binary(blob_bytes) + .map_err(|source| RetentionRootDecodeError::BlobId { index, source })?; + let layout_id = LayoutId::parse_binary(layout_bytes) + .map_err(|source| RetentionRootDecodeError::LayoutId { index, source })?; + let observed = RetentionAnchor::new(blob_id, layout_id); + if let Some(prior) = previous + && observed <= prior + { + return Err(RetentionRootDecodeError::NonCanonicalAnchorOrder { index }); + } + anchors.push(observed); + previous = Some(observed); + } + if anchors.len() == capacity { + Ok(anchors) + } else { + Err(RetentionRootDecodeError::Truncated { + expected: capacity + .checked_mul(ANCHOR_WIDTH) + .ok_or(RetentionRootDecodeError::LengthOverflow)?, + observed: encoded.len(), + }) + } +} diff --git a/src/adapters/retention/root_decode_error.rs b/src/adapters/retention/root_decode_error.rs new file mode 100644 index 0000000..4965385 --- /dev/null +++ b/src/adapters/retention/root_decode_error.rs @@ -0,0 +1,150 @@ +//! This boundary module owns typed retention root decoding failures. + +use std::collections::TryReserveError; + +use crate::{ + BlobIdBinaryParseError, LayoutIdBinaryParseError, RetentionClosureLimitError, + RetentionNamespaceError, RetentionProfileAdmissionError, RetentionRootError, + RootGenerationError, +}; + +/// Failure to decode and admit one version-2 retention root. +#[derive(Debug)] +pub enum RetentionRootDecodeError { + /// The byte string ended before its required exact length. + Truncated { + /// Required byte length. + expected: usize, + /// Observed byte length. + observed: usize, + }, + /// Bytes followed the required exact record. + TrailingData { + /// Required byte length. + expected: usize, + /// Observed byte length. + observed: usize, + }, + /// The fixed record magic was not canonical. + InvalidMagic { + /// Observed 16 magic bytes. + observed: [u8; 16], + }, + /// The format version is unsupported. + UnsupportedVersion { + /// Supported version. + expected: u16, + /// Observed version. + observed: u16, + }, + /// The fixed header width was not canonical. + InvalidHeaderLength { + /// Required header width. + expected: u16, + /// Observed width. + observed: u16, + }, + /// The record carried unsupported flags. + UnsupportedFlags { + /// Observed flag bits. + observed: u32, + }, + /// The declared total length disagreed with canonical field arithmetic. + DeclaredLengthMismatch { + /// Canonical computed length. + expected: u64, + /// Declared length. + observed: u64, + }, + /// Checked record-length arithmetic overflowed. + LengthOverflow, + /// The fixed anchor width was not canonical. + InvalidAnchorWidth { + /// Required anchor width. + expected: u16, + /// Observed anchor width. + observed: u16, + }, + /// A reserved field was nonzero. + NonZeroReserved { + /// Protocol field name. + field: &'static str, + }, + /// Root generation admission failed. + Generation { + /// Preserved generation failure. + source: RootGenerationError, + }, + /// Namespace admission failed. + Namespace { + /// Preserved namespace failure. + source: RetentionNamespaceError, + }, + /// The declared anchor count exceeded the fixed bound. + AnchorCountExceeded { + /// Fixed maximum count. + maximum: u32, + /// Observed count. + observed: u32, + }, + /// Realization-profile admission failed. + Profile { + /// Preserved profile failure. + source: RetentionProfileAdmissionError, + }, + /// Closure-limit admission failed. + ClosureLimit { + /// Preserved limit failure. + source: RetentionClosureLimitError, + }, + /// One anchor contained a malformed `BlobId`. + BlobId { + /// Zero-based anchor index. + index: u32, + /// Preserved coordinate failure. + source: BlobIdBinaryParseError, + }, + /// One anchor contained a malformed `LayoutId`. + LayoutId { + /// Zero-based anchor index. + index: u32, + /// Preserved coordinate failure. + source: LayoutIdBinaryParseError, + }, + /// Canonical anchor ordering was violated. + NonCanonicalAnchorOrder { + /// Zero-based index of the observed anchor. + index: u32, + }, + /// Anchor allocation was refused. + Allocation { + /// Preserved allocation failure. + source: TryReserveError, + }, + /// The anchor-set digest did not match the exact body. + AnchorSetDigestMismatch { + /// Computed canonical digest. + expected: [u8; 32], + /// Digest stored in the header. + observed: [u8; 32], + }, + /// The root digest did not match the exact header and body. + RootDigestMismatch { + /// Computed canonical digest. + expected: [u8; 32], + /// Digest stored in the record. + observed: [u8; 32], + }, + /// The checksum did not match the complete digest-bearing prefix. + ChecksumMismatch { + /// Computed canonical checksum. + expected: [u8; 32], + /// Checksum stored in the record. + observed: [u8; 32], + }, + /// Final semantic root admission failed. + Semantic { + /// Preserved semantic failure. + source: RetentionRootError, + }, +} diff --git a/src/adapters/retention/root_decode_error_display.rs b/src/adapters/retention/root_decode_error_display.rs new file mode 100644 index 0000000..f95e69f --- /dev/null +++ b/src/adapters/retention/root_decode_error_display.rs @@ -0,0 +1,122 @@ +//! This boundary module owns retention root decode diagnostics and sources. + +use std::{error::Error, fmt}; + +use super::RetentionRootDecodeError; +impl fmt::Display for RetentionRootDecodeError { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::Truncated { expected, observed } => { + write!( + formatter, + "retention root has {observed} bytes; expected {expected}" + ) + } + Self::TrailingData { expected, observed } => write!( + formatter, + "retention root has trailing data: expected {expected} bytes, observed {observed}" + ), + Self::InvalidMagic { observed } => { + write!(formatter, "invalid retention root magic {observed:02x?}") + } + Self::UnsupportedVersion { expected, observed } => write!( + formatter, + "unsupported retention root version {observed}; expected {expected}" + ), + Self::InvalidHeaderLength { expected, observed } => write!( + formatter, + "retention root header length {observed}; expected {expected}" + ), + Self::UnsupportedFlags { observed } => { + write!( + formatter, + "unsupported retention root flags {observed:#010x}" + ) + } + Self::DeclaredLengthMismatch { expected, observed } => write!( + formatter, + "retention root declares {observed} bytes; canonical fields require {expected}" + ), + Self::LengthOverflow => formatter.write_str("retention root length overflow"), + Self::InvalidAnchorWidth { expected, observed } => write!( + formatter, + "retention root anchor width {observed}; expected {expected}" + ), + Self::NonZeroReserved { field } => { + write!( + formatter, + "retention root {field} reserved bytes are nonzero" + ) + } + Self::Generation { source } => write!(formatter, "invalid root generation: {source}"), + Self::Namespace { source } => write!(formatter, "invalid root namespace: {source}"), + Self::AnchorCountExceeded { maximum, observed } => write!( + formatter, + "retention root declares {observed} anchors; maximum is {maximum}" + ), + Self::Profile { source } => write!(formatter, "invalid root profile: {source}"), + Self::ClosureLimit { source } => { + write!(formatter, "invalid root closure limit: {source}") + } + Self::BlobId { index, source } => { + write!( + formatter, + "invalid BlobId in retention anchor {index}: {source}" + ) + } + Self::LayoutId { index, source } => { + write!( + formatter, + "invalid LayoutId in retention anchor {index}: {source}" + ) + } + Self::NonCanonicalAnchorOrder { index, .. } => write!( + formatter, + "retention anchor {index} is not greater than its predecessor" + ), + Self::Allocation { .. } => { + formatter.write_str("retention root anchor allocation failed") + } + Self::AnchorSetDigestMismatch { .. } => { + formatter.write_str("retention root anchor-set digest mismatch") + } + Self::RootDigestMismatch { .. } => { + formatter.write_str("retention root digest mismatch") + } + Self::ChecksumMismatch { .. } => { + formatter.write_str("retention root checksum mismatch") + } + Self::Semantic { source } => write!(formatter, "invalid semantic root: {source}"), + } + } +} + +impl Error for RetentionRootDecodeError { + fn source(&self) -> Option<&(dyn Error + 'static)> { + match self { + Self::Generation { source } => Some(source), + Self::Namespace { source } => Some(source), + Self::Profile { source } => Some(source), + Self::ClosureLimit { source } => Some(source), + Self::BlobId { source, .. } => Some(source), + Self::LayoutId { source, .. } => Some(source), + Self::Allocation { source } => Some(source), + Self::Semantic { source } => Some(source), + Self::Truncated { .. } + | Self::TrailingData { .. } + | Self::InvalidMagic { .. } + | Self::UnsupportedVersion { .. } + | Self::InvalidHeaderLength { .. } + | Self::UnsupportedFlags { .. } + | Self::DeclaredLengthMismatch { .. } + | Self::LengthOverflow + | Self::InvalidAnchorWidth { .. } + | Self::NonZeroReserved { .. } + | Self::AnchorCountExceeded { .. } + | Self::NonCanonicalAnchorOrder { .. } + | Self::AnchorSetDigestMismatch { .. } + | Self::RootDigestMismatch { .. } + | Self::ChecksumMismatch { .. } => None, + } + } +} diff --git a/src/adapters/retention/root_decoder.rs b/src/adapters/retention/root_decoder.rs new file mode 100644 index 0000000..cf4a7b1 --- /dev/null +++ b/src/adapters/retention/root_decoder.rs @@ -0,0 +1,50 @@ +//! This boundary module owns canonical retention root decoding order. + +use super::root_header_decoder::HEADER_LENGTH; +use super::{ + AdmittedRetentionRoot, RetentionRootDecodeError, root_anchor_decoder, root_header_decoder, + root_integrity, root_semantic_header, +}; +use crate::{RetentionNamespace, RetentionPolicy, RetentionRoot, RetentionRootDigest}; + +pub(super) fn decode( + encoded: &[u8], +) -> Result, RetentionRootDecodeError> { + let header = root_header_decoder::decode(encoded)?; + let digest = root_integrity::verify(encoded, header.digest_offset, header.checksum_offset)?; + let namespace_end = HEADER_LENGTH + .checked_add(header.namespace_length) + .ok_or(RetentionRootDecodeError::LengthOverflow)?; + let namespace_bytes = + encoded + .get(HEADER_LENGTH..namespace_end) + .ok_or(RetentionRootDecodeError::Truncated { + expected: namespace_end, + observed: encoded.len(), + })?; + let anchor_bytes = encoded.get(namespace_end..header.digest_offset).ok_or( + RetentionRootDecodeError::Truncated { + expected: header.digest_offset, + observed: encoded.len(), + }, + )?; + root_integrity::verify_anchor_set(header.anchor_count, anchor_bytes, header.anchor_set_digest)?; + let admitted_header = root_semantic_header::admit(&header)?; + let namespace = RetentionNamespace::try_from(namespace_bytes) + .map_err(|source| RetentionRootDecodeError::Namespace { source })?; + let anchors = root_anchor_decoder::decode(anchor_bytes, header.anchor_count)?; + let policy = RetentionPolicy::new(admitted_header.profile, admitted_header.limits); + let root = RetentionRoot::new( + namespace, + admitted_header.generation, + policy, + admitted_header.predecessor, + anchors, + ) + .map_err(|source| RetentionRootDecodeError::Semantic { source })?; + Ok(AdmittedRetentionRoot::admitted( + encoded, + root, + RetentionRootDigest::from_hash(digest), + )) +} diff --git a/src/adapters/retention/root_field_decoder.rs b/src/adapters/retention/root_field_decoder.rs new file mode 100644 index 0000000..89b9f8c --- /dev/null +++ b/src/adapters/retention/root_field_decoder.rs @@ -0,0 +1,106 @@ +//! This boundary module owns fixed-width retention root field extraction. + +use std::cmp::Ordering; + +use super::RetentionRootDecodeError; + +pub(super) fn require_exact( + encoded: &[u8], + expected: usize, +) -> Result<(), RetentionRootDecodeError> { + match encoded.len().cmp(&expected) { + Ordering::Less => Err(RetentionRootDecodeError::Truncated { + expected, + observed: encoded.len(), + }), + Ordering::Equal => Ok(()), + Ordering::Greater => Err(RetentionRootDecodeError::TrailingData { + expected, + observed: encoded.len(), + }), + } +} + +pub(super) const fn require_minimum( + encoded: &[u8], + expected: usize, +) -> Result<(), RetentionRootDecodeError> { + if encoded.len() < expected { + Err(RetentionRootDecodeError::Truncated { + expected, + observed: encoded.len(), + }) + } else { + Ok(()) + } +} + +pub(super) fn require_zero( + encoded: &[u8], + offset: usize, + width: usize, + field: &'static str, +) -> Result<(), RetentionRootDecodeError> { + let end = offset + .checked_add(width) + .ok_or(RetentionRootDecodeError::LengthOverflow)?; + let bytes = encoded + .get(offset..end) + .ok_or(RetentionRootDecodeError::Truncated { + expected: end, + observed: encoded.len(), + })?; + if bytes.iter().all(|byte| *byte == 0) { + Ok(()) + } else { + Err(RetentionRootDecodeError::NonZeroReserved { field }) + } +} + +pub(super) fn require_u16( + encoded: &[u8], + offset: usize, + expected: u16, + error: F, +) -> Result<(), RetentionRootDecodeError> +where + F: FnOnce(u16, u16) -> RetentionRootDecodeError, +{ + let observed = read_u16(encoded, offset)?; + if observed == expected { + Ok(()) + } else { + Err(error(expected, observed)) + } +} + +pub(super) fn read_u16(encoded: &[u8], offset: usize) -> Result { + read_array(encoded, offset).map(u16::from_be_bytes) +} + +pub(super) fn read_u32(encoded: &[u8], offset: usize) -> Result { + read_array(encoded, offset).map(u32::from_be_bytes) +} + +pub(super) fn read_u64(encoded: &[u8], offset: usize) -> Result { + read_array(encoded, offset).map(u64::from_be_bytes) +} + +pub(super) fn read_array( + encoded: &[u8], + offset: usize, +) -> Result<[u8; WIDTH], RetentionRootDecodeError> { + let end = offset + .checked_add(WIDTH) + .ok_or(RetentionRootDecodeError::LengthOverflow)?; + let bytes = encoded + .get(offset..end) + .ok_or(RetentionRootDecodeError::Truncated { + expected: end, + observed: encoded.len(), + })?; + <[u8; WIDTH]>::try_from(bytes).map_err(|_| RetentionRootDecodeError::Truncated { + expected: end, + observed: encoded.len(), + }) +} diff --git a/src/adapters/retention/root_header_decoder.rs b/src/adapters/retention/root_header_decoder.rs new file mode 100644 index 0000000..0de5091 --- /dev/null +++ b/src/adapters/retention/root_header_decoder.rs @@ -0,0 +1,111 @@ +//! This boundary module owns retention root header framing admission. + +use super::RetentionRootDecodeError; +use super::root_field_decoder::{ + read_array, read_u16, read_u32, read_u64, require_exact, require_minimum, require_u16, + require_zero, +}; + +pub(super) const HEADER_LENGTH: usize = 192; +const ANCHOR_WIDTH: usize = 119; +const TRAILER_LENGTH: usize = 64; + +pub(super) struct DecodedRootHeader { + pub(super) generation: u64, + pub(super) namespace_length: usize, + pub(super) anchor_count: u32, + pub(super) profile_identity: u32, + pub(super) profile_version: u32, + pub(super) profile_digest: [u8; 32], + pub(super) closure_nodes: u64, + pub(super) closure_depth: u16, + pub(super) closure_encoded_bytes: u64, + pub(super) closure_physical_bytes: u64, + pub(super) predecessor: [u8; 32], + pub(super) anchor_set_digest: [u8; 32], + pub(super) digest_offset: usize, + pub(super) checksum_offset: usize, +} + +pub(super) fn decode(encoded: &[u8]) -> Result { + require_minimum(encoded, HEADER_LENGTH)?; + validate_fixed_fields(encoded)?; + let namespace_length = usize::from(read_u16(encoded, 40)?); + let anchor_count = read_u32(encoded, 44)?; + let total_length = canonical_length(namespace_length, anchor_count)?; + require_declared_length(encoded, total_length)?; + require_exact(encoded, total_length)?; + let checksum_offset = total_length + .checked_sub(32) + .ok_or(RetentionRootDecodeError::LengthOverflow)?; + let digest_offset = checksum_offset + .checked_sub(32) + .ok_or(RetentionRootDecodeError::LengthOverflow)?; + Ok(DecodedRootHeader { + generation: read_u64(encoded, 32)?, + namespace_length, + anchor_count, + profile_identity: read_u32(encoded, 48)?, + profile_version: read_u32(encoded, 52)?, + profile_digest: read_array(encoded, 56)?, + closure_nodes: read_u64(encoded, 88)?, + closure_depth: read_u16(encoded, 96)?, + closure_encoded_bytes: read_u64(encoded, 100)?, + closure_physical_bytes: read_u64(encoded, 108)?, + predecessor: read_array(encoded, 116)?, + anchor_set_digest: read_array(encoded, 148)?, + digest_offset, + checksum_offset, + }) +} + +fn validate_fixed_fields(encoded: &[u8]) -> Result<(), RetentionRootDecodeError> { + let magic = read_array(encoded, 0)?; + if magic != *b"KEEP:RET:ROOT2\0\0" { + return Err(RetentionRootDecodeError::InvalidMagic { observed: magic }); + } + require_u16(encoded, 16, 2, |expected, observed| { + RetentionRootDecodeError::UnsupportedVersion { expected, observed } + })?; + require_u16(encoded, 18, 192, |expected, observed| { + RetentionRootDecodeError::InvalidHeaderLength { expected, observed } + })?; + let flags = read_u32(encoded, 20)?; + if flags != 0 { + return Err(RetentionRootDecodeError::UnsupportedFlags { observed: flags }); + } + require_u16(encoded, 42, 119, |expected, observed| { + RetentionRootDecodeError::InvalidAnchorWidth { expected, observed } + })?; + require_zero(encoded, 98, 2, "limit")?; + require_zero(encoded, 180, 12, "trailing header") +} + +fn canonical_length( + namespace_length: usize, + anchor_count: u32, +) -> Result { + let anchors = usize::try_from(anchor_count) + .map_err(|_| RetentionRootDecodeError::LengthOverflow)? + .checked_mul(ANCHOR_WIDTH) + .ok_or(RetentionRootDecodeError::LengthOverflow)?; + HEADER_LENGTH + .checked_add(namespace_length) + .and_then(|length| length.checked_add(anchors)) + .and_then(|length| length.checked_add(TRAILER_LENGTH)) + .ok_or(RetentionRootDecodeError::LengthOverflow) +} + +fn require_declared_length( + encoded: &[u8], + total_length: usize, +) -> Result<(), RetentionRootDecodeError> { + let observed = read_u64(encoded, 24)?; + let expected = + u64::try_from(total_length).map_err(|_| RetentionRootDecodeError::LengthOverflow)?; + if observed == expected { + Ok(()) + } else { + Err(RetentionRootDecodeError::DeclaredLengthMismatch { expected, observed }) + } +} diff --git a/src/adapters/retention/root_integrity.rs b/src/adapters/retention/root_integrity.rs new file mode 100644 index 0000000..71f842d --- /dev/null +++ b/src/adapters/retention/root_integrity.rs @@ -0,0 +1,82 @@ +//! This boundary module owns retention root digest and checksum verification. + +use super::RetentionRootDecodeError; + +pub(super) fn verify( + encoded: &[u8], + digest_offset: usize, + checksum_offset: usize, +) -> Result<[u8; 32], RetentionRootDecodeError> { + let observed_checksum = read_digest(encoded, checksum_offset)?; + let checksum_preimage = + encoded + .get(..checksum_offset) + .ok_or(RetentionRootDecodeError::Truncated { + expected: checksum_offset, + observed: encoded.len(), + })?; + let expected_checksum = hash(b"keep.retention-root-checksum/v2\0", checksum_preimage); + if observed_checksum != expected_checksum { + return Err(RetentionRootDecodeError::ChecksumMismatch { + expected: expected_checksum, + observed: observed_checksum, + }); + } + + let observed_digest = read_digest(encoded, digest_offset)?; + let digest_preimage = + encoded + .get(..digest_offset) + .ok_or(RetentionRootDecodeError::Truncated { + expected: digest_offset, + observed: encoded.len(), + })?; + let expected_digest = hash(b"keep.retention-root/v2\0", digest_preimage); + if observed_digest != expected_digest { + return Err(RetentionRootDecodeError::RootDigestMismatch { + expected: expected_digest, + observed: observed_digest, + }); + } + Ok(expected_digest) +} + +pub(super) fn verify_anchor_set( + anchor_count: u32, + anchors: &[u8], + observed: [u8; 32], +) -> Result<(), RetentionRootDecodeError> { + let mut hasher = blake3::Hasher::new(); + hasher.update(b"keep.retention-anchor-set/v2\0"); + hasher.update(&anchor_count.to_be_bytes()); + hasher.update(anchors); + let expected = *hasher.finalize().as_bytes(); + if observed == expected { + Ok(()) + } else { + Err(RetentionRootDecodeError::AnchorSetDigestMismatch { expected, observed }) + } +} + +fn read_digest(encoded: &[u8], offset: usize) -> Result<[u8; 32], RetentionRootDecodeError> { + let end = offset + .checked_add(32) + .ok_or(RetentionRootDecodeError::LengthOverflow)?; + let bytes = encoded + .get(offset..end) + .ok_or(RetentionRootDecodeError::Truncated { + expected: end, + observed: encoded.len(), + })?; + <[u8; 32]>::try_from(bytes).map_err(|_| RetentionRootDecodeError::Truncated { + expected: end, + observed: encoded.len(), + }) +} + +fn hash(domain: &[u8], bytes: &[u8]) -> [u8; 32] { + let mut hasher = blake3::Hasher::new(); + hasher.update(domain); + hasher.update(bytes); + *hasher.finalize().as_bytes() +} diff --git a/src/adapters/retention/root_semantic_header.rs b/src/adapters/retention/root_semantic_header.rs new file mode 100644 index 0000000..17e9d9d --- /dev/null +++ b/src/adapters/retention/root_semantic_header.rs @@ -0,0 +1,55 @@ +//! This boundary module owns post-integrity retention header admission. + +use super::RetentionRootDecodeError; +use super::root_header_decoder::DecodedRootHeader; +use crate::{ + RegisteredRetentionProfile, RetentionClosureLimits, RetentionRoot, RetentionRootDigest, + RootGeneration, +}; + +pub(super) struct AdmittedRootHeader { + pub(super) generation: RootGeneration, + pub(super) profile: RegisteredRetentionProfile, + pub(super) limits: RetentionClosureLimits, + pub(super) predecessor: Option, +} + +pub(super) fn admit( + header: &DecodedRootHeader, +) -> Result { + if header.anchor_count > RetentionRoot::MAXIMUM_ANCHOR_COUNT { + return Err(RetentionRootDecodeError::AnchorCountExceeded { + maximum: RetentionRoot::MAXIMUM_ANCHOR_COUNT, + observed: header.anchor_count, + }); + } + let generation = RootGeneration::new(header.generation) + .map_err(|source| RetentionRootDecodeError::Generation { source })?; + let profile = RegisteredRetentionProfile::admit( + header.profile_identity, + header.profile_version, + header.profile_digest, + ) + .map_err(|source| RetentionRootDecodeError::Profile { source })?; + let limits = RetentionClosureLimits::new( + header.closure_nodes, + header.closure_depth, + header.closure_encoded_bytes, + header.closure_physical_bytes, + ) + .map_err(|source| RetentionRootDecodeError::ClosureLimit { source })?; + Ok(AdmittedRootHeader { + generation, + profile, + limits, + predecessor: predecessor(header.predecessor), + }) +} + +fn predecessor(bytes: [u8; 32]) -> Option { + if bytes == [0_u8; 32] { + None + } else { + Some(RetentionRootDigest::from_hash(bytes)) + } +} diff --git a/src/lib.rs b/src/lib.rs index 72dc778..e248621 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -22,8 +22,9 @@ //! continuation has a storage-independent planning and execution boundary plus //! a pinned writer-authorized filesystem adapter. Core retention namespaces, //! generations, realization policy, reconstruction anchors, and semantic roots -//! are validated; canonical root encoding is available. Retention decoding, -//! publication, recovery, and garbage collection remain intentionally absent. +//! are validated; canonical in-memory root encoding and decoding are available. +//! Retention publication, recovery, and garbage collection remain intentionally +//! absent. #[cfg(test)] extern crate self as keep; @@ -41,9 +42,9 @@ mod retention; #[doc(hidden)] pub use adapters::RepositoryInitializationStorage; pub use adapters::{ - AdmittedCatalog, AdmittedRecoveryStageBytes, AdmittedSegment, AdmittedSegmentRecord, - BlobIdBinaryParseError, BlobIdTextParseError, CanonicalCatalog, CanonicalLayoutRecord, - CanonicalPublicationHead, CanonicalRetentionRoot, CatalogAdmissionError, + AdmittedCatalog, AdmittedRecoveryStageBytes, AdmittedRetentionRoot, AdmittedSegment, + AdmittedSegmentRecord, BlobIdBinaryParseError, BlobIdTextParseError, CanonicalCatalog, + CanonicalLayoutRecord, CanonicalPublicationHead, CanonicalRetentionRoot, CatalogAdmissionError, CatalogAllocationPhase, CatalogDecodeError, CatalogEncodeError, CatalogEntryDecodeError, CatalogPublicationError, CatalogPublicationExpectation, CatalogPublicationOutcome, CatalogPublicationPhase, CatalogPublicationReadiness, CatalogPublicationReceipt, @@ -84,8 +85,8 @@ pub use adapters::{ RecoveryStageFingerprintAlgorithm, RecoveryStageFingerprintError, RecoveryStageLength, RecoveryStageMetadata, RecoveryStageMetadataError, RecoveryStageNamespacePhase, RecoveryStageParent, RecoveryStagePoolOutcome, RecoveryStageSynchronizationOutcome, - RetentionRootEncodeError, ReusableRecoverySegment, SealedSegment, SegmentDigest, - SegmentDurabilityPhase, SegmentHeader, SegmentHeaderError, SegmentPublication, + RetentionRootDecodeError, RetentionRootEncodeError, ReusableRecoverySegment, SealedSegment, + SegmentDigest, SegmentDurabilityPhase, SegmentHeader, SegmentHeaderError, SegmentPublication, SegmentPublicationError, SegmentReadError, SegmentReadPolicy, SegmentRecordAdmissionError, SegmentRecordChecksum, SegmentRecordDecodeError, SegmentRecordHeader, SegmentRecordHeaderError, SegmentRecordIdentity, SegmentRecordLength, SegmentRecordLimit, SegmentRecordLimitError, diff --git a/tests/retention_root_decoding.rs b/tests/retention_root_decoding.rs new file mode 100644 index 0000000..9405541 --- /dev/null +++ b/tests/retention_root_decoding.rs @@ -0,0 +1,189 @@ +//! Public decoding and integrity laws for version-2 retention roots. + +mod support; + +use std::io; + +use keep::{AdmittedRetentionRoot, RetentionRootDecodeError}; + +const ONE_ANCHOR_ROOT: &str = include_str!("../conformance/segment-store/v2/one-anchor-root.hex"); +const ANCHOR_SET_DIGEST_OFFSET: usize = 148; +const ANCHOR_BODY_OFFSET: usize = 195; +const ROOT_DIGEST_OFFSET: usize = 314; +const CHECKSUM_OFFSET: usize = 346; + +#[test] +fn frozen_root_decodes_to_one_complete_semantic_generation() +-> Result<(), Box> { + let bytes = fixture_bytes()?; + let admitted = AdmittedRetentionRoot::decode(&bytes)?; + assert_eq!(admitted.encoded(), bytes); + assert_eq!(admitted.root().namespace().as_bytes(), &[0x00, 0x2f, 0xff]); + assert_eq!(admitted.root().generation().get(), 1); + assert_eq!(admitted.root().anchor_count(), 1); + assert_eq!( + admitted.digest().as_bytes(), + bytes.get(314..346).ok_or_else(|| { + io::Error::other("frozen retention root lacks its embedded digest") + })? + ); + Ok(()) +} + +#[test] +fn root_framing_refuses_truncation_trailing_data_and_magic_substitution() +-> Result<(), Box> { + let bytes = fixture_bytes()?; + let mut truncated = bytes.clone(); + assert!(truncated.pop().is_some()); + assert!(matches!( + AdmittedRetentionRoot::decode(&truncated), + Err(RetentionRootDecodeError::Truncated { + expected: 378, + observed: 377, + }) + )); + + let mut trailing = bytes.clone(); + trailing.push(0); + assert!(matches!( + AdmittedRetentionRoot::decode(&trailing), + Err(RetentionRootDecodeError::TrailingData { + expected: 378, + observed: 379, + }) + )); + + let mut wrong_magic = bytes; + let first = wrong_magic + .first_mut() + .ok_or_else(|| io::Error::other("frozen retention root is empty"))?; + *first ^= 1; + assert!(matches!( + AdmittedRetentionRoot::decode(&wrong_magic), + Err(RetentionRootDecodeError::InvalidMagic { .. }) + )); + Ok(()) +} + +#[test] +fn root_checksum_and_digest_have_distinct_integrity_refusals() +-> Result<(), Box> { + let bytes = fixture_bytes()?; + let mut checksum_corruption = bytes.clone(); + let last = checksum_corruption + .last_mut() + .ok_or_else(|| io::Error::other("frozen retention root is empty"))?; + *last ^= 1; + assert!(matches!( + AdmittedRetentionRoot::decode(&checksum_corruption), + Err(RetentionRootDecodeError::ChecksumMismatch { .. }) + )); + + let mut digest_corruption = bytes; + let digest_byte = digest_corruption + .get_mut(314) + .ok_or_else(|| io::Error::other("frozen retention root lacks digest bytes"))?; + *digest_byte ^= 1; + refresh_checksum(&mut digest_corruption)?; + assert!(matches!( + AdmittedRetentionRoot::decode(&digest_corruption), + Err(RetentionRootDecodeError::RootDigestMismatch { .. }) + )); + Ok(()) +} + +#[test] +fn semantic_fields_are_admitted_only_after_complete_integrity() +-> Result<(), Box> { + let mut bytes = fixture_bytes()?; + bytes + .get_mut(32..40) + .ok_or_else(|| io::Error::other("frozen retention root lacks generation bytes"))? + .fill(0); + assert!(matches!( + AdmittedRetentionRoot::decode(&bytes), + Err(RetentionRootDecodeError::ChecksumMismatch { .. }) + )); + + refresh_root_digest_and_checksum(&mut bytes)?; + assert!(matches!( + AdmittedRetentionRoot::decode(&bytes), + Err(RetentionRootDecodeError::Generation { .. }) + )); + Ok(()) +} + +#[test] +fn anchor_set_integrity_precedes_nested_identity_admission() +-> Result<(), Box> { + let mut bytes = fixture_bytes()?; + let first_anchor_byte = bytes + .get_mut(ANCHOR_BODY_OFFSET) + .ok_or_else(|| io::Error::other("frozen retention root lacks its anchor body"))?; + *first_anchor_byte ^= 1; + refresh_root_digest_and_checksum(&mut bytes)?; + assert!(matches!( + AdmittedRetentionRoot::decode(&bytes), + Err(RetentionRootDecodeError::AnchorSetDigestMismatch { .. }) + )); + + refresh_anchor_set_digest(&mut bytes)?; + refresh_root_digest_and_checksum(&mut bytes)?; + assert!(matches!( + AdmittedRetentionRoot::decode(&bytes), + Err(RetentionRootDecodeError::BlobId { index: 0, .. }) + )); + Ok(()) +} + +fn fixture_bytes() -> Result, io::Error> { + let encoded = ONE_ANCHOR_ROOT + .strip_suffix('\n') + .ok_or_else(|| io::Error::other("retention root fixture lacks final newline"))?; + support::decode_hex(encoded) +} + +fn refresh_anchor_set_digest(bytes: &mut [u8]) -> Result<(), io::Error> { + let anchors = bytes + .get(ANCHOR_BODY_OFFSET..ROOT_DIGEST_OFFSET) + .ok_or_else(|| io::Error::other("retention root lacks its anchor body"))?; + let mut hasher = blake3::Hasher::new(); + hasher.update(b"keep.retention-anchor-set/v2\0"); + hasher.update(&1_u32.to_be_bytes()); + hasher.update(anchors); + let digest = *hasher.finalize().as_bytes(); + bytes + .get_mut(ANCHOR_SET_DIGEST_OFFSET..ANCHOR_SET_DIGEST_OFFSET + 32) + .ok_or_else(|| io::Error::other("retention root lacks its anchor-set digest"))? + .copy_from_slice(&digest); + Ok(()) +} + +fn refresh_root_digest_and_checksum(bytes: &mut [u8]) -> Result<(), io::Error> { + let preimage = bytes + .get(..ROOT_DIGEST_OFFSET) + .ok_or_else(|| io::Error::other("retention root lacks its root digest preimage"))?; + let mut hasher = blake3::Hasher::new(); + hasher.update(b"keep.retention-root/v2\0"); + hasher.update(preimage); + let digest = *hasher.finalize().as_bytes(); + bytes + .get_mut(ROOT_DIGEST_OFFSET..CHECKSUM_OFFSET) + .ok_or_else(|| io::Error::other("retention root lacks its root digest"))? + .copy_from_slice(&digest); + refresh_checksum(bytes) +} + +fn refresh_checksum(bytes: &mut [u8]) -> Result<(), io::Error> { + let checksum_offset = bytes + .len() + .checked_sub(32) + .ok_or_else(|| io::Error::other("retention root lacks a checksum"))?; + let (preimage, checksum_slot) = bytes.split_at_mut(checksum_offset); + let mut hasher = blake3::Hasher::new(); + hasher.update(b"keep.retention-root-checksum/v2\0"); + hasher.update(preimage); + checksum_slot.copy_from_slice(hasher.finalize().as_bytes()); + Ok(()) +} From fe684e2dd9fd776c2e0ae4c4fb5b2fd0ddc4d2ca Mon Sep 17 00:00:00 2001 From: James Ross Date: Wed, 29 Jul 2026 21:12:04 -0700 Subject: [PATCH 006/111] Fix: Align segment-store registry contract --- xtask/tests/layout_format_contract.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/xtask/tests/layout_format_contract.rs b/xtask/tests/layout_format_contract.rs index 6f1340c..71ccd52 100644 --- a/xtask/tests/layout_format_contract.rs +++ b/xtask/tests/layout_format_contract.rs @@ -91,8 +91,8 @@ fn format_registry_reports_flat_layout_as_implemented() { fn format_registry_reports_the_segment_store_implementation_boundary() { const EXPECTED_ROW: &str = "\ | [Durable Segment Store v1](segment-store-v1/README.md) | \ -`keep.segment-store/v1` | Specified in issue #14; segment I/O implemented in \ -issue #15; publication and recovery remain in issues #16–#17 | \ +`keep.segment-store/v1` | Implemented through initialization, publication, \ +restart, and recovery in issues #14–#17 | \ [Golden corpus](../../conformance/segment-store/v1/README.md) |"; assert!( From 99ed9977130ea86d66249047e2ef5030c3d3187b Mon Sep 17 00:00:00 2001 From: James Ross Date: Wed, 29 Jul 2026 21:13:35 -0700 Subject: [PATCH 007/111] Fix: Refresh segment-store documentation law --- xtask/tests/segment_store_implementation_documentation.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/xtask/tests/segment_store_implementation_documentation.rs b/xtask/tests/segment_store_implementation_documentation.rs index dcead01..cb616af 100644 --- a/xtask/tests/segment_store_implementation_documentation.rs +++ b/xtask/tests/segment_store_implementation_documentation.rs @@ -11,7 +11,10 @@ fn living_documentation_names_the_implemented_segment_boundary() { for (document, claim) in [ (ROOT_README, "`StagedSegment`"), (ROOT_README, "`AdmittedSegment`"), - (FORMAT_REGISTRY, "segment I/O implemented in issue #15"), + ( + FORMAT_REGISTRY, + "Implemented through initialization, publication, restart, and recovery in issues #14–#17", + ), ( FORMAT_README, "Segment writing and verified reading are implemented in issue #15", From e5d7521f3a0de640b9d3a44d79e523fdefd12b2a Mon Sep 17 00:00:00 2001 From: James Ross Date: Wed, 29 Jul 2026 21:15:20 -0700 Subject: [PATCH 008/111] Add: Admit canonical retention manifests --- CHANGELOG.md | 9 +- README.md | 7 +- docs/formats/segment-store-v2/requirements.md | 2 +- docs/formats/segment-store-v2/retention.md | 6 +- src/adapters/mod.rs | 5 +- src/adapters/retention.rs | 16 ++ src/adapters/retention/admitted_manifest.rs | 58 +++++++ src/adapters/retention/canonical_manifest.rs | 44 ++++++ .../retention/manifest_decode_error.rs | 124 +++++++++++++++ .../manifest_decode_error_display.rs | 109 +++++++++++++ src/adapters/retention/manifest_decoder.rs | 35 ++++ .../retention/manifest_encode_error.rs | 44 ++++++ src/adapters/retention/manifest_encoder.rs | 137 ++++++++++++++++ .../retention/manifest_entry_decoder.rs | 69 ++++++++ .../retention/manifest_field_decoder.rs | 106 +++++++++++++ .../retention/manifest_header_decoder.rs | 91 +++++++++++ src/adapters/retention/manifest_integrity.rs | 82 ++++++++++ .../retention/manifest_semantic_header.rs | 35 ++++ src/lib.rs | 35 ++-- src/retention/manifest.rs | 108 +++++++++++++ src/retention/manifest_digest.rs | 18 +++ src/retention/manifest_entry.rs | 42 +++++ src/retention/manifest_error.rs | 60 +++++++ src/retention/mod.rs | 8 + src/retention/namespace_digest.rs | 2 +- tests/retention_manifest_codec.rs | 92 +++++++++++ .../retention_manifest_codec/refusal_laws.rs | 149 ++++++++++++++++++ 27 files changed, 1464 insertions(+), 29 deletions(-) create mode 100644 src/adapters/retention/admitted_manifest.rs create mode 100644 src/adapters/retention/canonical_manifest.rs create mode 100644 src/adapters/retention/manifest_decode_error.rs create mode 100644 src/adapters/retention/manifest_decode_error_display.rs create mode 100644 src/adapters/retention/manifest_decoder.rs create mode 100644 src/adapters/retention/manifest_encode_error.rs create mode 100644 src/adapters/retention/manifest_encoder.rs create mode 100644 src/adapters/retention/manifest_entry_decoder.rs create mode 100644 src/adapters/retention/manifest_field_decoder.rs create mode 100644 src/adapters/retention/manifest_header_decoder.rs create mode 100644 src/adapters/retention/manifest_integrity.rs create mode 100644 src/adapters/retention/manifest_semantic_header.rs create mode 100644 src/retention/manifest.rs create mode 100644 src/retention/manifest_digest.rs create mode 100644 src/retention/manifest_entry.rs create mode 100644 src/retention/manifest_error.rs create mode 100644 tests/retention_manifest_codec.rs create mode 100644 tests/retention_manifest_codec/refusal_laws.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index d9ecd3e..233537f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -312,9 +312,12 @@ after its public API and format compatibility policies are established. encoder reproduces the independent version-2 golden bytes, and the decoder verifies framing, checksum, root digest, anchor-set digest, nested identities, resource bounds, canonical anchor order, and semantic invariants before - admission. Version-1 immutable bytes remain authoritative; production - version-2 writing remains unavailable until issue #19's executable evidence - is complete. + admission. Validated global manifest values and their canonical encoder and + decoder now reproduce the independent manifest fixture and enforce liveness + history, namespace uniqueness, bounds, ordering, and all three integrity + layers. Version-1 immutable bytes remain authoritative; production version-2 + writing remains unavailable until issue #19's executable evidence is + complete. - Accepted ADR-0009 defines caller-supplied retention namespaces, `BlobId`/`LayoutId` reconstruction anchors, fail-closed canonical closure, generation-checked retention publication, immutable liveness snapshots, diff --git a/README.md b/README.md index 6cc0d33..6198612 100644 --- a/README.md +++ b/README.md @@ -116,9 +116,10 @@ recovery classifiers and immutable-artifact admission, and reconstructs the exact published generation and visible one-zero chunk when `HEAD` exists. This matrix proves application process-death behavior; it does not simulate host power loss. Version-2 retention values and canonical in-memory root encoding -and decoding are implemented. Retention publication, recovery, compaction, and -garbage collection remain planned. Presence in the reference CAS does not -claim retention, crash recovery, or durability. +and decoding are implemented, as are the global manifest values and codec. +Retention-head codecs, publication, recovery, compaction, and garbage +collection remain planned. Presence in the reference CAS does not claim +retention, crash recovery, or durability. Run the complete debug-profile matrix: diff --git a/docs/formats/segment-store-v2/requirements.md b/docs/formats/segment-store-v2/requirements.md index aeebfad..89cd612 100644 --- a/docs/formats/segment-store-v2/requirements.md +++ b/docs/formats/segment-store-v2/requirements.md @@ -10,7 +10,7 @@ case is not evidence. | ID | Requirement | Evidence | Status | | --- | --- | --- | --- | | `KEEP-RETENTION-001` | `RetentionNamespace`, `RootGeneration`, `LivenessGeneration`, profile coordinates, limits, anchors, and digests are validated typed values | `tests/retention_values.rs` and `tests/retention_root_encoding.rs` | Implemented | -| `KEEP-RETENTION-002` | Root, manifest, and head codecs implement the exact canonical grammars and fixed bounds | independent golden corpus plus root encoder and decoder evidence in `tests/retention_root_encoding.rs` and `tests/retention_root_decoding.rs`; manifest/head codecs remain | In progress in #19 | +| `KEEP-RETENTION-002` | Root, manifest, and head codecs implement the exact canonical grammars and fixed bounds | independent golden corpus plus root and manifest evidence in `tests/retention_root_encoding.rs`, `tests/retention_root_decoding.rs`, and `tests/retention_manifest_codec.rs`; head codec remains | In progress in #19 | | `KEEP-RETENTION-003` | Every structural field, truncation boundary, ordering law, duplicate, overflow, flag, reserved byte, digest, checksum, and trailing byte has a precise refusal | corruption matrix and fuzz | Planned in #19 | | `KEEP-RETENTION-004` | Retain and release compare expected and observed generations and publish exact successors only | unit and model-based transition tests | Planned in #19 | | `KEEP-RETENTION-005` | Closure derivation is deterministic, bounded, cycle-safe, fail-closed, and verifies complete blob reconstruction | property, corruption, and adversarial catalog tests | Planned in #19 | diff --git a/docs/formats/segment-store-v2/retention.md b/docs/formats/segment-store-v2/retention.md index 3f57f19..46d6cbc 100644 --- a/docs/formats/segment-store-v2/retention.md +++ b/docs/formats/segment-store-v2/retention.md @@ -160,9 +160,9 @@ retention/roots// ``` Names with alternate width, case, suffix, generation, or digest refuse. Keep -implements validated in-memory root encoding and decoding with complete -integrity verification before semantic admission. Filesystem publication, -manifest/head codecs, transitions, recovery, and garbage collection remain absent. +implements validated in-memory root and manifest codecs with integrity before +semantic admission. Filesystem publication, the head codec, transitions, +recovery, and garbage collection remain absent. ## Global retention manifest diff --git a/src/adapters/mod.rs b/src/adapters/mod.rs index 111841a..7033c63 100644 --- a/src/adapters/mod.rs +++ b/src/adapters/mod.rs @@ -450,8 +450,9 @@ pub use recovery_stage_synchronization_outcome::RecoveryStageSynchronizationOutc #[cfg(feature = "repository-tasks")] pub use repository_initialization_storage::RepositoryInitializationStorage; pub use retention::{ - AdmittedRetentionRoot, CanonicalRetentionRoot, RetentionRootDecodeError, - RetentionRootEncodeError, + AdmittedRetentionManifest, AdmittedRetentionRoot, CanonicalRetentionManifest, + CanonicalRetentionRoot, RetentionManifestDecodeError, RetentionManifestEncodeError, + RetentionRootDecodeError, RetentionRootEncodeError, }; pub use sealed_segment::SealedSegment; pub use segment_digest::SegmentDigest; diff --git a/src/adapters/retention.rs b/src/adapters/retention.rs index 5fa3118..3cebdd7 100644 --- a/src/adapters/retention.rs +++ b/src/adapters/retention.rs @@ -1,7 +1,19 @@ //! This module owns canonical retention record boundary adapters. +mod admitted_manifest; mod admitted_root; +mod canonical_manifest; mod canonical_root; +mod manifest_decode_error; +mod manifest_decode_error_display; +mod manifest_decoder; +mod manifest_encode_error; +mod manifest_encoder; +mod manifest_entry_decoder; +mod manifest_field_decoder; +mod manifest_header_decoder; +mod manifest_integrity; +mod manifest_semantic_header; mod root_anchor_decoder; mod root_decode_error; mod root_decode_error_display; @@ -13,7 +25,11 @@ mod root_header_decoder; mod root_integrity; mod root_semantic_header; +pub use admitted_manifest::AdmittedRetentionManifest; pub use admitted_root::AdmittedRetentionRoot; +pub use canonical_manifest::CanonicalRetentionManifest; pub use canonical_root::CanonicalRetentionRoot; +pub use manifest_decode_error::RetentionManifestDecodeError; +pub use manifest_encode_error::RetentionManifestEncodeError; pub use root_decode_error::RetentionRootDecodeError; pub use root_encode_error::RetentionRootEncodeError; diff --git a/src/adapters/retention/admitted_manifest.rs b/src/adapters/retention/admitted_manifest.rs new file mode 100644 index 0000000..b06aa2d --- /dev/null +++ b/src/adapters/retention/admitted_manifest.rs @@ -0,0 +1,58 @@ +//! This boundary module owns one decoded and admitted retention manifest. + +use super::{RetentionManifestDecodeError, manifest_decoder}; +use crate::{RetentionManifest, RetentionManifestDigest}; + +/// Borrowed canonical bytes paired with their admitted semantic manifest. +/// +/// Decoding verifies exact framing, the complete-record checksum, manifest and +/// entry-set digests, ordered entries, resource bounds, and generation-history +/// invariants. Entry allocation is bounded by a verified count. Decoding +/// performs no I/O. +#[must_use] +#[derive(Debug, Eq, PartialEq)] +pub struct AdmittedRetentionManifest<'encoded> { + encoded: &'encoded [u8], + manifest: RetentionManifest, + digest: RetentionManifestDigest, +} + +impl<'encoded> AdmittedRetentionManifest<'encoded> { + /// Decodes and admits one exact canonical version-2 manifest record. + /// + /// # Errors + /// + /// Returns [`RetentionManifestDecodeError`] at the first violated framing, + /// integrity, resource-bound, ordering, or semantic invariant. + pub fn decode(encoded: &'encoded [u8]) -> Result { + manifest_decoder::decode(encoded) + } + + /// Returns the complete verified canonical bytes. + #[must_use] + pub const fn encoded(&self) -> &'encoded [u8] { + self.encoded + } + + /// Returns the admitted semantic manifest. + pub const fn manifest(&self) -> &RetentionManifest { + &self.manifest + } + + /// Returns the verified canonical manifest digest. + pub const fn digest(&self) -> RetentionManifestDigest { + self.digest + } + + pub(super) const fn admitted( + encoded: &'encoded [u8], + manifest: RetentionManifest, + digest: RetentionManifestDigest, + ) -> Self { + Self { + encoded, + manifest, + digest, + } + } +} diff --git a/src/adapters/retention/canonical_manifest.rs b/src/adapters/retention/canonical_manifest.rs new file mode 100644 index 0000000..8c08a28 --- /dev/null +++ b/src/adapters/retention/canonical_manifest.rs @@ -0,0 +1,44 @@ +//! This boundary module owns materialized canonical retention manifest bytes. + +use super::{RetentionManifestEncodeError, manifest_encoder}; +use crate::{RetentionManifest, RetentionManifestDigest}; + +/// Owned canonical version-2 retention manifest record. +/// +/// The complete record is materialized in memory after semantic bounds are +/// admitted and exact checked length calculation succeeds. +#[must_use] +#[derive(Debug, Eq, PartialEq)] +pub struct CanonicalRetentionManifest { + encoded: Vec, + digest: RetentionManifestDigest, +} + +impl CanonicalRetentionManifest { + /// Encodes one validated semantic retention manifest. + /// + /// # Errors + /// + /// Returns [`RetentionManifestEncodeError`] for checked length overflow, + /// allocation refusal, or an internal construction-length mismatch. + pub fn from_manifest( + manifest: &RetentionManifest, + ) -> Result { + manifest_encoder::encode(manifest) + } + + /// Returns the complete canonical manifest bytes. + #[must_use] + pub fn encoded(&self) -> &[u8] { + &self.encoded + } + + /// Returns the canonical manifest digest embedded in the record. + pub const fn digest(&self) -> RetentionManifestDigest { + self.digest + } + + pub(super) const fn admitted(encoded: Vec, digest: RetentionManifestDigest) -> Self { + Self { encoded, digest } + } +} diff --git a/src/adapters/retention/manifest_decode_error.rs b/src/adapters/retention/manifest_decode_error.rs new file mode 100644 index 0000000..4f00dc8 --- /dev/null +++ b/src/adapters/retention/manifest_decode_error.rs @@ -0,0 +1,124 @@ +//! This boundary module owns typed retention manifest decoding failures. + +use std::collections::TryReserveError; + +use crate::{LivenessGenerationError, RetentionManifestError, RootGenerationError}; + +/// Failure to decode and admit one version-2 retention manifest. +#[derive(Debug)] +pub enum RetentionManifestDecodeError { + /// The byte string ended before its required exact length. + Truncated { + /// Required byte length. + expected: usize, + /// Observed byte length. + observed: usize, + }, + /// Bytes followed the required exact record. + TrailingData { + /// Required byte length. + expected: usize, + /// Observed byte length. + observed: usize, + }, + /// The fixed record magic was not canonical. + InvalidMagic { + /// Observed 16 magic bytes. + observed: [u8; 16], + }, + /// The format version is unsupported. + UnsupportedVersion { + /// Supported version. + expected: u16, + /// Observed version. + observed: u16, + }, + /// The fixed header width was not canonical. + InvalidHeaderLength { + /// Required header width. + expected: u16, + /// Observed width. + observed: u16, + }, + /// The record carried unsupported flags. + UnsupportedFlags { + /// Observed flag bits. + observed: u32, + }, + /// The declared total length disagreed with canonical field arithmetic. + DeclaredLengthMismatch { + /// Canonical computed length. + expected: u64, + /// Declared length. + observed: u64, + }, + /// Checked record-length arithmetic overflowed. + LengthOverflow, + /// The fixed entry width was not canonical. + InvalidEntryWidth { + /// Required entry width. + expected: u16, + /// Observed entry width. + observed: u16, + }, + /// A reserved field was nonzero. + NonZeroReserved { + /// Protocol field name. + field: &'static str, + }, + /// Liveness-generation admission failed. + LivenessGeneration { + /// Preserved generation failure. + source: LivenessGenerationError, + }, + /// The declared entry count exceeded the fixed bound. + EntryCountExceeded { + /// Fixed maximum count. + maximum: u32, + /// Observed count. + observed: u32, + }, + /// One entry contained an invalid root generation. + RootGeneration { + /// Zero-based entry index. + index: u32, + /// Preserved generation failure. + source: RootGenerationError, + }, + /// Canonical namespace ordering was violated. + NonCanonicalEntryOrder { + /// Zero-based index of the observed entry. + index: u32, + }, + /// Entry allocation was refused. + Allocation { + /// Preserved allocation failure. + source: TryReserveError, + }, + /// The entry-set digest did not match the exact body. + EntrySetDigestMismatch { + /// Computed canonical digest. + expected: [u8; 32], + /// Digest stored in the header. + observed: [u8; 32], + }, + /// The manifest digest did not match the exact header and body. + ManifestDigestMismatch { + /// Computed canonical digest. + expected: [u8; 32], + /// Digest stored in the record. + observed: [u8; 32], + }, + /// The checksum did not match the complete digest-bearing prefix. + ChecksumMismatch { + /// Computed canonical checksum. + expected: [u8; 32], + /// Checksum stored in the record. + observed: [u8; 32], + }, + /// Final semantic manifest admission failed. + Semantic { + /// Preserved semantic failure. + source: RetentionManifestError, + }, +} diff --git a/src/adapters/retention/manifest_decode_error_display.rs b/src/adapters/retention/manifest_decode_error_display.rs new file mode 100644 index 0000000..76a81dd --- /dev/null +++ b/src/adapters/retention/manifest_decode_error_display.rs @@ -0,0 +1,109 @@ +//! This boundary module owns retention manifest decode diagnostics and sources. + +use std::{error::Error, fmt}; + +use super::RetentionManifestDecodeError; + +impl fmt::Display for RetentionManifestDecodeError { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::Truncated { expected, observed } => write!( + formatter, + "retention manifest has {observed} bytes; expected {expected}" + ), + Self::TrailingData { expected, observed } => write!( + formatter, + "retention manifest has trailing data: expected {expected} bytes, observed {observed}" + ), + Self::InvalidMagic { observed } => { + write!( + formatter, + "invalid retention manifest magic {observed:02x?}" + ) + } + Self::UnsupportedVersion { expected, observed } => write!( + formatter, + "unsupported retention manifest version {observed}; expected {expected}" + ), + Self::InvalidHeaderLength { expected, observed } => write!( + formatter, + "retention manifest header length {observed}; expected {expected}" + ), + Self::UnsupportedFlags { observed } => { + write!( + formatter, + "unsupported retention manifest flags {observed:#010x}" + ) + } + Self::DeclaredLengthMismatch { expected, observed } => write!( + formatter, + "retention manifest declares {observed} bytes; canonical fields require {expected}" + ), + Self::LengthOverflow => formatter.write_str("retention manifest length overflow"), + Self::InvalidEntryWidth { expected, observed } => write!( + formatter, + "retention manifest entry width {observed}; expected {expected}" + ), + Self::NonZeroReserved { field } => write!( + formatter, + "retention manifest {field} reserved bytes are nonzero" + ), + Self::LivenessGeneration { source } => { + write!(formatter, "invalid liveness generation: {source}") + } + Self::EntryCountExceeded { maximum, observed } => write!( + formatter, + "retention manifest declares {observed} entries; maximum is {maximum}" + ), + Self::RootGeneration { index, source } => write!( + formatter, + "invalid root generation in retention entry {index}: {source}" + ), + Self::NonCanonicalEntryOrder { index } => write!( + formatter, + "retention manifest entry {index} is not greater than its predecessor" + ), + Self::Allocation { .. } => { + formatter.write_str("retention manifest entry allocation failed") + } + Self::EntrySetDigestMismatch { .. } => { + formatter.write_str("retention manifest entry-set digest mismatch") + } + Self::ManifestDigestMismatch { .. } => { + formatter.write_str("retention manifest digest mismatch") + } + Self::ChecksumMismatch { .. } => { + formatter.write_str("retention manifest checksum mismatch") + } + Self::Semantic { source } => { + write!(formatter, "invalid semantic retention manifest: {source}") + } + } + } +} + +impl Error for RetentionManifestDecodeError { + fn source(&self) -> Option<&(dyn Error + 'static)> { + match self { + Self::LivenessGeneration { source } => Some(source), + Self::RootGeneration { source, .. } => Some(source), + Self::Allocation { source } => Some(source), + Self::Semantic { source } => Some(source), + Self::Truncated { .. } + | Self::TrailingData { .. } + | Self::InvalidMagic { .. } + | Self::UnsupportedVersion { .. } + | Self::InvalidHeaderLength { .. } + | Self::UnsupportedFlags { .. } + | Self::DeclaredLengthMismatch { .. } + | Self::LengthOverflow + | Self::InvalidEntryWidth { .. } + | Self::NonZeroReserved { .. } + | Self::EntryCountExceeded { .. } + | Self::NonCanonicalEntryOrder { .. } + | Self::EntrySetDigestMismatch { .. } + | Self::ManifestDigestMismatch { .. } + | Self::ChecksumMismatch { .. } => None, + } + } +} diff --git a/src/adapters/retention/manifest_decoder.rs b/src/adapters/retention/manifest_decoder.rs new file mode 100644 index 0000000..6169c40 --- /dev/null +++ b/src/adapters/retention/manifest_decoder.rs @@ -0,0 +1,35 @@ +//! This boundary module owns canonical retention manifest decoding order. + +use super::manifest_header_decoder::HEADER_LENGTH; +use super::{ + AdmittedRetentionManifest, RetentionManifestDecodeError, manifest_entry_decoder, + manifest_header_decoder, manifest_integrity, manifest_semantic_header, +}; +use crate::{RetentionManifest, RetentionManifestDigest}; + +pub(super) fn decode( + encoded: &[u8], +) -> Result, RetentionManifestDecodeError> { + let header = manifest_header_decoder::decode(encoded)?; + let digest = manifest_integrity::verify(encoded, header.digest_offset, header.checksum_offset)?; + let entry_bytes = encoded.get(HEADER_LENGTH..header.digest_offset).ok_or( + RetentionManifestDecodeError::Truncated { + expected: header.digest_offset, + observed: encoded.len(), + }, + )?; + manifest_integrity::verify_entry_set(header.entry_count, entry_bytes, header.entry_set_digest)?; + let admitted_header = manifest_semantic_header::admit(&header)?; + let entries = manifest_entry_decoder::decode(entry_bytes, header.entry_count)?; + let manifest = RetentionManifest::new( + admitted_header.generation, + admitted_header.predecessor, + entries, + ) + .map_err(|source| RetentionManifestDecodeError::Semantic { source })?; + Ok(AdmittedRetentionManifest::admitted( + encoded, + manifest, + RetentionManifestDigest::from_hash(digest), + )) +} diff --git a/src/adapters/retention/manifest_encode_error.rs b/src/adapters/retention/manifest_encode_error.rs new file mode 100644 index 0000000..90a4385 --- /dev/null +++ b/src/adapters/retention/manifest_encode_error.rs @@ -0,0 +1,44 @@ +//! This boundary module owns retention manifest encoding failures. + +use std::{collections::TryReserveError, error::Error, fmt}; + +/// Failure to encode one canonical retention manifest record. +#[derive(Debug)] +pub enum RetentionManifestEncodeError { + /// Checked record-length arithmetic overflowed. + LengthOverflow, + /// Canonical byte allocation was refused. + Allocation { + /// Preserved allocation failure. + source: TryReserveError, + }, + /// Internal construction produced a noncanonical length. + ConstructionLength { + /// Required length. + expected: usize, + /// Constructed length. + observed: usize, + }, +} + +impl fmt::Display for RetentionManifestEncodeError { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::LengthOverflow => formatter.write_str("retention manifest length overflow"), + Self::Allocation { .. } => formatter.write_str("retention manifest allocation failed"), + Self::ConstructionLength { expected, observed } => write!( + formatter, + "retention manifest construction produced {observed} bytes; expected {expected}" + ), + } + } +} + +impl Error for RetentionManifestEncodeError { + fn source(&self) -> Option<&(dyn Error + 'static)> { + match self { + Self::Allocation { source } => Some(source), + Self::LengthOverflow | Self::ConstructionLength { .. } => None, + } + } +} diff --git a/src/adapters/retention/manifest_encoder.rs b/src/adapters/retention/manifest_encoder.rs new file mode 100644 index 0000000..1c53399 --- /dev/null +++ b/src/adapters/retention/manifest_encoder.rs @@ -0,0 +1,137 @@ +//! This boundary module owns canonical version-2 retention manifest encoding. + +use super::{CanonicalRetentionManifest, RetentionManifestEncodeError}; +use crate::{RetentionManifest, RetentionManifestDigest}; + +const HEADER_LENGTH: usize = 160; +const ENTRY_WIDTH: usize = 72; +const TRAILER_LENGTH: usize = 64; + +struct EncodingPlan { + total_length: usize, + digest_preimage_length: usize, + entry_set_digest: [u8; 32], +} + +pub(super) fn encode( + manifest: &RetentionManifest, +) -> Result { + let plan = plan(manifest)?; + let mut encoded = Vec::new(); + encoded + .try_reserve_exact(plan.total_length) + .map_err(|source| RetentionManifestEncodeError::Allocation { source })?; + write_header(&mut encoded, manifest, &plan)?; + write_entries(&mut encoded, manifest); + require_length(&encoded, plan.digest_preimage_length)?; + let digest = hash(b"keep.retention-manifest/v2\0", &encoded); + encoded.extend_from_slice(&digest); + let checksum = hash(b"keep.retention-manifest-checksum/v2\0", &encoded); + encoded.extend_from_slice(&checksum); + require_length(&encoded, plan.total_length)?; + Ok(CanonicalRetentionManifest::admitted( + encoded, + RetentionManifestDigest::from_hash(digest), + )) +} + +fn plan(manifest: &RetentionManifest) -> Result { + let entry_bytes = usize::try_from(manifest.entry_count()) + .map_err(|_| RetentionManifestEncodeError::LengthOverflow)? + .checked_mul(ENTRY_WIDTH) + .ok_or(RetentionManifestEncodeError::LengthOverflow)?; + let digest_preimage_length = HEADER_LENGTH + .checked_add(entry_bytes) + .ok_or(RetentionManifestEncodeError::LengthOverflow)?; + let total_length = digest_preimage_length + .checked_add(TRAILER_LENGTH) + .ok_or(RetentionManifestEncodeError::LengthOverflow)?; + Ok(EncodingPlan { + total_length, + digest_preimage_length, + entry_set_digest: entry_set_digest(manifest), + }) +} + +fn entry_set_digest(manifest: &RetentionManifest) -> [u8; 32] { + let mut hasher = blake3::Hasher::new(); + hasher.update(b"keep.retention-manifest-entries/v2\0"); + hasher.update(&manifest.entry_count().to_be_bytes()); + for entry in manifest.entries() { + hasher.update(entry.namespace().as_bytes()); + hasher.update(&entry.root_generation().get().to_be_bytes()); + hasher.update(entry.root_digest().as_bytes()); + } + *hasher.finalize().as_bytes() +} + +fn write_header( + encoded: &mut Vec, + manifest: &RetentionManifest, + plan: &EncodingPlan, +) -> Result<(), RetentionManifestEncodeError> { + encoded.extend_from_slice(b"KEEP:RET:LIVE2\0\0"); + push_u16(encoded, 2); + push_u16(encoded, 160); + push_u32(encoded, 0); + push_u64( + encoded, + u64::try_from(plan.total_length) + .map_err(|_| RetentionManifestEncodeError::LengthOverflow)?, + ); + push_u64(encoded, manifest.generation().get()); + push_u16(encoded, 72); + push_u16(encoded, 0); + push_u32(encoded, manifest.entry_count()); + encoded.extend_from_slice(&predecessor_bytes(manifest)); + encoded.extend_from_slice(&plan.entry_set_digest); + encoded.extend_from_slice(&[0_u8; 48]); + require_length(encoded, HEADER_LENGTH) +} + +fn predecessor_bytes(manifest: &RetentionManifest) -> [u8; 32] { + manifest + .predecessor() + .map_or([0_u8; 32], |digest| *digest.as_bytes()) +} + +fn write_entries(encoded: &mut Vec, manifest: &RetentionManifest) { + for entry in manifest.entries() { + encoded.extend_from_slice(entry.namespace().as_bytes()); + push_u64(encoded, entry.root_generation().get()); + encoded.extend_from_slice(entry.root_digest().as_bytes()); + } +} + +fn hash(domain: &[u8], bytes: &[u8]) -> [u8; 32] { + let mut hasher = blake3::Hasher::new(); + hasher.update(domain); + hasher.update(bytes); + *hasher.finalize().as_bytes() +} + +fn push_u16(encoded: &mut Vec, value: u16) { + encoded.extend_from_slice(&value.to_be_bytes()); +} + +fn push_u32(encoded: &mut Vec, value: u32) { + encoded.extend_from_slice(&value.to_be_bytes()); +} + +fn push_u64(encoded: &mut Vec, value: u64) { + encoded.extend_from_slice(&value.to_be_bytes()); +} + +const fn require_length( + encoded: &[u8], + expected: usize, +) -> Result<(), RetentionManifestEncodeError> { + if encoded.len() == expected { + Ok(()) + } else { + Err(RetentionManifestEncodeError::ConstructionLength { + expected, + observed: encoded.len(), + }) + } +} diff --git a/src/adapters/retention/manifest_entry_decoder.rs b/src/adapters/retention/manifest_entry_decoder.rs new file mode 100644 index 0000000..46d09fb --- /dev/null +++ b/src/adapters/retention/manifest_entry_decoder.rs @@ -0,0 +1,69 @@ +//! This boundary module owns canonical retention manifest entry decoding. + +use super::RetentionManifestDecodeError; +use super::manifest_field_decoder::require_exact; +use crate::{ + RetentionManifestEntry, RetentionNamespaceDigest, RetentionRootDigest, RootGeneration, +}; + +const ENTRY_WIDTH: usize = 72; + +pub(super) fn decode( + encoded: &[u8], + entry_count: u32, +) -> Result, RetentionManifestDecodeError> { + let capacity = + usize::try_from(entry_count).map_err(|_| RetentionManifestDecodeError::LengthOverflow)?; + let expected_length = capacity + .checked_mul(ENTRY_WIDTH) + .ok_or(RetentionManifestDecodeError::LengthOverflow)?; + require_exact(encoded, expected_length)?; + let mut entries = Vec::new(); + entries + .try_reserve_exact(capacity) + .map_err(|source| RetentionManifestDecodeError::Allocation { source })?; + let mut previous = None; + for (position, bytes) in encoded.chunks_exact(ENTRY_WIDTH).enumerate() { + let index = + u32::try_from(position).map_err(|_| RetentionManifestDecodeError::LengthOverflow)?; + let namespace = RetentionNamespaceDigest::from_hash(read_array(bytes, 0)?); + let root_generation = RootGeneration::new(read_u64(bytes, 32)?) + .map_err(|source| RetentionManifestDecodeError::RootGeneration { index, source })?; + let root_digest = RetentionRootDigest::from_hash(read_array(bytes, 40)?); + if let Some(prior) = previous + && namespace <= prior + { + return Err(RetentionManifestDecodeError::NonCanonicalEntryOrder { index }); + } + entries.push(RetentionManifestEntry::new( + namespace, + root_generation, + root_digest, + )); + previous = Some(namespace); + } + Ok(entries) +} + +fn read_u64(encoded: &[u8], offset: usize) -> Result { + read_array(encoded, offset).map(u64::from_be_bytes) +} + +fn read_array( + encoded: &[u8], + offset: usize, +) -> Result<[u8; WIDTH], RetentionManifestDecodeError> { + let end = offset + .checked_add(WIDTH) + .ok_or(RetentionManifestDecodeError::LengthOverflow)?; + let bytes = encoded + .get(offset..end) + .ok_or(RetentionManifestDecodeError::Truncated { + expected: end, + observed: encoded.len(), + })?; + <[u8; WIDTH]>::try_from(bytes).map_err(|_| RetentionManifestDecodeError::Truncated { + expected: end, + observed: encoded.len(), + }) +} diff --git a/src/adapters/retention/manifest_field_decoder.rs b/src/adapters/retention/manifest_field_decoder.rs new file mode 100644 index 0000000..80b1ae4 --- /dev/null +++ b/src/adapters/retention/manifest_field_decoder.rs @@ -0,0 +1,106 @@ +//! This boundary module owns fixed-width retention manifest field extraction. + +use std::cmp::Ordering; + +use super::RetentionManifestDecodeError; + +pub(super) fn require_exact( + encoded: &[u8], + expected: usize, +) -> Result<(), RetentionManifestDecodeError> { + match encoded.len().cmp(&expected) { + Ordering::Less => Err(RetentionManifestDecodeError::Truncated { + expected, + observed: encoded.len(), + }), + Ordering::Equal => Ok(()), + Ordering::Greater => Err(RetentionManifestDecodeError::TrailingData { + expected, + observed: encoded.len(), + }), + } +} + +pub(super) const fn require_minimum( + encoded: &[u8], + expected: usize, +) -> Result<(), RetentionManifestDecodeError> { + if encoded.len() < expected { + Err(RetentionManifestDecodeError::Truncated { + expected, + observed: encoded.len(), + }) + } else { + Ok(()) + } +} + +pub(super) fn require_zero( + encoded: &[u8], + offset: usize, + width: usize, + field: &'static str, +) -> Result<(), RetentionManifestDecodeError> { + let end = offset + .checked_add(width) + .ok_or(RetentionManifestDecodeError::LengthOverflow)?; + let bytes = encoded + .get(offset..end) + .ok_or(RetentionManifestDecodeError::Truncated { + expected: end, + observed: encoded.len(), + })?; + if bytes.iter().all(|byte| *byte == 0) { + Ok(()) + } else { + Err(RetentionManifestDecodeError::NonZeroReserved { field }) + } +} + +pub(super) fn require_u16( + encoded: &[u8], + offset: usize, + expected: u16, + error: F, +) -> Result<(), RetentionManifestDecodeError> +where + F: FnOnce(u16, u16) -> RetentionManifestDecodeError, +{ + let observed = read_u16(encoded, offset)?; + if observed == expected { + Ok(()) + } else { + Err(error(expected, observed)) + } +} + +pub(super) fn read_u16(encoded: &[u8], offset: usize) -> Result { + read_array(encoded, offset).map(u16::from_be_bytes) +} + +pub(super) fn read_u32(encoded: &[u8], offset: usize) -> Result { + read_array(encoded, offset).map(u32::from_be_bytes) +} + +pub(super) fn read_u64(encoded: &[u8], offset: usize) -> Result { + read_array(encoded, offset).map(u64::from_be_bytes) +} + +pub(super) fn read_array( + encoded: &[u8], + offset: usize, +) -> Result<[u8; WIDTH], RetentionManifestDecodeError> { + let end = offset + .checked_add(WIDTH) + .ok_or(RetentionManifestDecodeError::LengthOverflow)?; + let bytes = encoded + .get(offset..end) + .ok_or(RetentionManifestDecodeError::Truncated { + expected: end, + observed: encoded.len(), + })?; + <[u8; WIDTH]>::try_from(bytes).map_err(|_| RetentionManifestDecodeError::Truncated { + expected: end, + observed: encoded.len(), + }) +} diff --git a/src/adapters/retention/manifest_header_decoder.rs b/src/adapters/retention/manifest_header_decoder.rs new file mode 100644 index 0000000..d577564 --- /dev/null +++ b/src/adapters/retention/manifest_header_decoder.rs @@ -0,0 +1,91 @@ +//! This boundary module owns retention manifest header framing admission. + +use super::RetentionManifestDecodeError; +use super::manifest_field_decoder::{ + read_array, read_u32, read_u64, require_exact, require_minimum, require_u16, require_zero, +}; + +pub(super) const HEADER_LENGTH: usize = 160; +const ENTRY_WIDTH: usize = 72; +const TRAILER_LENGTH: usize = 64; + +pub(super) struct DecodedManifestHeader { + pub(super) generation: u64, + pub(super) entry_count: u32, + pub(super) predecessor: [u8; 32], + pub(super) entry_set_digest: [u8; 32], + pub(super) digest_offset: usize, + pub(super) checksum_offset: usize, +} + +pub(super) fn decode( + encoded: &[u8], +) -> Result { + require_minimum(encoded, HEADER_LENGTH)?; + validate_fixed_fields(encoded)?; + let entry_count = read_u32(encoded, 44)?; + let total_length = canonical_length(entry_count)?; + require_declared_length(encoded, total_length)?; + require_exact(encoded, total_length)?; + let checksum_offset = total_length + .checked_sub(32) + .ok_or(RetentionManifestDecodeError::LengthOverflow)?; + let digest_offset = checksum_offset + .checked_sub(32) + .ok_or(RetentionManifestDecodeError::LengthOverflow)?; + Ok(DecodedManifestHeader { + generation: read_u64(encoded, 32)?, + entry_count, + predecessor: read_array(encoded, 48)?, + entry_set_digest: read_array(encoded, 80)?, + digest_offset, + checksum_offset, + }) +} + +fn validate_fixed_fields(encoded: &[u8]) -> Result<(), RetentionManifestDecodeError> { + let magic = read_array(encoded, 0)?; + if magic != *b"KEEP:RET:LIVE2\0\0" { + return Err(RetentionManifestDecodeError::InvalidMagic { observed: magic }); + } + require_u16(encoded, 16, 2, |expected, observed| { + RetentionManifestDecodeError::UnsupportedVersion { expected, observed } + })?; + require_u16(encoded, 18, 160, |expected, observed| { + RetentionManifestDecodeError::InvalidHeaderLength { expected, observed } + })?; + let flags = read_u32(encoded, 20)?; + if flags != 0 { + return Err(RetentionManifestDecodeError::UnsupportedFlags { observed: flags }); + } + require_u16(encoded, 40, 72, |expected, observed| { + RetentionManifestDecodeError::InvalidEntryWidth { expected, observed } + })?; + require_zero(encoded, 42, 2, "entry")?; + require_zero(encoded, 112, 48, "trailing header") +} + +fn canonical_length(entry_count: u32) -> Result { + let entries = usize::try_from(entry_count) + .map_err(|_| RetentionManifestDecodeError::LengthOverflow)? + .checked_mul(ENTRY_WIDTH) + .ok_or(RetentionManifestDecodeError::LengthOverflow)?; + HEADER_LENGTH + .checked_add(entries) + .and_then(|length| length.checked_add(TRAILER_LENGTH)) + .ok_or(RetentionManifestDecodeError::LengthOverflow) +} + +fn require_declared_length( + encoded: &[u8], + total_length: usize, +) -> Result<(), RetentionManifestDecodeError> { + let observed = read_u64(encoded, 24)?; + let expected = + u64::try_from(total_length).map_err(|_| RetentionManifestDecodeError::LengthOverflow)?; + if observed == expected { + Ok(()) + } else { + Err(RetentionManifestDecodeError::DeclaredLengthMismatch { expected, observed }) + } +} diff --git a/src/adapters/retention/manifest_integrity.rs b/src/adapters/retention/manifest_integrity.rs new file mode 100644 index 0000000..9e4e741 --- /dev/null +++ b/src/adapters/retention/manifest_integrity.rs @@ -0,0 +1,82 @@ +//! This boundary module owns retention manifest integrity verification. + +use super::RetentionManifestDecodeError; + +pub(super) fn verify( + encoded: &[u8], + digest_offset: usize, + checksum_offset: usize, +) -> Result<[u8; 32], RetentionManifestDecodeError> { + let observed_checksum = read_digest(encoded, checksum_offset)?; + let checksum_preimage = + encoded + .get(..checksum_offset) + .ok_or(RetentionManifestDecodeError::Truncated { + expected: checksum_offset, + observed: encoded.len(), + })?; + let expected_checksum = hash(b"keep.retention-manifest-checksum/v2\0", checksum_preimage); + if observed_checksum != expected_checksum { + return Err(RetentionManifestDecodeError::ChecksumMismatch { + expected: expected_checksum, + observed: observed_checksum, + }); + } + + let observed_digest = read_digest(encoded, digest_offset)?; + let digest_preimage = + encoded + .get(..digest_offset) + .ok_or(RetentionManifestDecodeError::Truncated { + expected: digest_offset, + observed: encoded.len(), + })?; + let expected_digest = hash(b"keep.retention-manifest/v2\0", digest_preimage); + if observed_digest != expected_digest { + return Err(RetentionManifestDecodeError::ManifestDigestMismatch { + expected: expected_digest, + observed: observed_digest, + }); + } + Ok(expected_digest) +} + +pub(super) fn verify_entry_set( + entry_count: u32, + entries: &[u8], + observed: [u8; 32], +) -> Result<(), RetentionManifestDecodeError> { + let mut hasher = blake3::Hasher::new(); + hasher.update(b"keep.retention-manifest-entries/v2\0"); + hasher.update(&entry_count.to_be_bytes()); + hasher.update(entries); + let expected = *hasher.finalize().as_bytes(); + if observed == expected { + Ok(()) + } else { + Err(RetentionManifestDecodeError::EntrySetDigestMismatch { expected, observed }) + } +} + +fn read_digest(encoded: &[u8], offset: usize) -> Result<[u8; 32], RetentionManifestDecodeError> { + let end = offset + .checked_add(32) + .ok_or(RetentionManifestDecodeError::LengthOverflow)?; + let bytes = encoded + .get(offset..end) + .ok_or(RetentionManifestDecodeError::Truncated { + expected: end, + observed: encoded.len(), + })?; + <[u8; 32]>::try_from(bytes).map_err(|_| RetentionManifestDecodeError::Truncated { + expected: end, + observed: encoded.len(), + }) +} + +fn hash(domain: &[u8], bytes: &[u8]) -> [u8; 32] { + let mut hasher = blake3::Hasher::new(); + hasher.update(domain); + hasher.update(bytes); + *hasher.finalize().as_bytes() +} diff --git a/src/adapters/retention/manifest_semantic_header.rs b/src/adapters/retention/manifest_semantic_header.rs new file mode 100644 index 0000000..8135a08 --- /dev/null +++ b/src/adapters/retention/manifest_semantic_header.rs @@ -0,0 +1,35 @@ +//! This boundary module owns post-integrity retention manifest header admission. + +use super::RetentionManifestDecodeError; +use super::manifest_header_decoder::DecodedManifestHeader; +use crate::{LivenessGeneration, RetentionManifest, RetentionManifestDigest}; + +pub(super) struct AdmittedManifestHeader { + pub(super) generation: LivenessGeneration, + pub(super) predecessor: Option, +} + +pub(super) fn admit( + header: &DecodedManifestHeader, +) -> Result { + if header.entry_count > RetentionManifest::MAXIMUM_ENTRY_COUNT { + return Err(RetentionManifestDecodeError::EntryCountExceeded { + maximum: RetentionManifest::MAXIMUM_ENTRY_COUNT, + observed: header.entry_count, + }); + } + let generation = LivenessGeneration::new(header.generation) + .map_err(|source| RetentionManifestDecodeError::LivenessGeneration { source })?; + Ok(AdmittedManifestHeader { + generation, + predecessor: predecessor(header.predecessor), + }) +} + +fn predecessor(bytes: [u8; 32]) -> Option { + if bytes == [0_u8; 32] { + None + } else { + Some(RetentionManifestDigest::from_hash(bytes)) + } +} diff --git a/src/lib.rs b/src/lib.rs index e248621..8e07d58 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -22,9 +22,9 @@ //! continuation has a storage-independent planning and execution boundary plus //! a pinned writer-authorized filesystem adapter. Core retention namespaces, //! generations, realization policy, reconstruction anchors, and semantic roots -//! are validated; canonical in-memory root encoding and decoding are available. -//! Retention publication, recovery, and garbage collection remain intentionally -//! absent. +//! are validated; canonical in-memory root and manifest encoding and decoding +//! are available. Retention-head codecs, publication, recovery, and garbage +//! collection remain intentionally absent. #[cfg(test)] extern crate self as keep; @@ -42,16 +42,17 @@ mod retention; #[doc(hidden)] pub use adapters::RepositoryInitializationStorage; pub use adapters::{ - AdmittedCatalog, AdmittedRecoveryStageBytes, AdmittedRetentionRoot, AdmittedSegment, - AdmittedSegmentRecord, BlobIdBinaryParseError, BlobIdTextParseError, CanonicalCatalog, - CanonicalLayoutRecord, CanonicalPublicationHead, CanonicalRetentionRoot, CatalogAdmissionError, - CatalogAllocationPhase, CatalogDecodeError, CatalogEncodeError, CatalogEntryDecodeError, - CatalogPublicationError, CatalogPublicationExpectation, CatalogPublicationOutcome, - CatalogPublicationPhase, CatalogPublicationReadiness, CatalogPublicationReceipt, - CatalogPublicationStorage, CatalogRestartArtifact, CatalogRestartByteLimit, - CatalogRestartByteLimitError, CatalogRestartError, CatalogRestartPhase, CatalogRestartPolicy, - CatalogSnapshot, CatalogSnapshotError, CatalogSuccessor, CatalogTransitionError, - ChecksummedCatalog, ChecksummedPublicationHead, ChecksummedSegmentRecord, ClosedSegment, + AdmittedCatalog, AdmittedRecoveryStageBytes, AdmittedRetentionManifest, AdmittedRetentionRoot, + AdmittedSegment, AdmittedSegmentRecord, BlobIdBinaryParseError, BlobIdTextParseError, + CanonicalCatalog, CanonicalLayoutRecord, CanonicalPublicationHead, CanonicalRetentionManifest, + CanonicalRetentionRoot, CatalogAdmissionError, CatalogAllocationPhase, CatalogDecodeError, + CatalogEncodeError, CatalogEntryDecodeError, CatalogPublicationError, + CatalogPublicationExpectation, CatalogPublicationOutcome, CatalogPublicationPhase, + CatalogPublicationReadiness, CatalogPublicationReceipt, CatalogPublicationStorage, + CatalogRestartArtifact, CatalogRestartByteLimit, CatalogRestartByteLimitError, + CatalogRestartError, CatalogRestartPhase, CatalogRestartPolicy, CatalogSnapshot, + CatalogSnapshotError, CatalogSuccessor, CatalogTransitionError, ChecksummedCatalog, + ChecksummedPublicationHead, ChecksummedSegmentRecord, ClosedSegment, FilesystemCatalogPublicationError, FilesystemCatalogPublisher, FilesystemCatalogSnapshot, FilesystemPlatformAdmission, FilesystemPlatformAdmissionError, FilesystemRecoveryInventoryReader, FilesystemRecoveryNextHeadFinalizationOpenError, @@ -85,8 +86,9 @@ pub use adapters::{ RecoveryStageFingerprintAlgorithm, RecoveryStageFingerprintError, RecoveryStageLength, RecoveryStageMetadata, RecoveryStageMetadataError, RecoveryStageNamespacePhase, RecoveryStageParent, RecoveryStagePoolOutcome, RecoveryStageSynchronizationOutcome, - RetentionRootDecodeError, RetentionRootEncodeError, ReusableRecoverySegment, SealedSegment, - SegmentDigest, SegmentDurabilityPhase, SegmentHeader, SegmentHeaderError, SegmentPublication, + RetentionManifestDecodeError, RetentionManifestEncodeError, RetentionRootDecodeError, + RetentionRootEncodeError, ReusableRecoverySegment, SealedSegment, SegmentDigest, + SegmentDurabilityPhase, SegmentHeader, SegmentHeaderError, SegmentPublication, SegmentPublicationError, SegmentReadError, SegmentReadPolicy, SegmentRecordAdmissionError, SegmentRecordChecksum, SegmentRecordDecodeError, SegmentRecordHeader, SegmentRecordHeaderError, SegmentRecordIdentity, SegmentRecordLength, SegmentRecordLimit, SegmentRecordLimitError, @@ -125,7 +127,8 @@ pub use reference::{ }; pub use retention::{ LivenessGeneration, LivenessGenerationError, RegisteredRetentionProfile, RetentionAnchor, - RetentionClosureLimit, RetentionClosureLimitError, RetentionClosureLimits, RetentionNamespace, + RetentionClosureLimit, RetentionClosureLimitError, RetentionClosureLimits, RetentionManifest, + RetentionManifestDigest, RetentionManifestEntry, RetentionManifestError, RetentionNamespace, RetentionNamespaceDigest, RetentionNamespaceError, RetentionPolicy, RetentionProfileAdmissionError, RetentionRoot, RetentionRootDigest, RetentionRootError, RootGeneration, RootGenerationError, diff --git a/src/retention/manifest.rs b/src/retention/manifest.rs new file mode 100644 index 0000000..5095cdb --- /dev/null +++ b/src/retention/manifest.rs @@ -0,0 +1,108 @@ +//! This module owns one canonical semantic global retention manifest. + +use super::{ + LivenessGeneration, RetentionManifestDigest, RetentionManifestEntry, RetentionManifestError, +}; + +/// Complete namespace-to-root view at one global liveness generation. +/// +/// Entries are stored in strict namespace-digest order. Construction sorts +/// caller input, refuses duplicate namespaces, and allocates no additional +/// buffer beyond the supplied `Vec`. +#[must_use] +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct RetentionManifest { + generation: LivenessGeneration, + predecessor: Option, + entries: Vec, + entry_count: u32, +} + +impl RetentionManifest { + /// Maximum admitted namespace entries. + pub const MAXIMUM_ENTRY_COUNT: u32 = 4_096; + + /// Admits one complete semantic manifest. + /// + /// # Errors + /// + /// Returns a typed generation-history, entry-count, or duplicate-namespace + /// failure before the value is admitted. + pub fn new( + generation: LivenessGeneration, + predecessor: Option, + mut entries: Vec, + ) -> Result { + admit_predecessor(generation, predecessor)?; + let observed = entries.len(); + let entry_count = + u32::try_from(observed).map_err(|_| RetentionManifestError::EntryCountExceeded { + maximum: Self::MAXIMUM_ENTRY_COUNT, + observed, + })?; + if entry_count > Self::MAXIMUM_ENTRY_COUNT { + return Err(RetentionManifestError::EntryCountExceeded { + maximum: Self::MAXIMUM_ENTRY_COUNT, + observed, + }); + } + entries.sort_unstable_by_key(|entry| entry.namespace()); + refuse_duplicate(&entries)?; + Ok(Self { + generation, + predecessor, + entries, + entry_count, + }) + } + + /// Returns the exact global liveness generation. + pub const fn generation(&self) -> LivenessGeneration { + self.generation + } + + /// Returns the preceding manifest digest, if this is a successor. + pub const fn predecessor(&self) -> Option { + self.predecessor + } + + /// Returns entries in strict namespace-digest order. + pub fn entries(&self) -> &[RetentionManifestEntry] { + &self.entries + } + + /// Returns the bounded entry count. + pub const fn entry_count(&self) -> u32 { + self.entry_count + } +} + +fn admit_predecessor( + generation: LivenessGeneration, + predecessor: Option, +) -> Result<(), RetentionManifestError> { + if generation.get() == 1 { + return predecessor.map_or(Ok(()), |observed| { + Err(RetentionManifestError::InitialGenerationHasPredecessor { observed }) + }); + } + if predecessor.is_some() { + Ok(()) + } else { + Err(RetentionManifestError::MissingPredecessor { generation }) + } +} + +fn refuse_duplicate(entries: &[RetentionManifestEntry]) -> Result<(), RetentionManifestError> { + for pair in entries.windows(2) { + let [previous, observed] = pair else { + continue; + }; + if previous.namespace() == observed.namespace() { + return Err(RetentionManifestError::DuplicateNamespace { + namespace: previous.namespace(), + }); + } + } + Ok(()) +} diff --git a/src/retention/manifest_digest.rs b/src/retention/manifest_digest.rs new file mode 100644 index 0000000..bfcc292 --- /dev/null +++ b/src/retention/manifest_digest.rs @@ -0,0 +1,18 @@ +//! This module owns canonical global retention manifest identity. + +/// Canonical BLAKE3-256 identity of one complete retention manifest record. +#[must_use] +#[derive(Clone, Copy, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)] +pub struct RetentionManifestDigest([u8; 32]); + +impl RetentionManifestDigest { + pub(crate) const fn from_hash(bytes: [u8; 32]) -> Self { + Self(bytes) + } + + /// Returns the exact 32 digest bytes. + #[must_use] + pub const fn as_bytes(&self) -> &[u8; 32] { + &self.0 + } +} diff --git a/src/retention/manifest_entry.rs b/src/retention/manifest_entry.rs new file mode 100644 index 0000000..dcb0941 --- /dev/null +++ b/src/retention/manifest_entry.rs @@ -0,0 +1,42 @@ +//! This module owns one semantic retention manifest entry. + +use super::{RetentionNamespaceDigest, RetentionRootDigest, RootGeneration}; + +/// Exact current root coordinate for one retention namespace. +#[must_use] +#[derive(Clone, Copy, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)] +pub struct RetentionManifestEntry { + namespace: RetentionNamespaceDigest, + root_generation: RootGeneration, + root_digest: RetentionRootDigest, +} + +impl RetentionManifestEntry { + /// Combines already-validated namespace and root coordinates. + pub const fn new( + namespace: RetentionNamespaceDigest, + root_generation: RootGeneration, + root_digest: RetentionRootDigest, + ) -> Self { + Self { + namespace, + root_generation, + root_digest, + } + } + + /// Returns the namespace digest selected by this entry. + pub const fn namespace(self) -> RetentionNamespaceDigest { + self.namespace + } + + /// Returns the exact current namespace root generation. + pub const fn root_generation(self) -> RootGeneration { + self.root_generation + } + + /// Returns the exact current namespace root digest. + pub const fn root_digest(self) -> RetentionRootDigest { + self.root_digest + } +} diff --git a/src/retention/manifest_error.rs b/src/retention/manifest_error.rs new file mode 100644 index 0000000..1f4ba84 --- /dev/null +++ b/src/retention/manifest_error.rs @@ -0,0 +1,60 @@ +//! This module owns typed semantic retention manifest failures. + +use std::{error::Error, fmt}; + +use super::{LivenessGeneration, RetentionManifestDigest, RetentionNamespaceDigest}; + +/// Failure to construct one canonical semantic retention manifest. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum RetentionManifestError { + /// Generation one carried an impossible predecessor. + InitialGenerationHasPredecessor { + /// Observed predecessor digest. + observed: RetentionManifestDigest, + }, + /// A successor generation omitted its required predecessor. + MissingPredecessor { + /// Successor generation lacking a predecessor. + generation: LivenessGeneration, + }, + /// The caller supplied too many namespace entries. + EntryCountExceeded { + /// Fixed maximum entry count. + maximum: u32, + /// Observed entry count. + observed: usize, + }, + /// The caller supplied one namespace more than once. + DuplicateNamespace { + /// Exact duplicated namespace digest. + namespace: RetentionNamespaceDigest, + }, +} + +impl fmt::Display for RetentionManifestError { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::InitialGenerationHasPredecessor { observed } => write!( + formatter, + "initial retention manifest has predecessor {:?}", + observed.as_bytes() + ), + Self::MissingPredecessor { generation } => write!( + formatter, + "retention manifest generation {} requires a predecessor", + generation.get() + ), + Self::EntryCountExceeded { maximum, observed } => write!( + formatter, + "retention manifest has {observed} entries; maximum is {maximum}" + ), + Self::DuplicateNamespace { namespace } => write!( + formatter, + "retention manifest repeats namespace {:?}", + namespace.as_bytes() + ), + } + } +} + +impl Error for RetentionManifestError {} diff --git a/src/retention/mod.rs b/src/retention/mod.rs index a03e461..38ddba9 100644 --- a/src/retention/mod.rs +++ b/src/retention/mod.rs @@ -11,6 +11,10 @@ mod closure_limit_error; mod closure_limits; mod liveness_generation; mod liveness_generation_error; +mod manifest; +mod manifest_digest; +mod manifest_entry; +mod manifest_error; mod namespace; mod namespace_digest; mod namespace_error; @@ -29,6 +33,10 @@ pub use closure_limit_error::RetentionClosureLimitError; pub use closure_limits::RetentionClosureLimits; pub use liveness_generation::LivenessGeneration; pub use liveness_generation_error::LivenessGenerationError; +pub use manifest::RetentionManifest; +pub use manifest_digest::RetentionManifestDigest; +pub use manifest_entry::RetentionManifestEntry; +pub use manifest_error::RetentionManifestError; pub use namespace::RetentionNamespace; pub use namespace_digest::RetentionNamespaceDigest; pub use namespace_error::RetentionNamespaceError; diff --git a/src/retention/namespace_digest.rs b/src/retention/namespace_digest.rs index d68dda4..a695432 100644 --- a/src/retention/namespace_digest.rs +++ b/src/retention/namespace_digest.rs @@ -9,7 +9,7 @@ pub struct RetentionNamespaceDigest([u8; 32]); impl RetentionNamespaceDigest { - pub(super) const fn from_hash(bytes: [u8; 32]) -> Self { + pub(crate) const fn from_hash(bytes: [u8; 32]) -> Self { Self(bytes) } diff --git a/tests/retention_manifest_codec.rs b/tests/retention_manifest_codec.rs new file mode 100644 index 0000000..a17fb86 --- /dev/null +++ b/tests/retention_manifest_codec.rs @@ -0,0 +1,92 @@ +//! Public semantic and canonical-codec laws for retention manifests. + +#[path = "retention_manifest_codec/refusal_laws.rs"] +mod refusal_laws; +mod support; + +use std::io; + +use keep::{ + AdmittedRetentionManifest, AdmittedRetentionRoot, CanonicalRetentionManifest, + LivenessGeneration, RetentionManifest, RetentionManifestEntry, RetentionManifestError, +}; + +pub(crate) const ONE_ANCHOR_ROOT: &str = + include_str!("../conformance/segment-store/v2/one-anchor-root.hex"); +pub(crate) const ONE_ROOT_MANIFEST: &str = + include_str!("../conformance/segment-store/v2/one-root-manifest.hex"); +pub(crate) const ENTRY_SET_DIGEST_OFFSET: usize = 80; +pub(crate) const ENTRY_BODY_OFFSET: usize = 160; +pub(crate) const MANIFEST_DIGEST_OFFSET: usize = 232; +pub(crate) const CHECKSUM_OFFSET: usize = 264; + +#[test] +fn one_root_manifest_has_one_semantic_and_canonical_representation() +-> Result<(), Box> { + let root_bytes = fixture_bytes(ONE_ANCHOR_ROOT)?; + let root = AdmittedRetentionRoot::decode(&root_bytes)?; + let entry = RetentionManifestEntry::new( + root.root().namespace().digest(), + root.root().generation(), + root.digest(), + ); + let generation = LivenessGeneration::new(1)?; + let manifest = RetentionManifest::new(generation, None, vec![entry])?; + let canonical = CanonicalRetentionManifest::from_manifest(&manifest)?; + let manifest_bytes = fixture_bytes(ONE_ROOT_MANIFEST)?; + assert_eq!(canonical.encoded(), manifest_bytes); + + let admitted = AdmittedRetentionManifest::decode(&manifest_bytes)?; + assert_eq!(admitted.encoded(), manifest_bytes); + assert_eq!(admitted.manifest(), &manifest); + assert_eq!(admitted.digest(), canonical.digest()); + assert_eq!( + admitted.digest().as_bytes(), + manifest_bytes + .get(MANIFEST_DIGEST_OFFSET..MANIFEST_DIGEST_OFFSET + 32) + .ok_or_else(|| io::Error::other("frozen manifest lacks its digest"))? + ); + Ok(()) +} + +#[test] +fn manifest_history_and_namespace_set_are_admitted_canonically() +-> Result<(), Box> { + let entry = fixture_entry()?; + let manifest_bytes = fixture_bytes(ONE_ROOT_MANIFEST)?; + let predecessor = AdmittedRetentionManifest::decode(&manifest_bytes)?.digest(); + assert!(matches!( + RetentionManifest::new(LivenessGeneration::new(1)?, Some(predecessor), vec![entry]), + Err(RetentionManifestError::InitialGenerationHasPredecessor { .. }) + )); + assert!(matches!( + RetentionManifest::new(LivenessGeneration::new(2)?, None, vec![entry]), + Err(RetentionManifestError::MissingPredecessor { .. }) + )); + assert!(matches!( + RetentionManifest::new( + LivenessGeneration::new(2)?, + Some(predecessor), + vec![entry, entry], + ), + Err(RetentionManifestError::DuplicateNamespace { .. }) + )); + Ok(()) +} + +fn fixture_entry() -> Result> { + let root_bytes = fixture_bytes(ONE_ANCHOR_ROOT)?; + let root = AdmittedRetentionRoot::decode(&root_bytes)?; + Ok(RetentionManifestEntry::new( + root.root().namespace().digest(), + root.root().generation(), + root.digest(), + )) +} + +pub(crate) fn fixture_bytes(fixture: &str) -> Result, io::Error> { + let encoded = fixture + .strip_suffix('\n') + .ok_or_else(|| io::Error::other("retention fixture lacks final newline"))?; + support::decode_hex(encoded) +} diff --git a/tests/retention_manifest_codec/refusal_laws.rs b/tests/retention_manifest_codec/refusal_laws.rs new file mode 100644 index 0000000..8fc11a2 --- /dev/null +++ b/tests/retention_manifest_codec/refusal_laws.rs @@ -0,0 +1,149 @@ +//! Framing, integrity, and semantic refusal laws for retention manifests. + +use std::io; + +use keep::{AdmittedRetentionManifest, RetentionManifestDecodeError}; + +use super::{ + CHECKSUM_OFFSET, ENTRY_BODY_OFFSET, ENTRY_SET_DIGEST_OFFSET, MANIFEST_DIGEST_OFFSET, + ONE_ROOT_MANIFEST, fixture_bytes, +}; + +#[test] +fn manifest_framing_and_integrity_have_exact_first_refusals() +-> Result<(), Box> { + let bytes = fixture_bytes(ONE_ROOT_MANIFEST)?; + let mut truncated = bytes.clone(); + assert!(truncated.pop().is_some()); + assert!(matches!( + AdmittedRetentionManifest::decode(&truncated), + Err(RetentionManifestDecodeError::Truncated { + expected: 296, + observed: 295, + }) + )); + + let mut trailing = bytes.clone(); + trailing.push(0); + assert!(matches!( + AdmittedRetentionManifest::decode(&trailing), + Err(RetentionManifestDecodeError::TrailingData { + expected: 296, + observed: 297, + }) + )); + + let mut checksum_corruption = bytes.clone(); + let last = checksum_corruption + .last_mut() + .ok_or_else(|| io::Error::other("frozen manifest is empty"))?; + *last ^= 1; + assert!(matches!( + AdmittedRetentionManifest::decode(&checksum_corruption), + Err(RetentionManifestDecodeError::ChecksumMismatch { .. }) + )); + + let mut digest_corruption = bytes; + let digest_byte = digest_corruption + .get_mut(MANIFEST_DIGEST_OFFSET) + .ok_or_else(|| io::Error::other("frozen manifest lacks its digest"))?; + *digest_byte ^= 1; + refresh_checksum(&mut digest_corruption)?; + assert!(matches!( + AdmittedRetentionManifest::decode(&digest_corruption), + Err(RetentionManifestDecodeError::ManifestDigestMismatch { .. }) + )); + Ok(()) +} + +#[test] +fn complete_integrity_precedes_manifest_semantics() -> Result<(), Box> { + let mut bytes = fixture_bytes(ONE_ROOT_MANIFEST)?; + bytes + .get_mut(32..40) + .ok_or_else(|| io::Error::other("frozen manifest lacks generation bytes"))? + .fill(0); + assert!(matches!( + AdmittedRetentionManifest::decode(&bytes), + Err(RetentionManifestDecodeError::ChecksumMismatch { .. }) + )); + + refresh_manifest_digest_and_checksum(&mut bytes)?; + assert!(matches!( + AdmittedRetentionManifest::decode(&bytes), + Err(RetentionManifestDecodeError::LivenessGeneration { .. }) + )); + Ok(()) +} + +#[test] +fn entry_set_integrity_precedes_nested_root_generation_admission() +-> Result<(), Box> { + let mut bytes = fixture_bytes(ONE_ROOT_MANIFEST)?; + let first_entry_byte = bytes + .get_mut(ENTRY_BODY_OFFSET) + .ok_or_else(|| io::Error::other("frozen manifest lacks its entry body"))?; + *first_entry_byte ^= 1; + refresh_manifest_digest_and_checksum(&mut bytes)?; + assert!(matches!( + AdmittedRetentionManifest::decode(&bytes), + Err(RetentionManifestDecodeError::EntrySetDigestMismatch { .. }) + )); + + bytes + .get_mut(ENTRY_BODY_OFFSET + 32..ENTRY_BODY_OFFSET + 40) + .ok_or_else(|| io::Error::other("frozen manifest lacks root generation bytes"))? + .fill(0); + refresh_entry_set_digest(&mut bytes)?; + refresh_manifest_digest_and_checksum(&mut bytes)?; + assert!(matches!( + AdmittedRetentionManifest::decode(&bytes), + Err(RetentionManifestDecodeError::RootGeneration { index: 0, .. }) + )); + Ok(()) +} + +fn refresh_entry_set_digest(bytes: &mut [u8]) -> Result<(), io::Error> { + let entries = bytes + .get(ENTRY_BODY_OFFSET..MANIFEST_DIGEST_OFFSET) + .ok_or_else(|| io::Error::other("retention manifest lacks its entry body"))?; + let mut hasher = blake3::Hasher::new(); + hasher.update(b"keep.retention-manifest-entries/v2\0"); + hasher.update(&1_u32.to_be_bytes()); + hasher.update(entries); + let digest = *hasher.finalize().as_bytes(); + bytes + .get_mut(ENTRY_SET_DIGEST_OFFSET..ENTRY_SET_DIGEST_OFFSET + 32) + .ok_or_else(|| io::Error::other("retention manifest lacks its entry-set digest"))? + .copy_from_slice(&digest); + Ok(()) +} + +fn refresh_manifest_digest_and_checksum(bytes: &mut [u8]) -> Result<(), io::Error> { + let preimage = bytes + .get(..MANIFEST_DIGEST_OFFSET) + .ok_or_else(|| io::Error::other("retention manifest lacks its digest preimage"))?; + let mut hasher = blake3::Hasher::new(); + hasher.update(b"keep.retention-manifest/v2\0"); + hasher.update(preimage); + let digest = *hasher.finalize().as_bytes(); + bytes + .get_mut(MANIFEST_DIGEST_OFFSET..CHECKSUM_OFFSET) + .ok_or_else(|| io::Error::other("retention manifest lacks its digest"))? + .copy_from_slice(&digest); + refresh_checksum(bytes) +} + +fn refresh_checksum(bytes: &mut [u8]) -> Result<(), io::Error> { + let (preimage, trailer) = bytes + .split_at_mut_checked(CHECKSUM_OFFSET) + .ok_or_else(|| io::Error::other("retention manifest lacks a checksum"))?; + let checksum_slot = trailer + .get_mut(..blake3::OUT_LEN) + .ok_or_else(|| io::Error::other("retention manifest checksum is truncated"))?; + let mut hasher = blake3::Hasher::new(); + hasher.update(b"keep.retention-manifest-checksum/v2\0"); + hasher.update(preimage); + checksum_slot.copy_from_slice(hasher.finalize().as_bytes()); + Ok(()) +} From 109466777065cade453549017c11a2ebefc7044a Mon Sep 17 00:00:00 2001 From: James Ross Date: Wed, 29 Jul 2026 21:30:02 -0700 Subject: [PATCH 009/111] Add: Admit canonical retention heads --- CHANGELOG.md | 8 +- README.md | 9 +- docs/formats/segment-store-v2/requirements.md | 2 +- docs/formats/segment-store-v2/retention.md | 6 +- src/adapters/mod.rs | 5 +- src/adapters/retention.rs | 9 + src/adapters/retention/canonical_head.rs | 28 +++ src/adapters/retention/checksummed_head.rs | 45 +++++ src/adapters/retention/head_decode_error.rs | 66 +++++++ .../retention/head_decode_error_display.rs | 68 +++++++ src/adapters/retention/head_decoder.rs | 134 +++++++++++++ src/adapters/retention/head_encoder.rs | 32 +++ src/lib.rs | 87 ++++---- src/retention/head.rs | 74 +++++++ src/retention/head_error.rs | 39 ++++ src/retention/manifest_length.rs | 55 ++++++ src/retention/manifest_length_error.rs | 45 +++++ src/retention/mod.rs | 8 + tests/retention_head_codec.rs | 187 ++++++++++++++++++ 19 files changed, 850 insertions(+), 57 deletions(-) create mode 100644 src/adapters/retention/canonical_head.rs create mode 100644 src/adapters/retention/checksummed_head.rs create mode 100644 src/adapters/retention/head_decode_error.rs create mode 100644 src/adapters/retention/head_decode_error_display.rs create mode 100644 src/adapters/retention/head_decoder.rs create mode 100644 src/adapters/retention/head_encoder.rs create mode 100644 src/retention/head.rs create mode 100644 src/retention/head_error.rs create mode 100644 src/retention/manifest_length.rs create mode 100644 src/retention/manifest_length_error.rs create mode 100644 tests/retention_head_codec.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index 233537f..5acdfb9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -315,9 +315,11 @@ after its public API and format compatibility policies are established. admission. Validated global manifest values and their canonical encoder and decoder now reproduce the independent manifest fixture and enforce liveness history, namespace uniqueness, bounds, ordering, and all three integrity - layers. Version-1 immutable bytes remain authoritative; production version-2 - writing remains unavailable until issue #19's executable evidence is - complete. + layers. Typed manifest lengths and semantic global heads now reproduce and + admit the exact 144-byte head fixture with fixed framing, checksum-first + semantic admission, and explicit generation-history laws. Version-1 + immutable bytes remain authoritative; production version-2 writing remains + unavailable until issue #19's executable evidence is complete. - Accepted ADR-0009 defines caller-supplied retention namespaces, `BlobId`/`LayoutId` reconstruction anchors, fail-closed canonical closure, generation-checked retention publication, immutable liveness snapshots, diff --git a/README.md b/README.md index 6198612..465fdf7 100644 --- a/README.md +++ b/README.md @@ -115,11 +115,10 @@ bytes, checks hard-link identity and writer-lock release, runs the production recovery classifiers and immutable-artifact admission, and reconstructs the exact published generation and visible one-zero chunk when `HEAD` exists. This matrix proves application process-death behavior; it does not simulate host -power loss. Version-2 retention values and canonical in-memory root encoding -and decoding are implemented, as are the global manifest values and codec. -Retention-head codecs, publication, recovery, compaction, and garbage -collection remain planned. Presence in the reference CAS does not claim -retention, crash recovery, or durability. +power loss. Version-2 retention values and canonical in-memory root, global +manifest, and retention-head codecs are implemented. Retention publication, +recovery, compaction, and garbage collection remain planned. Presence in the +reference CAS does not claim retention, crash recovery, or durability. Run the complete debug-profile matrix: diff --git a/docs/formats/segment-store-v2/requirements.md b/docs/formats/segment-store-v2/requirements.md index 89cd612..9defde7 100644 --- a/docs/formats/segment-store-v2/requirements.md +++ b/docs/formats/segment-store-v2/requirements.md @@ -10,7 +10,7 @@ case is not evidence. | ID | Requirement | Evidence | Status | | --- | --- | --- | --- | | `KEEP-RETENTION-001` | `RetentionNamespace`, `RootGeneration`, `LivenessGeneration`, profile coordinates, limits, anchors, and digests are validated typed values | `tests/retention_values.rs` and `tests/retention_root_encoding.rs` | Implemented | -| `KEEP-RETENTION-002` | Root, manifest, and head codecs implement the exact canonical grammars and fixed bounds | independent golden corpus plus root and manifest evidence in `tests/retention_root_encoding.rs`, `tests/retention_root_decoding.rs`, and `tests/retention_manifest_codec.rs`; head codec remains | In progress in #19 | +| `KEEP-RETENTION-002` | Root, manifest, and head codecs implement the exact canonical grammars and fixed bounds | independent golden corpus plus `tests/retention_root_encoding.rs`, `tests/retention_root_decoding.rs`, `tests/retention_manifest_codec.rs`, and `tests/retention_head_codec.rs` | Implemented | | `KEEP-RETENTION-003` | Every structural field, truncation boundary, ordering law, duplicate, overflow, flag, reserved byte, digest, checksum, and trailing byte has a precise refusal | corruption matrix and fuzz | Planned in #19 | | `KEEP-RETENTION-004` | Retain and release compare expected and observed generations and publish exact successors only | unit and model-based transition tests | Planned in #19 | | `KEEP-RETENTION-005` | Closure derivation is deterministic, bounded, cycle-safe, fail-closed, and verifies complete blob reconstruction | property, corruption, and adversarial catalog tests | Planned in #19 | diff --git a/docs/formats/segment-store-v2/retention.md b/docs/formats/segment-store-v2/retention.md index 46d6cbc..1823e50 100644 --- a/docs/formats/segment-store-v2/retention.md +++ b/docs/formats/segment-store-v2/retention.md @@ -160,9 +160,9 @@ retention/roots// ``` Names with alternate width, case, suffix, generation, or digest refuse. Keep -implements validated in-memory root and manifest codecs with integrity before -semantic admission. Filesystem publication, the head codec, transitions, -recovery, and garbage collection remain absent. +implements validated in-memory root, manifest, and head codecs with integrity +before semantic admission. Filesystem publication, transitions, recovery, and +garbage collection remain absent. ## Global retention manifest diff --git a/src/adapters/mod.rs b/src/adapters/mod.rs index 7033c63..b49e065 100644 --- a/src/adapters/mod.rs +++ b/src/adapters/mod.rs @@ -450,8 +450,9 @@ pub use recovery_stage_synchronization_outcome::RecoveryStageSynchronizationOutc #[cfg(feature = "repository-tasks")] pub use repository_initialization_storage::RepositoryInitializationStorage; pub use retention::{ - AdmittedRetentionManifest, AdmittedRetentionRoot, CanonicalRetentionManifest, - CanonicalRetentionRoot, RetentionManifestDecodeError, RetentionManifestEncodeError, + AdmittedRetentionManifest, AdmittedRetentionRoot, CanonicalRetentionHead, + CanonicalRetentionManifest, CanonicalRetentionRoot, ChecksummedRetentionHead, + RetentionHeadDecodeError, RetentionManifestDecodeError, RetentionManifestEncodeError, RetentionRootDecodeError, RetentionRootEncodeError, }; pub use sealed_segment::SealedSegment; diff --git a/src/adapters/retention.rs b/src/adapters/retention.rs index 3cebdd7..ec3f2ff 100644 --- a/src/adapters/retention.rs +++ b/src/adapters/retention.rs @@ -2,8 +2,14 @@ mod admitted_manifest; mod admitted_root; +mod canonical_head; mod canonical_manifest; mod canonical_root; +mod checksummed_head; +mod head_decode_error; +mod head_decode_error_display; +mod head_decoder; +mod head_encoder; mod manifest_decode_error; mod manifest_decode_error_display; mod manifest_decoder; @@ -27,8 +33,11 @@ mod root_semantic_header; pub use admitted_manifest::AdmittedRetentionManifest; pub use admitted_root::AdmittedRetentionRoot; +pub use canonical_head::CanonicalRetentionHead; pub use canonical_manifest::CanonicalRetentionManifest; pub use canonical_root::CanonicalRetentionRoot; +pub use checksummed_head::ChecksummedRetentionHead; +pub use head_decode_error::RetentionHeadDecodeError; pub use manifest_decode_error::RetentionManifestDecodeError; pub use manifest_encode_error::RetentionManifestEncodeError; pub use root_decode_error::RetentionRootDecodeError; diff --git a/src/adapters/retention/canonical_head.rs b/src/adapters/retention/canonical_head.rs new file mode 100644 index 0000000..0f70013 --- /dev/null +++ b/src/adapters/retention/canonical_head.rs @@ -0,0 +1,28 @@ +//! This boundary module owns materialized canonical retention-head bytes. + +use super::head_encoder; +use crate::RetentionHead; + +/// Owned canonical version-2 global retention-head record. +#[must_use] +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub struct CanonicalRetentionHead { + encoded: [u8; 144], +} + +impl CanonicalRetentionHead { + /// Encodes one validated semantic retention head. + pub fn from_head(head: &RetentionHead) -> Self { + head_encoder::encode(head) + } + + /// Returns the complete exact canonical bytes. + #[must_use] + pub const fn encoded(&self) -> &[u8; 144] { + &self.encoded + } + + pub(super) const fn admitted(encoded: [u8; 144]) -> Self { + Self { encoded } + } +} diff --git a/src/adapters/retention/checksummed_head.rs b/src/adapters/retention/checksummed_head.rs new file mode 100644 index 0000000..6630847 --- /dev/null +++ b/src/adapters/retention/checksummed_head.rs @@ -0,0 +1,45 @@ +//! This boundary module owns a framing- and checksum-verified retention head. + +use super::{RetentionHeadDecodeError, head_decoder}; +use crate::RetentionHead; + +/// Borrowed canonical retention-head bytes with admitted semantic coordinates. +/// +/// This state does not prove that the named manifest exists or that its entries +/// name admitted namespace roots. A reader must bind those artifacts before +/// treating this value as a complete retention snapshot. +#[must_use] +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub struct ChecksummedRetentionHead<'encoded> { + encoded: &'encoded [u8], + head: RetentionHead, +} + +impl<'encoded> ChecksummedRetentionHead<'encoded> { + /// Decodes exact version-2 framing and verifies the head checksum. + /// + /// This operation performs no allocation or I/O. + /// + /// # Errors + /// + /// Returns [`RetentionHeadDecodeError`] for wrong framing, unsupported or + /// noncanonical fields, checksum disagreement, or invalid coordinates. + pub fn decode(encoded: &'encoded [u8]) -> Result { + head_decoder::decode(encoded) + } + + /// Returns the exact borrowed canonical bytes. + #[must_use] + pub const fn encoded(&self) -> &'encoded [u8] { + self.encoded + } + + /// Returns the admitted semantic head. + pub const fn head(&self) -> &RetentionHead { + &self.head + } + + pub(super) const fn admitted(encoded: &'encoded [u8], head: RetentionHead) -> Self { + Self { encoded, head } + } +} diff --git a/src/adapters/retention/head_decode_error.rs b/src/adapters/retention/head_decode_error.rs new file mode 100644 index 0000000..dee202b --- /dev/null +++ b/src/adapters/retention/head_decode_error.rs @@ -0,0 +1,66 @@ +//! This boundary module owns typed retention-head decoding failures. + +use crate::{LivenessGenerationError, RetentionHeadError, RetentionManifestLengthError}; + +/// Failure to decode and admit one version-2 retention head. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum RetentionHeadDecodeError { + /// The input was not exactly one complete fixed-width head. + WrongLength { + /// Required fixed width. + expected: usize, + /// Observed input width. + observed: usize, + }, + /// The fixed record magic was not canonical. + InvalidMagic { + /// Observed 16 magic bytes. + observed: [u8; 16], + }, + /// The format version is unsupported. + UnsupportedVersion { + /// Supported version. + expected: u16, + /// Observed version. + observed: u16, + }, + /// The fixed record length field was noncanonical. + InvalidRecordLength { + /// Required record length. + expected: u16, + /// Observed record length. + observed: u16, + }, + /// The record carried unsupported flags. + UnsupportedFlags { + /// Observed flag bits. + observed: u32, + }, + /// Reserved bytes were nonzero. + NonZeroReserved { + /// Observed reserved bytes. + observed: [u8; 8], + }, + /// The checksum did not match the exact prefix. + ChecksumMismatch { + /// Computed canonical checksum. + expected: [u8; 32], + /// Checksum stored in the record. + observed: [u8; 32], + }, + /// Liveness-generation admission failed. + LivenessGeneration { + /// Preserved generation failure. + source: LivenessGenerationError, + }, + /// Manifest-length admission failed. + ManifestLength { + /// Preserved manifest-length failure. + source: RetentionManifestLengthError, + }, + /// Final semantic head admission failed. + Semantic { + /// Preserved semantic failure. + source: RetentionHeadError, + }, +} diff --git a/src/adapters/retention/head_decode_error_display.rs b/src/adapters/retention/head_decode_error_display.rs new file mode 100644 index 0000000..601e2c9 --- /dev/null +++ b/src/adapters/retention/head_decode_error_display.rs @@ -0,0 +1,68 @@ +//! This boundary module owns retention-head decode diagnostics and sources. + +use std::{error::Error, fmt}; + +use super::RetentionHeadDecodeError; + +impl fmt::Display for RetentionHeadDecodeError { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::WrongLength { expected, observed } => write!( + formatter, + "retention head has {observed} bytes; expected {expected}" + ), + Self::InvalidMagic { observed } => { + write!(formatter, "invalid retention head magic {observed:02x?}") + } + Self::UnsupportedVersion { expected, observed } => write!( + formatter, + "unsupported retention head version {observed}; expected {expected}" + ), + Self::InvalidRecordLength { expected, observed } => write!( + formatter, + "retention head record length {observed}; expected {expected}" + ), + Self::UnsupportedFlags { observed } => { + write!( + formatter, + "unsupported retention head flags {observed:#010x}" + ) + } + Self::NonZeroReserved { observed } => { + write!( + formatter, + "retention head reserved bytes are nonzero: {observed:02x?}" + ) + } + Self::ChecksumMismatch { .. } => { + formatter.write_str("retention head checksum mismatch") + } + Self::LivenessGeneration { source } => { + write!(formatter, "invalid retention-head generation: {source}") + } + Self::ManifestLength { source } => { + write!(formatter, "invalid retention manifest length: {source}") + } + Self::Semantic { source } => { + write!(formatter, "invalid semantic retention head: {source}") + } + } + } +} + +impl Error for RetentionHeadDecodeError { + fn source(&self) -> Option<&(dyn Error + 'static)> { + match self { + Self::LivenessGeneration { source } => Some(source), + Self::ManifestLength { source } => Some(source), + Self::Semantic { source } => Some(source), + Self::WrongLength { .. } + | Self::InvalidMagic { .. } + | Self::UnsupportedVersion { .. } + | Self::InvalidRecordLength { .. } + | Self::UnsupportedFlags { .. } + | Self::NonZeroReserved { .. } + | Self::ChecksumMismatch { .. } => None, + } + } +} diff --git a/src/adapters/retention/head_decoder.rs b/src/adapters/retention/head_decoder.rs new file mode 100644 index 0000000..76d9462 --- /dev/null +++ b/src/adapters/retention/head_decoder.rs @@ -0,0 +1,134 @@ +//! This boundary module owns canonical retention-head decoding order. + +use super::{ChecksummedRetentionHead, RetentionHeadDecodeError}; +use crate::{LivenessGeneration, RetentionHead, RetentionManifestDigest, RetentionManifestLength}; + +pub(super) const ENCODED_LENGTH: usize = 144; +pub(super) const CHECKSUM_OFFSET: usize = 112; +pub(super) const MAGIC: [u8; 16] = *b"KEEP:RET:HEAD2\0\0"; +pub(super) const VERSION: u16 = 2; +pub(super) const RECORD_LENGTH: u16 = 144; +const CHECKSUM_DOMAIN: &[u8] = b"keep.retention-head-checksum/v2\0"; + +pub(super) fn decode( + encoded: &[u8], +) -> Result, RetentionHeadDecodeError> { + require_length(encoded)?; + validate_fixed_fields(encoded)?; + verify_checksum(encoded)?; + let generation = LivenessGeneration::new(read_u64(encoded, 24)?) + .map_err(|source| RetentionHeadDecodeError::LivenessGeneration { source })?; + let manifest_length = RetentionManifestLength::new(read_u64(encoded, 32)?) + .map_err(|source| RetentionHeadDecodeError::ManifestLength { source })?; + let manifest_digest = RetentionManifestDigest::from_hash(read_array(encoded, 40)?); + let predecessor = predecessor(read_array(encoded, 72)?); + let head = RetentionHead::new(generation, manifest_length, manifest_digest, predecessor) + .map_err(|source| RetentionHeadDecodeError::Semantic { source })?; + Ok(ChecksummedRetentionHead::admitted(encoded, head)) +} + +fn validate_fixed_fields(encoded: &[u8]) -> Result<(), RetentionHeadDecodeError> { + let magic = read_array(encoded, 0)?; + if magic != MAGIC { + return Err(RetentionHeadDecodeError::InvalidMagic { observed: magic }); + } + let version = read_u16(encoded, 16)?; + if version != VERSION { + return Err(RetentionHeadDecodeError::UnsupportedVersion { + expected: VERSION, + observed: version, + }); + } + let record_length = read_u16(encoded, 18)?; + if record_length != RECORD_LENGTH { + return Err(RetentionHeadDecodeError::InvalidRecordLength { + expected: RECORD_LENGTH, + observed: record_length, + }); + } + let flags = read_u32(encoded, 20)?; + if flags != 0 { + return Err(RetentionHeadDecodeError::UnsupportedFlags { observed: flags }); + } + let reserved = read_array(encoded, 104)?; + if reserved != [0_u8; 8] { + return Err(RetentionHeadDecodeError::NonZeroReserved { observed: reserved }); + } + Ok(()) +} + +fn verify_checksum(encoded: &[u8]) -> Result<(), RetentionHeadDecodeError> { + let preimage = encoded + .get(..CHECKSUM_OFFSET) + .ok_or(RetentionHeadDecodeError::WrongLength { + expected: ENCODED_LENGTH, + observed: encoded.len(), + })?; + let observed = read_array(encoded, CHECKSUM_OFFSET)?; + let expected = checksum(preimage); + if observed == expected { + Ok(()) + } else { + Err(RetentionHeadDecodeError::ChecksumMismatch { expected, observed }) + } +} + +pub(super) fn checksum(preimage: &[u8]) -> [u8; 32] { + let mut hasher = blake3::Hasher::new(); + hasher.update(CHECKSUM_DOMAIN); + hasher.update(preimage); + *hasher.finalize().as_bytes() +} + +fn predecessor(bytes: [u8; 32]) -> Option { + if bytes == [0_u8; 32] { + None + } else { + Some(RetentionManifestDigest::from_hash(bytes)) + } +} + +const fn require_length(encoded: &[u8]) -> Result<(), RetentionHeadDecodeError> { + if encoded.len() == ENCODED_LENGTH { + Ok(()) + } else { + Err(RetentionHeadDecodeError::WrongLength { + expected: ENCODED_LENGTH, + observed: encoded.len(), + }) + } +} + +fn read_u16(encoded: &[u8], offset: usize) -> Result { + read_array(encoded, offset).map(u16::from_be_bytes) +} + +fn read_u32(encoded: &[u8], offset: usize) -> Result { + read_array(encoded, offset).map(u32::from_be_bytes) +} + +fn read_u64(encoded: &[u8], offset: usize) -> Result { + read_array(encoded, offset).map(u64::from_be_bytes) +} + +fn read_array( + encoded: &[u8], + offset: usize, +) -> Result<[u8; WIDTH], RetentionHeadDecodeError> { + let Some(end) = offset.checked_add(WIDTH) else { + return Err(RetentionHeadDecodeError::WrongLength { + expected: ENCODED_LENGTH, + observed: encoded.len(), + }); + }; + let bytes = encoded + .get(offset..end) + .ok_or(RetentionHeadDecodeError::WrongLength { + expected: ENCODED_LENGTH, + observed: encoded.len(), + })?; + <[u8; WIDTH]>::try_from(bytes).map_err(|_| RetentionHeadDecodeError::WrongLength { + expected: ENCODED_LENGTH, + observed: encoded.len(), + }) +} diff --git a/src/adapters/retention/head_encoder.rs b/src/adapters/retention/head_encoder.rs new file mode 100644 index 0000000..6565476 --- /dev/null +++ b/src/adapters/retention/head_encoder.rs @@ -0,0 +1,32 @@ +//! This boundary module owns canonical version-2 retention-head encoding. + +use super::{CanonicalRetentionHead, head_decoder as format}; +use crate::RetentionHead; + +pub(super) fn encode(head: &RetentionHead) -> CanonicalRetentionHead { + let mut encoded = [0_u8; format::ENCODED_LENGTH]; + let (preimage, checksum_slot) = encoded.split_at_mut(format::CHECKSUM_OFFSET); + let (magic, remaining) = preimage.split_at_mut(16); + magic.copy_from_slice(&format::MAGIC); + let (version, remaining) = remaining.split_at_mut(2); + version.copy_from_slice(&format::VERSION.to_be_bytes()); + let (record_length, remaining) = remaining.split_at_mut(2); + record_length.copy_from_slice(&format::RECORD_LENGTH.to_be_bytes()); + let (flags, remaining) = remaining.split_at_mut(4); + flags.copy_from_slice(&0_u32.to_be_bytes()); + let (generation, remaining) = remaining.split_at_mut(8); + generation.copy_from_slice(&head.generation().get().to_be_bytes()); + let (manifest_length, remaining) = remaining.split_at_mut(8); + manifest_length.copy_from_slice(&head.manifest_length().get().to_be_bytes()); + let (manifest_digest, remaining) = remaining.split_at_mut(32); + manifest_digest.copy_from_slice(head.manifest_digest().as_bytes()); + let (predecessor, remaining) = remaining.split_at_mut(32); + predecessor.copy_from_slice( + &head + .predecessor() + .map_or([0_u8; 32], |digest| *digest.as_bytes()), + ); + let (_reserved, _complete) = remaining.split_at_mut(8); + checksum_slot.copy_from_slice(&format::checksum(preimage)); + CanonicalRetentionHead::admitted(encoded) +} diff --git a/src/lib.rs b/src/lib.rs index 8e07d58..957b5ca 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -22,8 +22,8 @@ //! continuation has a storage-independent planning and execution boundary plus //! a pinned writer-authorized filesystem adapter. Core retention namespaces, //! generations, realization policy, reconstruction anchors, and semantic roots -//! are validated; canonical in-memory root and manifest encoding and decoding -//! are available. Retention-head codecs, publication, recovery, and garbage +//! are validated; canonical in-memory root, manifest, and head encoding and +//! decoding are available. Retention publication, recovery, and garbage //! collection remain intentionally absent. #[cfg(test)] @@ -44,41 +44,41 @@ pub use adapters::RepositoryInitializationStorage; pub use adapters::{ AdmittedCatalog, AdmittedRecoveryStageBytes, AdmittedRetentionManifest, AdmittedRetentionRoot, AdmittedSegment, AdmittedSegmentRecord, BlobIdBinaryParseError, BlobIdTextParseError, - CanonicalCatalog, CanonicalLayoutRecord, CanonicalPublicationHead, CanonicalRetentionManifest, - CanonicalRetentionRoot, CatalogAdmissionError, CatalogAllocationPhase, CatalogDecodeError, - CatalogEncodeError, CatalogEntryDecodeError, CatalogPublicationError, - CatalogPublicationExpectation, CatalogPublicationOutcome, CatalogPublicationPhase, - CatalogPublicationReadiness, CatalogPublicationReceipt, CatalogPublicationStorage, - CatalogRestartArtifact, CatalogRestartByteLimit, CatalogRestartByteLimitError, - CatalogRestartError, CatalogRestartPhase, CatalogRestartPolicy, CatalogSnapshot, - CatalogSnapshotError, CatalogSuccessor, CatalogTransitionError, ChecksummedCatalog, - ChecksummedPublicationHead, ChecksummedSegmentRecord, ClosedSegment, - FilesystemCatalogPublicationError, FilesystemCatalogPublisher, FilesystemCatalogSnapshot, - FilesystemPlatformAdmission, FilesystemPlatformAdmissionError, - FilesystemRecoveryInventoryReader, FilesystemRecoveryNextHeadFinalizationOpenError, - FilesystemRecoveryNextHeadFinalizer, FilesystemRecoverySegmentResumeOpenError, - FilesystemRecoverySegmentResumer, FilesystemRecoverySegmentStage, - FilesystemRecoveryStageCompleter, FilesystemRecoveryStageCompletionOpenError, - FilesystemRecoveryStageDiscardOpenError, FilesystemRecoveryStageDiscarder, - FilesystemRecoveryStageError, FilesystemSegmentStage, FilesystemWriterLock, LayoutDecodeError, - LayoutDecodePolicy, LayoutEncodeError, LayoutIdBinaryParseError, LayoutIdTextParseError, - OpenedReusableSegment, PublicationHeadDecodeError, RecoveryCatalogStage, - RecoveryCatalogStageError, RecoveryEntryName, RecoveryEntryNameError, RecoveryEntryRole, - RecoveryInventory, RecoveryInventoryEntry, RecoveryInventoryError, RecoveryInventoryLimit, - RecoveryInventoryLimitError, RecoveryInventoryOperation, RecoveryInventoryStorage, - RecoveryNameClassificationError, RecoveryNameManifest, RecoveryNamedEntry, RecoveryNamespace, - RecoveryNextHeadFinalizationError, RecoveryNextHeadFinalizationOutcome, - RecoveryNextHeadFinalizationPlanError, RecoveryNextHeadFinalizationReadiness, - RecoveryNextHeadFinalizationReceipt, RecoveryNextHeadFinalizationRequest, - RecoveryNextHeadFinalizationStorage, RecoveryNextHeadFinalizationStorageError, - RecoveryNextHeadFinalizationTarget, RecoveryNextHeadStage, RecoveryNextHeadStageError, - RecoveryPoolNameError, RecoveryRequiredEntry, RecoverySegmentResumeError, - RecoverySegmentResumePlanError, RecoverySegmentResumeRequest, RecoverySegmentResumeStorage, - RecoverySegmentResumeStorageError, RecoverySegmentStage, RecoverySegmentStageError, - RecoverySegmentTruncation, RecoveryStage, RecoveryStageAssessment, - RecoveryStageAssessmentError, RecoveryStageByteAdmissionError, RecoveryStageCompletionError, - RecoveryStageCompletionPlanError, RecoveryStageCompletionPool, RecoveryStageCompletionReceipt, - RecoveryStageCompletionRequest, RecoveryStageCompletionStorage, + CanonicalCatalog, CanonicalLayoutRecord, CanonicalPublicationHead, CanonicalRetentionHead, + CanonicalRetentionManifest, CanonicalRetentionRoot, CatalogAdmissionError, + CatalogAllocationPhase, CatalogDecodeError, CatalogEncodeError, CatalogEntryDecodeError, + CatalogPublicationError, CatalogPublicationExpectation, CatalogPublicationOutcome, + CatalogPublicationPhase, CatalogPublicationReadiness, CatalogPublicationReceipt, + CatalogPublicationStorage, CatalogRestartArtifact, CatalogRestartByteLimit, + CatalogRestartByteLimitError, CatalogRestartError, CatalogRestartPhase, CatalogRestartPolicy, + CatalogSnapshot, CatalogSnapshotError, CatalogSuccessor, CatalogTransitionError, + ChecksummedCatalog, ChecksummedPublicationHead, ChecksummedRetentionHead, + ChecksummedSegmentRecord, ClosedSegment, FilesystemCatalogPublicationError, + FilesystemCatalogPublisher, FilesystemCatalogSnapshot, FilesystemPlatformAdmission, + FilesystemPlatformAdmissionError, FilesystemRecoveryInventoryReader, + FilesystemRecoveryNextHeadFinalizationOpenError, FilesystemRecoveryNextHeadFinalizer, + FilesystemRecoverySegmentResumeOpenError, FilesystemRecoverySegmentResumer, + FilesystemRecoverySegmentStage, FilesystemRecoveryStageCompleter, + FilesystemRecoveryStageCompletionOpenError, FilesystemRecoveryStageDiscardOpenError, + FilesystemRecoveryStageDiscarder, FilesystemRecoveryStageError, FilesystemSegmentStage, + FilesystemWriterLock, LayoutDecodeError, LayoutDecodePolicy, LayoutEncodeError, + LayoutIdBinaryParseError, LayoutIdTextParseError, OpenedReusableSegment, + PublicationHeadDecodeError, RecoveryCatalogStage, RecoveryCatalogStageError, RecoveryEntryName, + RecoveryEntryNameError, RecoveryEntryRole, RecoveryInventory, RecoveryInventoryEntry, + RecoveryInventoryError, RecoveryInventoryLimit, RecoveryInventoryLimitError, + RecoveryInventoryOperation, RecoveryInventoryStorage, RecoveryNameClassificationError, + RecoveryNameManifest, RecoveryNamedEntry, RecoveryNamespace, RecoveryNextHeadFinalizationError, + RecoveryNextHeadFinalizationOutcome, RecoveryNextHeadFinalizationPlanError, + RecoveryNextHeadFinalizationReadiness, RecoveryNextHeadFinalizationReceipt, + RecoveryNextHeadFinalizationRequest, RecoveryNextHeadFinalizationStorage, + RecoveryNextHeadFinalizationStorageError, RecoveryNextHeadFinalizationTarget, + RecoveryNextHeadStage, RecoveryNextHeadStageError, RecoveryPoolNameError, + RecoveryRequiredEntry, RecoverySegmentResumeError, RecoverySegmentResumePlanError, + RecoverySegmentResumeRequest, RecoverySegmentResumeStorage, RecoverySegmentResumeStorageError, + RecoverySegmentStage, RecoverySegmentStageError, RecoverySegmentTruncation, RecoveryStage, + RecoveryStageAssessment, RecoveryStageAssessmentError, RecoveryStageByteAdmissionError, + RecoveryStageCompletionError, RecoveryStageCompletionPlanError, RecoveryStageCompletionPool, + RecoveryStageCompletionReceipt, RecoveryStageCompletionRequest, RecoveryStageCompletionStorage, RecoveryStageCompletionStorageError, RecoveryStageCompletionTarget, RecoveryStageDiscardError, RecoveryStageDiscardOutcome, RecoveryStageDiscardPlanError, RecoveryStageDiscardReason, RecoveryStageDiscardReceipt, RecoveryStageDiscardRequest, RecoveryStageDiscardStorage, @@ -86,9 +86,9 @@ pub use adapters::{ RecoveryStageFingerprintAlgorithm, RecoveryStageFingerprintError, RecoveryStageLength, RecoveryStageMetadata, RecoveryStageMetadataError, RecoveryStageNamespacePhase, RecoveryStageParent, RecoveryStagePoolOutcome, RecoveryStageSynchronizationOutcome, - RetentionManifestDecodeError, RetentionManifestEncodeError, RetentionRootDecodeError, - RetentionRootEncodeError, ReusableRecoverySegment, SealedSegment, SegmentDigest, - SegmentDurabilityPhase, SegmentHeader, SegmentHeaderError, SegmentPublication, + RetentionHeadDecodeError, RetentionManifestDecodeError, RetentionManifestEncodeError, + RetentionRootDecodeError, RetentionRootEncodeError, ReusableRecoverySegment, SealedSegment, + SegmentDigest, SegmentDurabilityPhase, SegmentHeader, SegmentHeaderError, SegmentPublication, SegmentPublicationError, SegmentReadError, SegmentReadPolicy, SegmentRecordAdmissionError, SegmentRecordChecksum, SegmentRecordDecodeError, SegmentRecordHeader, SegmentRecordHeaderError, SegmentRecordIdentity, SegmentRecordLength, SegmentRecordLimit, SegmentRecordLimitError, @@ -127,9 +127,10 @@ pub use reference::{ }; pub use retention::{ LivenessGeneration, LivenessGenerationError, RegisteredRetentionProfile, RetentionAnchor, - RetentionClosureLimit, RetentionClosureLimitError, RetentionClosureLimits, RetentionManifest, - RetentionManifestDigest, RetentionManifestEntry, RetentionManifestError, RetentionNamespace, - RetentionNamespaceDigest, RetentionNamespaceError, RetentionPolicy, + RetentionClosureLimit, RetentionClosureLimitError, RetentionClosureLimits, RetentionHead, + RetentionHeadError, RetentionManifest, RetentionManifestDigest, RetentionManifestEntry, + RetentionManifestError, RetentionManifestLength, RetentionManifestLengthError, + RetentionNamespace, RetentionNamespaceDigest, RetentionNamespaceError, RetentionPolicy, RetentionProfileAdmissionError, RetentionRoot, RetentionRootDigest, RetentionRootError, RootGeneration, RootGenerationError, }; diff --git a/src/retention/head.rs b/src/retention/head.rs new file mode 100644 index 0000000..907a080 --- /dev/null +++ b/src/retention/head.rs @@ -0,0 +1,74 @@ +//! This module owns one semantic global retention head. + +use super::{ + LivenessGeneration, RetentionHeadError, RetentionManifestDigest, RetentionManifestLength, +}; + +/// Exact coordinate of the globally selected retention manifest. +#[must_use] +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub struct RetentionHead { + generation: LivenessGeneration, + manifest_length: RetentionManifestLength, + manifest_digest: RetentionManifestDigest, + predecessor: Option, +} + +impl RetentionHead { + /// Admits one semantic global retention-head coordinate. + /// + /// # Errors + /// + /// Returns [`RetentionHeadError`] when initial or successor history is + /// malformed. + pub fn new( + generation: LivenessGeneration, + manifest_length: RetentionManifestLength, + manifest_digest: RetentionManifestDigest, + predecessor: Option, + ) -> Result { + admit_predecessor(generation, predecessor)?; + Ok(Self { + generation, + manifest_length, + manifest_digest, + predecessor, + }) + } + + /// Returns the selected global liveness generation. + pub const fn generation(self) -> LivenessGeneration { + self.generation + } + + /// Returns the exact selected manifest length. + pub const fn manifest_length(self) -> RetentionManifestLength { + self.manifest_length + } + + /// Returns the exact selected manifest digest. + pub const fn manifest_digest(self) -> RetentionManifestDigest { + self.manifest_digest + } + + /// Returns the preceding manifest digest, if this is a successor. + pub const fn predecessor(self) -> Option { + self.predecessor + } +} + +fn admit_predecessor( + generation: LivenessGeneration, + predecessor: Option, +) -> Result<(), RetentionHeadError> { + if generation.get() == 1 { + return predecessor.map_or(Ok(()), |observed| { + Err(RetentionHeadError::InitialGenerationHasPredecessor { observed }) + }); + } + if predecessor.is_some() { + Ok(()) + } else { + Err(RetentionHeadError::MissingPredecessor { generation }) + } +} diff --git a/src/retention/head_error.rs b/src/retention/head_error.rs new file mode 100644 index 0000000..9b4dfd0 --- /dev/null +++ b/src/retention/head_error.rs @@ -0,0 +1,39 @@ +//! This module owns semantic retention head failures. + +use std::{error::Error, fmt}; + +use super::{LivenessGeneration, RetentionManifestDigest}; + +/// Failure to construct one semantic retention head. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum RetentionHeadError { + /// Generation one carried an impossible predecessor. + InitialGenerationHasPredecessor { + /// Observed predecessor digest. + observed: RetentionManifestDigest, + }, + /// A successor generation omitted its required predecessor. + MissingPredecessor { + /// Successor generation lacking a predecessor. + generation: LivenessGeneration, + }, +} + +impl fmt::Display for RetentionHeadError { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::InitialGenerationHasPredecessor { observed } => write!( + formatter, + "initial retention head has predecessor {:?}", + observed.as_bytes() + ), + Self::MissingPredecessor { generation } => write!( + formatter, + "retention head generation {} requires a predecessor", + generation.get() + ), + } + } +} + +impl Error for RetentionHeadError {} diff --git a/src/retention/manifest_length.rs b/src/retention/manifest_length.rs new file mode 100644 index 0000000..311d2cc --- /dev/null +++ b/src/retention/manifest_length.rs @@ -0,0 +1,55 @@ +//! This module owns canonical retention manifest byte lengths. + +use super::RetentionManifestLengthError; + +const HEADER_LENGTH: u64 = 160; +const ENTRY_LENGTH: u64 = 72; +const TRAILER_LENGTH: u64 = 64; +const MINIMUM_VALUE: u64 = HEADER_LENGTH + TRAILER_LENGTH; +const MAXIMUM_VALUE: u64 = 295_136; + +/// Exact canonical byte length of one complete version-2 retention manifest. +#[must_use] +#[derive(Clone, Copy, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)] +pub struct RetentionManifestLength(u64); + +impl RetentionManifestLength { + /// Smallest complete version-2 retention manifest length. + pub const MINIMUM: Self = Self(MINIMUM_VALUE); + + /// Largest complete version-2 retention manifest length. + pub const MAXIMUM: Self = Self(MAXIMUM_VALUE); + + /// Admits one complete canonical retention manifest length. + /// + /// # Errors + /// + /// Returns [`RetentionManifestLengthError`] when `value` exceeds the + /// format bound or cannot contain a whole number of fixed-width entries. + pub const fn new(value: u64) -> Result { + if value < MINIMUM_VALUE || value > MAXIMUM_VALUE { + return Err(RetentionManifestLengthError::OutOfBounds { + minimum: MINIMUM_VALUE, + maximum: MAXIMUM_VALUE, + observed: value, + }); + } + let Some(entry_bytes) = value.checked_sub(MINIMUM_VALUE) else { + return Err(RetentionManifestLengthError::OutOfBounds { + minimum: MINIMUM_VALUE, + maximum: MAXIMUM_VALUE, + observed: value, + }); + }; + if !entry_bytes.is_multiple_of(ENTRY_LENGTH) { + return Err(RetentionManifestLengthError::NotCongruent { observed: value }); + } + Ok(Self(value)) + } + + /// Returns the exact admitted byte length. + #[must_use] + pub const fn get(self) -> u64 { + self.0 + } +} diff --git a/src/retention/manifest_length_error.rs b/src/retention/manifest_length_error.rs new file mode 100644 index 0000000..d3a4ca2 --- /dev/null +++ b/src/retention/manifest_length_error.rs @@ -0,0 +1,45 @@ +//! This module owns retention manifest length admission failures. + +use std::{error::Error, fmt}; + +/// Failure to admit a canonical retention manifest byte length. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum RetentionManifestLengthError { + /// The length is outside the version-2 manifest bounds. + OutOfBounds { + /// Smallest complete manifest length. + minimum: u64, + /// Largest permitted manifest length. + maximum: u64, + /// Length supplied by the boundary. + observed: u64, + }, + /// The length cannot contain a whole number of fixed-width entries. + NotCongruent { + /// Length supplied by the boundary. + observed: u64, + }, +} + +impl fmt::Display for RetentionManifestLengthError { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::OutOfBounds { + minimum, + maximum, + observed, + } => write!( + formatter, + "retention manifest length {observed} is outside {minimum}..={maximum}" + ), + Self::NotCongruent { observed } => { + write!( + formatter, + "retention manifest length {observed} is not congruent" + ) + } + } + } +} + +impl Error for RetentionManifestLengthError {} diff --git a/src/retention/mod.rs b/src/retention/mod.rs index 38ddba9..1b6edba 100644 --- a/src/retention/mod.rs +++ b/src/retention/mod.rs @@ -9,12 +9,16 @@ mod anchor; mod closure_limit; mod closure_limit_error; mod closure_limits; +mod head; +mod head_error; mod liveness_generation; mod liveness_generation_error; mod manifest; mod manifest_digest; mod manifest_entry; mod manifest_error; +mod manifest_length; +mod manifest_length_error; mod namespace; mod namespace_digest; mod namespace_error; @@ -31,12 +35,16 @@ pub use anchor::RetentionAnchor; pub use closure_limit::RetentionClosureLimit; pub use closure_limit_error::RetentionClosureLimitError; pub use closure_limits::RetentionClosureLimits; +pub use head::RetentionHead; +pub use head_error::RetentionHeadError; pub use liveness_generation::LivenessGeneration; pub use liveness_generation_error::LivenessGenerationError; pub use manifest::RetentionManifest; pub use manifest_digest::RetentionManifestDigest; pub use manifest_entry::RetentionManifestEntry; pub use manifest_error::RetentionManifestError; +pub use manifest_length::RetentionManifestLength; +pub use manifest_length_error::RetentionManifestLengthError; pub use namespace::RetentionNamespace; pub use namespace_digest::RetentionNamespaceDigest; pub use namespace_error::RetentionNamespaceError; diff --git a/tests/retention_head_codec.rs b/tests/retention_head_codec.rs new file mode 100644 index 0000000..b3bd0ce --- /dev/null +++ b/tests/retention_head_codec.rs @@ -0,0 +1,187 @@ +//! Public semantic and canonical-codec laws for the retention head. + +mod support; + +use std::io; + +use keep::{ + CanonicalRetentionHead, ChecksummedRetentionHead, LivenessGeneration, RetentionHead, + RetentionHeadDecodeError, RetentionHeadError, RetentionManifestLength, + RetentionManifestLengthError, +}; + +const ONE_ROOT_MANIFEST: &str = + include_str!("../conformance/segment-store/v2/one-root-manifest.hex"); +const ONE_ROOT_HEAD: &str = include_str!("../conformance/segment-store/v2/one-root-head.hex"); +const CHECKSUM_OFFSET: usize = 112; + +#[test] +fn one_root_head_has_one_semantic_and_canonical_representation() +-> Result<(), Box> { + let manifest_bytes = fixture_bytes(ONE_ROOT_MANIFEST)?; + let manifest = keep::AdmittedRetentionManifest::decode(&manifest_bytes)?; + let manifest_length = RetentionManifestLength::new(u64::try_from(manifest_bytes.len())?)?; + let head = RetentionHead::new( + manifest.manifest().generation(), + manifest_length, + manifest.digest(), + manifest.manifest().predecessor(), + )?; + + let canonical = CanonicalRetentionHead::from_head(&head); + let head_bytes = fixture_bytes(ONE_ROOT_HEAD)?; + assert_eq!(canonical.encoded(), head_bytes.as_slice()); + + let checksummed = ChecksummedRetentionHead::decode(&head_bytes)?; + assert_eq!(checksummed.encoded(), head_bytes); + assert_eq!(checksummed.head(), &head); + Ok(()) +} + +#[test] +fn manifest_length_and_head_history_are_admitted_exactly() -> Result<(), Box> +{ + assert_eq!(RetentionManifestLength::new(224)?.get(), 224); + assert_eq!(RetentionManifestLength::new(295_136)?.get(), 295_136); + assert!(matches!( + RetentionManifestLength::new(223), + Err(RetentionManifestLengthError::OutOfBounds { .. }) + )); + assert!(matches!( + RetentionManifestLength::new(225), + Err(RetentionManifestLengthError::NotCongruent { .. }) + )); + + let head_bytes = fixture_bytes(ONE_ROOT_HEAD)?; + let head = ChecksummedRetentionHead::decode(&head_bytes)?; + assert!(matches!( + RetentionHead::new( + LivenessGeneration::new(1)?, + head.head().manifest_length(), + head.head().manifest_digest(), + Some(head.head().manifest_digest()), + ), + Err(RetentionHeadError::InitialGenerationHasPredecessor { .. }) + )); + assert!(matches!( + RetentionHead::new( + LivenessGeneration::new(2)?, + head.head().manifest_length(), + head.head().manifest_digest(), + None, + ), + Err(RetentionHeadError::MissingPredecessor { .. }) + )); + Ok(()) +} + +#[test] +fn head_framing_and_integrity_have_exact_first_refusals() -> Result<(), Box> +{ + let bytes = fixture_bytes(ONE_ROOT_HEAD)?; + let mut truncated = bytes.clone(); + assert!(truncated.pop().is_some()); + assert!(matches!( + ChecksummedRetentionHead::decode(&truncated), + Err(RetentionHeadDecodeError::WrongLength { + expected: 144, + observed: 143, + }) + )); + + let mut trailing = bytes.clone(); + trailing.push(0); + assert!(matches!( + ChecksummedRetentionHead::decode(&trailing), + Err(RetentionHeadDecodeError::WrongLength { + expected: 144, + observed: 145, + }) + )); + + let mut wrong_magic = bytes.clone(); + let first = wrong_magic + .first_mut() + .ok_or_else(|| io::Error::other("frozen retention head is empty"))?; + *first ^= 1; + assert!(matches!( + ChecksummedRetentionHead::decode(&wrong_magic), + Err(RetentionHeadDecodeError::InvalidMagic { .. }) + )); + + let mut checksum_corruption = bytes; + let last = checksum_corruption + .last_mut() + .ok_or_else(|| io::Error::other("frozen retention head is empty"))?; + *last ^= 1; + assert!(matches!( + ChecksummedRetentionHead::decode(&checksum_corruption), + Err(RetentionHeadDecodeError::ChecksumMismatch { .. }) + )); + Ok(()) +} + +#[test] +fn complete_integrity_precedes_head_semantics() -> Result<(), Box> { + let mut bytes = fixture_bytes(ONE_ROOT_HEAD)?; + bytes + .get_mut(24..32) + .ok_or_else(|| io::Error::other("frozen retention head lacks generation bytes"))? + .fill(0); + assert!(matches!( + ChecksummedRetentionHead::decode(&bytes), + Err(RetentionHeadDecodeError::ChecksumMismatch { .. }) + )); + + refresh_checksum(&mut bytes)?; + assert!(matches!( + ChecksummedRetentionHead::decode(&bytes), + Err(RetentionHeadDecodeError::LivenessGeneration { .. }) + )); + + let mut noncanonical_length = fixture_bytes(ONE_ROOT_HEAD)?; + noncanonical_length + .get_mut(32..40) + .ok_or_else(|| io::Error::other("frozen retention head lacks manifest length bytes"))? + .copy_from_slice(&225_u64.to_be_bytes()); + refresh_checksum(&mut noncanonical_length)?; + assert!(matches!( + ChecksummedRetentionHead::decode(&noncanonical_length), + Err(RetentionHeadDecodeError::ManifestLength { .. }) + )); + + let mut missing_predecessor = fixture_bytes(ONE_ROOT_HEAD)?; + missing_predecessor + .get_mut(24..32) + .ok_or_else(|| io::Error::other("frozen retention head lacks generation bytes"))? + .copy_from_slice(&2_u64.to_be_bytes()); + refresh_checksum(&mut missing_predecessor)?; + assert!(matches!( + ChecksummedRetentionHead::decode(&missing_predecessor), + Err(RetentionHeadDecodeError::Semantic { + source: RetentionHeadError::MissingPredecessor { .. }, + }) + )); + Ok(()) +} + +fn fixture_bytes(fixture: &str) -> Result, io::Error> { + let encoded = fixture + .strip_suffix('\n') + .ok_or_else(|| io::Error::other("retention fixture lacks final newline"))?; + support::decode_hex(encoded) +} + +fn refresh_checksum(bytes: &mut [u8]) -> Result<(), io::Error> { + let (preimage, trailer) = bytes + .split_at_mut_checked(CHECKSUM_OFFSET) + .ok_or_else(|| io::Error::other("retention head lacks its checksum"))?; + let checksum = trailer + .get_mut(..blake3::OUT_LEN) + .ok_or_else(|| io::Error::other("retention head checksum is truncated"))?; + let mut hasher = blake3::Hasher::new(); + hasher.update(b"keep.retention-head-checksum/v2\0"); + hasher.update(preimage); + checksum.copy_from_slice(hasher.finalize().as_bytes()); + Ok(()) +} From 500b45250ebca9f5a7c8a7eb108d4c87cdb71c51 Mon Sep 17 00:00:00 2001 From: James Ross Date: Wed, 29 Jul 2026 21:34:19 -0700 Subject: [PATCH 010/111] Refactor: Isolate retention adapter exports --- src/adapters/mod.rs | 8 +--- src/lib.rs | 109 +++++++++++++++++++++++--------------------- 2 files changed, 57 insertions(+), 60 deletions(-) diff --git a/src/adapters/mod.rs b/src/adapters/mod.rs index b49e065..2e72a15 100644 --- a/src/adapters/mod.rs +++ b/src/adapters/mod.rs @@ -234,7 +234,7 @@ mod recovery_stage_pool_outcome; mod recovery_stage_synchronization_outcome; #[cfg(feature = "repository-tasks")] mod repository_initialization_storage; -mod retention; +pub(crate) mod retention; mod sealed_segment; mod segment_digest; mod segment_digest_builder; @@ -449,12 +449,6 @@ pub use recovery_stage_pool_outcome::RecoveryStagePoolOutcome; pub use recovery_stage_synchronization_outcome::RecoveryStageSynchronizationOutcome; #[cfg(feature = "repository-tasks")] pub use repository_initialization_storage::RepositoryInitializationStorage; -pub use retention::{ - AdmittedRetentionManifest, AdmittedRetentionRoot, CanonicalRetentionHead, - CanonicalRetentionManifest, CanonicalRetentionRoot, ChecksummedRetentionHead, - RetentionHeadDecodeError, RetentionManifestDecodeError, RetentionManifestEncodeError, - RetentionRootDecodeError, RetentionRootEncodeError, -}; pub use sealed_segment::SealedSegment; pub use segment_digest::SegmentDigest; pub use segment_header::SegmentHeader; diff --git a/src/lib.rs b/src/lib.rs index 957b5ca..e63b8f6 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -41,44 +41,49 @@ mod retention; #[cfg(feature = "repository-tasks")] #[doc(hidden)] pub use adapters::RepositoryInitializationStorage; +pub use adapters::retention::{ + AdmittedRetentionManifest, AdmittedRetentionRoot, CanonicalRetentionHead, + CanonicalRetentionManifest, CanonicalRetentionRoot, ChecksummedRetentionHead, + RetentionHeadDecodeError, RetentionManifestDecodeError, RetentionManifestEncodeError, + RetentionRootDecodeError, RetentionRootEncodeError, +}; pub use adapters::{ - AdmittedCatalog, AdmittedRecoveryStageBytes, AdmittedRetentionManifest, AdmittedRetentionRoot, - AdmittedSegment, AdmittedSegmentRecord, BlobIdBinaryParseError, BlobIdTextParseError, - CanonicalCatalog, CanonicalLayoutRecord, CanonicalPublicationHead, CanonicalRetentionHead, - CanonicalRetentionManifest, CanonicalRetentionRoot, CatalogAdmissionError, - CatalogAllocationPhase, CatalogDecodeError, CatalogEncodeError, CatalogEntryDecodeError, - CatalogPublicationError, CatalogPublicationExpectation, CatalogPublicationOutcome, - CatalogPublicationPhase, CatalogPublicationReadiness, CatalogPublicationReceipt, - CatalogPublicationStorage, CatalogRestartArtifact, CatalogRestartByteLimit, - CatalogRestartByteLimitError, CatalogRestartError, CatalogRestartPhase, CatalogRestartPolicy, - CatalogSnapshot, CatalogSnapshotError, CatalogSuccessor, CatalogTransitionError, - ChecksummedCatalog, ChecksummedPublicationHead, ChecksummedRetentionHead, - ChecksummedSegmentRecord, ClosedSegment, FilesystemCatalogPublicationError, - FilesystemCatalogPublisher, FilesystemCatalogSnapshot, FilesystemPlatformAdmission, - FilesystemPlatformAdmissionError, FilesystemRecoveryInventoryReader, - FilesystemRecoveryNextHeadFinalizationOpenError, FilesystemRecoveryNextHeadFinalizer, - FilesystemRecoverySegmentResumeOpenError, FilesystemRecoverySegmentResumer, - FilesystemRecoverySegmentStage, FilesystemRecoveryStageCompleter, - FilesystemRecoveryStageCompletionOpenError, FilesystemRecoveryStageDiscardOpenError, - FilesystemRecoveryStageDiscarder, FilesystemRecoveryStageError, FilesystemSegmentStage, - FilesystemWriterLock, LayoutDecodeError, LayoutDecodePolicy, LayoutEncodeError, - LayoutIdBinaryParseError, LayoutIdTextParseError, OpenedReusableSegment, - PublicationHeadDecodeError, RecoveryCatalogStage, RecoveryCatalogStageError, RecoveryEntryName, - RecoveryEntryNameError, RecoveryEntryRole, RecoveryInventory, RecoveryInventoryEntry, - RecoveryInventoryError, RecoveryInventoryLimit, RecoveryInventoryLimitError, - RecoveryInventoryOperation, RecoveryInventoryStorage, RecoveryNameClassificationError, - RecoveryNameManifest, RecoveryNamedEntry, RecoveryNamespace, RecoveryNextHeadFinalizationError, - RecoveryNextHeadFinalizationOutcome, RecoveryNextHeadFinalizationPlanError, - RecoveryNextHeadFinalizationReadiness, RecoveryNextHeadFinalizationReceipt, - RecoveryNextHeadFinalizationRequest, RecoveryNextHeadFinalizationStorage, - RecoveryNextHeadFinalizationStorageError, RecoveryNextHeadFinalizationTarget, - RecoveryNextHeadStage, RecoveryNextHeadStageError, RecoveryPoolNameError, - RecoveryRequiredEntry, RecoverySegmentResumeError, RecoverySegmentResumePlanError, - RecoverySegmentResumeRequest, RecoverySegmentResumeStorage, RecoverySegmentResumeStorageError, - RecoverySegmentStage, RecoverySegmentStageError, RecoverySegmentTruncation, RecoveryStage, - RecoveryStageAssessment, RecoveryStageAssessmentError, RecoveryStageByteAdmissionError, - RecoveryStageCompletionError, RecoveryStageCompletionPlanError, RecoveryStageCompletionPool, - RecoveryStageCompletionReceipt, RecoveryStageCompletionRequest, RecoveryStageCompletionStorage, + AdmittedCatalog, AdmittedRecoveryStageBytes, AdmittedSegment, AdmittedSegmentRecord, + BlobIdBinaryParseError, BlobIdTextParseError, CanonicalCatalog, CanonicalLayoutRecord, + CanonicalPublicationHead, CatalogAdmissionError, CatalogAllocationPhase, CatalogDecodeError, + CatalogEncodeError, CatalogEntryDecodeError, CatalogPublicationError, + CatalogPublicationExpectation, CatalogPublicationOutcome, CatalogPublicationPhase, + CatalogPublicationReadiness, CatalogPublicationReceipt, CatalogPublicationStorage, + CatalogRestartArtifact, CatalogRestartByteLimit, CatalogRestartByteLimitError, + CatalogRestartError, CatalogRestartPhase, CatalogRestartPolicy, CatalogSnapshot, + CatalogSnapshotError, CatalogSuccessor, CatalogTransitionError, ChecksummedCatalog, + ChecksummedPublicationHead, ChecksummedSegmentRecord, ClosedSegment, + FilesystemCatalogPublicationError, FilesystemCatalogPublisher, FilesystemCatalogSnapshot, + FilesystemPlatformAdmission, FilesystemPlatformAdmissionError, + FilesystemRecoveryInventoryReader, FilesystemRecoveryNextHeadFinalizationOpenError, + FilesystemRecoveryNextHeadFinalizer, FilesystemRecoverySegmentResumeOpenError, + FilesystemRecoverySegmentResumer, FilesystemRecoverySegmentStage, + FilesystemRecoveryStageCompleter, FilesystemRecoveryStageCompletionOpenError, + FilesystemRecoveryStageDiscardOpenError, FilesystemRecoveryStageDiscarder, + FilesystemRecoveryStageError, FilesystemSegmentStage, FilesystemWriterLock, LayoutDecodeError, + LayoutDecodePolicy, LayoutEncodeError, LayoutIdBinaryParseError, LayoutIdTextParseError, + OpenedReusableSegment, PublicationHeadDecodeError, RecoveryCatalogStage, + RecoveryCatalogStageError, RecoveryEntryName, RecoveryEntryNameError, RecoveryEntryRole, + RecoveryInventory, RecoveryInventoryEntry, RecoveryInventoryError, RecoveryInventoryLimit, + RecoveryInventoryLimitError, RecoveryInventoryOperation, RecoveryInventoryStorage, + RecoveryNameClassificationError, RecoveryNameManifest, RecoveryNamedEntry, RecoveryNamespace, + RecoveryNextHeadFinalizationError, RecoveryNextHeadFinalizationOutcome, + RecoveryNextHeadFinalizationPlanError, RecoveryNextHeadFinalizationReadiness, + RecoveryNextHeadFinalizationReceipt, RecoveryNextHeadFinalizationRequest, + RecoveryNextHeadFinalizationStorage, RecoveryNextHeadFinalizationStorageError, + RecoveryNextHeadFinalizationTarget, RecoveryNextHeadStage, RecoveryNextHeadStageError, + RecoveryPoolNameError, RecoveryRequiredEntry, RecoverySegmentResumeError, + RecoverySegmentResumePlanError, RecoverySegmentResumeRequest, RecoverySegmentResumeStorage, + RecoverySegmentResumeStorageError, RecoverySegmentStage, RecoverySegmentStageError, + RecoverySegmentTruncation, RecoveryStage, RecoveryStageAssessment, + RecoveryStageAssessmentError, RecoveryStageByteAdmissionError, RecoveryStageCompletionError, + RecoveryStageCompletionPlanError, RecoveryStageCompletionPool, RecoveryStageCompletionReceipt, + RecoveryStageCompletionRequest, RecoveryStageCompletionStorage, RecoveryStageCompletionStorageError, RecoveryStageCompletionTarget, RecoveryStageDiscardError, RecoveryStageDiscardOutcome, RecoveryStageDiscardPlanError, RecoveryStageDiscardReason, RecoveryStageDiscardReceipt, RecoveryStageDiscardRequest, RecoveryStageDiscardStorage, @@ -86,22 +91,20 @@ pub use adapters::{ RecoveryStageFingerprintAlgorithm, RecoveryStageFingerprintError, RecoveryStageLength, RecoveryStageMetadata, RecoveryStageMetadataError, RecoveryStageNamespacePhase, RecoveryStageParent, RecoveryStagePoolOutcome, RecoveryStageSynchronizationOutcome, - RetentionHeadDecodeError, RetentionManifestDecodeError, RetentionManifestEncodeError, - RetentionRootDecodeError, RetentionRootEncodeError, ReusableRecoverySegment, SealedSegment, - SegmentDigest, SegmentDurabilityPhase, SegmentHeader, SegmentHeaderError, SegmentPublication, - SegmentPublicationError, SegmentReadError, SegmentReadPolicy, SegmentRecordAdmissionError, - SegmentRecordChecksum, SegmentRecordDecodeError, SegmentRecordHeader, SegmentRecordHeaderError, - SegmentRecordIdentity, SegmentRecordLength, SegmentRecordLimit, SegmentRecordLimitError, - SegmentRecordPayloadLength, SegmentRecords, SegmentSeal, SegmentSealError, SegmentStage, - SegmentStageCreateError, SegmentWriteError, SegmentWritePhase, StagedSegment, - StorageProfileIdParseError, StoreInitializationError, StoreInitializationPhase, - StoreInitializationReceipt, StoreInitializationStorage, WriterLockAcquireError, - WriterLockAcquirePhase, admit_recovery_stage_bytes, assess_recovery_stage, - classify_recovery_catalog_stage, classify_recovery_names, classify_recovery_next_head_stage, - classify_recovery_segment_stage, execute_recovery_next_head_finalization, - execute_recovery_segment_resume, execute_recovery_stage_completion, - execute_recovery_stage_discard, fingerprint_recovery_stage, initialize_store, - plan_recovery_next_head_finalization, plan_recovery_segment_resume, + ReusableRecoverySegment, SealedSegment, SegmentDigest, SegmentDurabilityPhase, SegmentHeader, + SegmentHeaderError, SegmentPublication, SegmentPublicationError, SegmentReadError, + SegmentReadPolicy, SegmentRecordAdmissionError, SegmentRecordChecksum, + SegmentRecordDecodeError, SegmentRecordHeader, SegmentRecordHeaderError, SegmentRecordIdentity, + SegmentRecordLength, SegmentRecordLimit, SegmentRecordLimitError, SegmentRecordPayloadLength, + SegmentRecords, SegmentSeal, SegmentSealError, SegmentStage, SegmentStageCreateError, + SegmentWriteError, SegmentWritePhase, StagedSegment, StorageProfileIdParseError, + StoreInitializationError, StoreInitializationPhase, StoreInitializationReceipt, + StoreInitializationStorage, WriterLockAcquireError, WriterLockAcquirePhase, + admit_recovery_stage_bytes, assess_recovery_stage, classify_recovery_catalog_stage, + classify_recovery_names, classify_recovery_next_head_stage, classify_recovery_segment_stage, + execute_recovery_next_head_finalization, execute_recovery_segment_resume, + execute_recovery_stage_completion, execute_recovery_stage_discard, fingerprint_recovery_stage, + initialize_store, plan_recovery_next_head_finalization, plan_recovery_segment_resume, plan_recovery_stage_completion, plan_recovery_stage_discard, publish_catalog_generation, read_recovery_inventory, }; From c98682e159845f5e194122e8b3f0afda9bf0cee2 Mon Sep 17 00:00:00 2001 From: James Ross Date: Wed, 29 Jul 2026 21:40:53 -0700 Subject: [PATCH 011/111] Fix: Route retention exports through adapters --- src/adapters/mod.rs | 3 ++- src/lib.rs | 12 ++++++------ 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/src/adapters/mod.rs b/src/adapters/mod.rs index 2e72a15..d55d271 100644 --- a/src/adapters/mod.rs +++ b/src/adapters/mod.rs @@ -234,7 +234,7 @@ mod recovery_stage_pool_outcome; mod recovery_stage_synchronization_outcome; #[cfg(feature = "repository-tasks")] mod repository_initialization_storage; -pub(crate) mod retention; +mod retention; mod sealed_segment; mod segment_digest; mod segment_digest_builder; @@ -449,6 +449,7 @@ pub use recovery_stage_pool_outcome::RecoveryStagePoolOutcome; pub use recovery_stage_synchronization_outcome::RecoveryStageSynchronizationOutcome; #[cfg(feature = "repository-tasks")] pub use repository_initialization_storage::RepositoryInitializationStorage; +pub use retention::*; pub use sealed_segment::SealedSegment; pub use segment_digest::SegmentDigest; pub use segment_header::SegmentHeader; diff --git a/src/lib.rs b/src/lib.rs index e63b8f6..5dde8b2 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -41,12 +41,6 @@ mod retention; #[cfg(feature = "repository-tasks")] #[doc(hidden)] pub use adapters::RepositoryInitializationStorage; -pub use adapters::retention::{ - AdmittedRetentionManifest, AdmittedRetentionRoot, CanonicalRetentionHead, - CanonicalRetentionManifest, CanonicalRetentionRoot, ChecksummedRetentionHead, - RetentionHeadDecodeError, RetentionManifestDecodeError, RetentionManifestEncodeError, - RetentionRootDecodeError, RetentionRootEncodeError, -}; pub use adapters::{ AdmittedCatalog, AdmittedRecoveryStageBytes, AdmittedSegment, AdmittedSegmentRecord, BlobIdBinaryParseError, BlobIdTextParseError, CanonicalCatalog, CanonicalLayoutRecord, @@ -108,6 +102,12 @@ pub use adapters::{ plan_recovery_stage_completion, plan_recovery_stage_discard, publish_catalog_generation, read_recovery_inventory, }; +pub use adapters::{ + AdmittedRetentionManifest, AdmittedRetentionRoot, CanonicalRetentionHead, + CanonicalRetentionManifest, CanonicalRetentionRoot, ChecksummedRetentionHead, + RetentionHeadDecodeError, RetentionManifestDecodeError, RetentionManifestEncodeError, + RetentionRootDecodeError, RetentionRootEncodeError, +}; pub use blob::{ BlobHashError, BlobHasher, BlobId, BlobLength, BlobReadError, ByteLength, ByteOffset, ByteRange, ByteRangeError, From a4e3837bb56104dee8ea14b0d75490aa347d13c9 Mon Sep 17 00:00:00 2001 From: James Ross Date: Wed, 29 Jul 2026 21:48:23 -0700 Subject: [PATCH 012/111] Add: Plan retention root transitions --- CHANGELOG.md | 10 +- README.md | 5 +- docs/formats/segment-store-v2/requirements.md | 4 +- docs/formats/segment-store-v2/retention.md | 6 +- src/adapters/retention.rs | 6 + src/adapters/retention/transition_error.rs | 112 ++++++++++++++++ src/adapters/retention/transition_planner.rs | 120 ++++++++++++++++++ .../retention/transition_readiness.rs | 35 +++++ src/lib.rs | 17 ++- src/retention/generation_expectation.rs | 13 ++ src/retention/mod.rs | 2 + src/retention/root_generation.rs | 3 + tests/retention_transition.rs | 96 ++++++++++++++ tests/retention_transition/refusal_laws.rs | 98 ++++++++++++++ 14 files changed, 510 insertions(+), 17 deletions(-) create mode 100644 src/adapters/retention/transition_error.rs create mode 100644 src/adapters/retention/transition_planner.rs create mode 100644 src/adapters/retention/transition_readiness.rs create mode 100644 src/retention/generation_expectation.rs create mode 100644 tests/retention_transition.rs create mode 100644 tests/retention_transition/refusal_laws.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index 5acdfb9..792cd93 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -317,9 +317,13 @@ after its public API and format compatibility policies are established. history, namespace uniqueness, bounds, ordering, and all three integrity layers. Typed manifest lengths and semantic global heads now reproduce and admit the exact 144-byte head fixture with fixed framing, checksum-first - semantic admission, and explicit generation-history laws. Version-1 - immutable bytes remain authoritative; production version-2 writing remains - unavailable until issue #19's executable evidence is complete. + semantic admission, and explicit generation-history laws. Storage-independent + transition planning now compares absent or exact-generation expectations, + admits only same-namespace exact successors, preserves expected and observed + stale coordinates, and distinguishes byte-identical already-committed + replay. Version-1 immutable bytes remain authoritative; production version-2 + writing remains unavailable until issue #19's executable evidence is + complete. - Accepted ADR-0009 defines caller-supplied retention namespaces, `BlobId`/`LayoutId` reconstruction anchors, fail-closed canonical closure, generation-checked retention publication, immutable liveness snapshots, diff --git a/README.md b/README.md index 465fdf7..1b8c6a5 100644 --- a/README.md +++ b/README.md @@ -115,8 +115,9 @@ bytes, checks hard-link identity and writer-lock release, runs the production recovery classifiers and immutable-artifact admission, and reconstructs the exact published generation and visible one-zero chunk when `HEAD` exists. This matrix proves application process-death behavior; it does not simulate host -power loss. Version-2 retention values and canonical in-memory root, global -manifest, and retention-head codecs are implemented. Retention publication, +power loss. Version-2 retention values; canonical in-memory root, global +manifest, and retention-head codecs; and storage-independent expected-state +transition planning are implemented. Closure verification, publication, recovery, compaction, and garbage collection remain planned. Presence in the reference CAS does not claim retention, crash recovery, or durability. diff --git a/docs/formats/segment-store-v2/requirements.md b/docs/formats/segment-store-v2/requirements.md index 9defde7..bba03ed 100644 --- a/docs/formats/segment-store-v2/requirements.md +++ b/docs/formats/segment-store-v2/requirements.md @@ -12,12 +12,12 @@ case is not evidence. | `KEEP-RETENTION-001` | `RetentionNamespace`, `RootGeneration`, `LivenessGeneration`, profile coordinates, limits, anchors, and digests are validated typed values | `tests/retention_values.rs` and `tests/retention_root_encoding.rs` | Implemented | | `KEEP-RETENTION-002` | Root, manifest, and head codecs implement the exact canonical grammars and fixed bounds | independent golden corpus plus `tests/retention_root_encoding.rs`, `tests/retention_root_decoding.rs`, `tests/retention_manifest_codec.rs`, and `tests/retention_head_codec.rs` | Implemented | | `KEEP-RETENTION-003` | Every structural field, truncation boundary, ordering law, duplicate, overflow, flag, reserved byte, digest, checksum, and trailing byte has a precise refusal | corruption matrix and fuzz | Planned in #19 | -| `KEEP-RETENTION-004` | Retain and release compare expected and observed generations and publish exact successors only | unit and model-based transition tests | Planned in #19 | +| `KEEP-RETENTION-004` | Retain and release compare expected and observed generations and publish exact successors only | storage-independent planning in `tests/retention_transition.rs`; operation and publication evidence remains | In progress in #19 | | `KEEP-RETENTION-005` | Closure derivation is deterministic, bounded, cycle-safe, fail-closed, and verifies complete blob reconstruction | property, corruption, and adversarial catalog tests | Planned in #19 | | `KEEP-RETENTION-006` | Publication follows the exact ordered durability protocol, including new namespace-directory admission and retention of fixed-stage evidence until head commit, and returns only after cleanup synchronization | operation-order and `KEEP-CRASH-036..=052` crash-injection tests | Planned in #19 | | `KEEP-RETENTION-007` | Restart resolves every fixed-stage crash prefix to one documented lawful state or typed ambiguity | debug and release crash matrix | Planned in #19 | | `KEEP-RETENTION-008` | Readers double-collect catalog and retention heads and bind one complete catalog, manifest, and root-generation view under a `ReaderFence` | immutable snapshot and concurrency tests | Planned in #19 | -| `KEEP-RETENTION-009` | Exact already-committed retry is idempotent only while its successor remains current | retry and stale-successor tests | Planned in #19 | +| `KEEP-RETENTION-009` | Exact already-committed retry is idempotent only while its successor remains current | byte-identical readiness and stale-state planning in `tests/retention_transition.rs`; publication retry remains | In progress in #19 | | `KEEP-RETENTION-010` | Model operation sequences agree with a deterministic namespace-to-anchor-set map and never admit caller identity, paths, clocks, or application policy | model-based and source-architecture tests | Planned in #19 | diff --git a/docs/formats/segment-store-v2/retention.md b/docs/formats/segment-store-v2/retention.md index 1823e50..1601d98 100644 --- a/docs/formats/segment-store-v2/retention.md +++ b/docs/formats/segment-store-v2/retention.md @@ -160,9 +160,9 @@ retention/roots// ``` Names with alternate width, case, suffix, generation, or digest refuse. Keep -implements validated in-memory root, manifest, and head codecs with integrity -before semantic admission. Filesystem publication, transitions, recovery, and -garbage collection remain absent. +implements validated in-memory root, manifest, and head codecs plus +storage-independent expected-state transition planning. Closure verification, +filesystem publication, recovery, and garbage collection remain absent. ## Global retention manifest diff --git a/src/adapters/retention.rs b/src/adapters/retention.rs index ec3f2ff..f2d09c2 100644 --- a/src/adapters/retention.rs +++ b/src/adapters/retention.rs @@ -30,6 +30,9 @@ mod root_field_decoder; mod root_header_decoder; mod root_integrity; mod root_semantic_header; +mod transition_error; +mod transition_planner; +mod transition_readiness; pub use admitted_manifest::AdmittedRetentionManifest; pub use admitted_root::AdmittedRetentionRoot; @@ -42,3 +45,6 @@ pub use manifest_decode_error::RetentionManifestDecodeError; pub use manifest_encode_error::RetentionManifestEncodeError; pub use root_decode_error::RetentionRootDecodeError; pub use root_encode_error::RetentionRootEncodeError; +pub use transition_error::RetentionTransitionError; +pub use transition_planner::plan_retention_transition; +pub use transition_readiness::RetentionTransitionReadiness; diff --git a/src/adapters/retention/transition_error.rs b/src/adapters/retention/transition_error.rs new file mode 100644 index 0000000..1b8d887 --- /dev/null +++ b/src/adapters/retention/transition_error.rs @@ -0,0 +1,112 @@ +//! This boundary module owns exact retention transition planning failures. + +use std::{error::Error, fmt}; + +use crate::retention::RetentionGenerationExpectation; +use crate::{RetentionNamespaceDigest, RetentionRootDigest, RootGeneration, RootGenerationError}; + +/// Failure to admit one candidate retention root transition. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum RetentionTransitionError { + /// The observed namespace state disagreed with the caller expectation. + StaleGeneration { + /// Caller-supplied expected state. + expected: RetentionGenerationExpectation, + /// Exact observed current generation, or normal absence. + observed: Option, + }, + /// The candidate named a different namespace from the current root. + NamespaceMismatch { + /// Current namespace digest. + expected: RetentionNamespaceDigest, + /// Candidate namespace digest. + observed: RetentionNamespaceDigest, + }, + /// The current generation has no representable successor. + GenerationExhausted { + /// Preserved checked-generation failure. + source: RootGenerationError, + }, + /// The candidate generation was not the exact required successor. + CandidateGeneration { + /// Required candidate generation. + expected: RootGeneration, + /// Observed candidate generation. + observed: RootGeneration, + }, + /// The candidate did not name the current root digest. + CandidatePredecessor { + /// Required predecessor digest. + expected: RetentionRootDigest, + /// Candidate predecessor coordinate. + observed: Option, + }, +} + +impl fmt::Display for RetentionTransitionError { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::StaleGeneration { + expected: RetentionGenerationExpectation::Absent, + observed: Some(observed), + } => write!( + formatter, + "retention generation is stale: expected absence, observed generation {}", + observed.get() + ), + Self::StaleGeneration { + expected: RetentionGenerationExpectation::Current(expected), + observed: None, + } => write!( + formatter, + "retention generation is stale: expected generation {}, observed absence", + expected.get() + ), + Self::StaleGeneration { + expected: RetentionGenerationExpectation::Current(expected), + observed: Some(observed), + } => write!( + formatter, + "retention generation is stale: expected generation {}, observed generation {}", + expected.get(), + observed.get() + ), + Self::StaleGeneration { + expected: RetentionGenerationExpectation::Absent, + observed: None, + } => formatter.write_str( + "retention generation stale-state error carried matching absent coordinates", + ), + Self::NamespaceMismatch { .. } => { + formatter.write_str("retention candidate namespace mismatch") + } + Self::GenerationExhausted { source } => { + write!( + formatter, + "retention root generation is exhausted: {source}" + ) + } + Self::CandidateGeneration { expected, observed } => write!( + formatter, + "retention candidate generation must be {}, observed {}", + expected.get(), + observed.get() + ), + Self::CandidatePredecessor { .. } => { + formatter.write_str("retention candidate predecessor digest mismatch") + } + } + } +} + +impl Error for RetentionTransitionError { + fn source(&self) -> Option<&(dyn Error + 'static)> { + match self { + Self::GenerationExhausted { source } => Some(source), + Self::StaleGeneration { .. } + | Self::NamespaceMismatch { .. } + | Self::CandidateGeneration { .. } + | Self::CandidatePredecessor { .. } => None, + } + } +} diff --git a/src/adapters/retention/transition_planner.rs b/src/adapters/retention/transition_planner.rs new file mode 100644 index 0000000..894e0a4 --- /dev/null +++ b/src/adapters/retention/transition_planner.rs @@ -0,0 +1,120 @@ +//! This boundary module owns storage-independent retention transition planning. + +use super::{AdmittedRetentionRoot, RetentionTransitionError, RetentionTransitionReadiness}; +use crate::RootGeneration; +use crate::retention::RetentionGenerationExpectation; + +/// Compares one expected, observed, and fully admitted candidate root. +/// +/// The returned readiness performs no I/O and proves no closure availability +/// or durability. Exact byte-identical replay is admitted only while the +/// candidate remains the current root and the expectation names its prior +/// state. +/// +/// # Errors +/// +/// Returns [`RetentionTransitionError`] for stale state, namespace mismatch, +/// generation exhaustion, or a non-successor candidate. +pub fn plan_retention_transition<'encoded>( + expected: RetentionGenerationExpectation, + current: Option<&AdmittedRetentionRoot<'_>>, + candidate: AdmittedRetentionRoot<'encoded>, +) -> Result, RetentionTransitionError> { + if is_exact_replay(expected, current, &candidate)? { + return Ok(RetentionTransitionReadiness::AlreadyCommitted { candidate }); + } + require_expected_state(expected, current)?; + match current { + Some(current) => validate_successor(current, &candidate)?, + None => validate_initial(&candidate)?, + } + Ok(RetentionTransitionReadiness::Publish { candidate }) +} + +fn is_exact_replay( + expected: RetentionGenerationExpectation, + current: Option<&AdmittedRetentionRoot<'_>>, + candidate: &AdmittedRetentionRoot<'_>, +) -> Result { + let Some(current) = current else { + return Ok(false); + }; + if current.encoded() != candidate.encoded() { + return Ok(false); + } + let candidate_generation = candidate.root().generation(); + match expected { + RetentionGenerationExpectation::Absent => { + Ok(candidate_generation == RootGeneration::INITIAL) + } + RetentionGenerationExpectation::Current(generation) => { + let successor = generation + .successor() + .map_err(|source| RetentionTransitionError::GenerationExhausted { source })?; + Ok(candidate_generation == successor) + } + } +} + +fn require_expected_state( + expected: RetentionGenerationExpectation, + current: Option<&AdmittedRetentionRoot<'_>>, +) -> Result<(), RetentionTransitionError> { + let observed = current.map(|root| root.root().generation()); + let matches = match expected { + RetentionGenerationExpectation::Absent => observed.is_none(), + RetentionGenerationExpectation::Current(generation) => observed == Some(generation), + }; + if matches { + Ok(()) + } else { + Err(RetentionTransitionError::StaleGeneration { expected, observed }) + } +} + +fn validate_initial(candidate: &AdmittedRetentionRoot<'_>) -> Result<(), RetentionTransitionError> { + let observed = candidate.root().generation(); + if observed == RootGeneration::INITIAL { + Ok(()) + } else { + Err(RetentionTransitionError::CandidateGeneration { + expected: RootGeneration::INITIAL, + observed, + }) + } +} + +fn validate_successor( + current: &AdmittedRetentionRoot<'_>, + candidate: &AdmittedRetentionRoot<'_>, +) -> Result<(), RetentionTransitionError> { + let expected_namespace = current.root().namespace().digest(); + let observed_namespace = candidate.root().namespace().digest(); + if observed_namespace != expected_namespace { + return Err(RetentionTransitionError::NamespaceMismatch { + expected: expected_namespace, + observed: observed_namespace, + }); + } + let expected_generation = current + .root() + .generation() + .successor() + .map_err(|source| RetentionTransitionError::GenerationExhausted { source })?; + let observed_generation = candidate.root().generation(); + if observed_generation != expected_generation { + return Err(RetentionTransitionError::CandidateGeneration { + expected: expected_generation, + observed: observed_generation, + }); + } + let expected_predecessor = current.digest(); + let observed_predecessor = candidate.root().predecessor(); + if observed_predecessor != Some(expected_predecessor) { + return Err(RetentionTransitionError::CandidatePredecessor { + expected: expected_predecessor, + observed: observed_predecessor, + }); + } + Ok(()) +} diff --git a/src/adapters/retention/transition_readiness.rs b/src/adapters/retention/transition_readiness.rs new file mode 100644 index 0000000..2231f15 --- /dev/null +++ b/src/adapters/retention/transition_readiness.rs @@ -0,0 +1,35 @@ +//! This boundary module owns admitted retention transition readiness. + +use super::AdmittedRetentionRoot; + +/// Result of comparing one expected, observed, and candidate root. +#[must_use] +#[derive(Debug, Eq, PartialEq)] +pub enum RetentionTransitionReadiness<'encoded> { + /// The candidate is the exact next root and still requires publication. + Publish { + /// Fully admitted candidate root. + candidate: AdmittedRetentionRoot<'encoded>, + }, + /// The exact candidate bytes are already the current published root. + AlreadyCommitted { + /// Fully admitted byte-identical replay candidate. + candidate: AdmittedRetentionRoot<'encoded>, + }, +} + +impl<'encoded> RetentionTransitionReadiness<'encoded> { + /// Borrows the fully admitted candidate root. + pub const fn candidate(&self) -> &AdmittedRetentionRoot<'encoded> { + match self { + Self::Publish { candidate } | Self::AlreadyCommitted { candidate } => candidate, + } + } + + /// Consumes the readiness proof and returns the admitted candidate root. + pub fn into_candidate(self) -> AdmittedRetentionRoot<'encoded> { + match self { + Self::Publish { candidate } | Self::AlreadyCommitted { candidate } => candidate, + } + } +} diff --git a/src/lib.rs b/src/lib.rs index 5dde8b2..ee12705 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -23,8 +23,9 @@ //! a pinned writer-authorized filesystem adapter. Core retention namespaces, //! generations, realization policy, reconstruction anchors, and semantic roots //! are validated; canonical in-memory root, manifest, and head encoding and -//! decoding are available. Retention publication, recovery, and garbage -//! collection remain intentionally absent. +//! decoding plus storage-independent expected-state transition planning are +//! available. Closure verification, retention publication, recovery, and +//! garbage collection remain intentionally absent. #[cfg(test)] extern crate self as keep; @@ -106,7 +107,8 @@ pub use adapters::{ AdmittedRetentionManifest, AdmittedRetentionRoot, CanonicalRetentionHead, CanonicalRetentionManifest, CanonicalRetentionRoot, ChecksummedRetentionHead, RetentionHeadDecodeError, RetentionManifestDecodeError, RetentionManifestEncodeError, - RetentionRootDecodeError, RetentionRootEncodeError, + RetentionRootDecodeError, RetentionRootEncodeError, RetentionTransitionError, + RetentionTransitionReadiness, plan_retention_transition, }; pub use blob::{ BlobHashError, BlobHasher, BlobId, BlobLength, BlobReadError, ByteLength, ByteOffset, @@ -130,10 +132,11 @@ pub use reference::{ }; pub use retention::{ LivenessGeneration, LivenessGenerationError, RegisteredRetentionProfile, RetentionAnchor, - RetentionClosureLimit, RetentionClosureLimitError, RetentionClosureLimits, RetentionHead, - RetentionHeadError, RetentionManifest, RetentionManifestDigest, RetentionManifestEntry, - RetentionManifestError, RetentionManifestLength, RetentionManifestLengthError, - RetentionNamespace, RetentionNamespaceDigest, RetentionNamespaceError, RetentionPolicy, + RetentionClosureLimit, RetentionClosureLimitError, RetentionClosureLimits, + RetentionGenerationExpectation, RetentionHead, RetentionHeadError, RetentionManifest, + RetentionManifestDigest, RetentionManifestEntry, RetentionManifestError, + RetentionManifestLength, RetentionManifestLengthError, RetentionNamespace, + RetentionNamespaceDigest, RetentionNamespaceError, RetentionPolicy, RetentionProfileAdmissionError, RetentionRoot, RetentionRootDigest, RetentionRootError, RootGeneration, RootGenerationError, }; diff --git a/src/retention/generation_expectation.rs b/src/retention/generation_expectation.rs new file mode 100644 index 0000000..75dfb0b --- /dev/null +++ b/src/retention/generation_expectation.rs @@ -0,0 +1,13 @@ +//! This module owns caller-supplied retention generation expectations. + +use super::RootGeneration; + +/// Expected current state of one retention namespace. +#[must_use] +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum RetentionGenerationExpectation { + /// The namespace must not yet have a published root. + Absent, + /// The namespace must have exactly this current root generation. + Current(RootGeneration), +} diff --git a/src/retention/mod.rs b/src/retention/mod.rs index 1b6edba..72567e3 100644 --- a/src/retention/mod.rs +++ b/src/retention/mod.rs @@ -9,6 +9,7 @@ mod anchor; mod closure_limit; mod closure_limit_error; mod closure_limits; +mod generation_expectation; mod head; mod head_error; mod liveness_generation; @@ -35,6 +36,7 @@ pub use anchor::RetentionAnchor; pub use closure_limit::RetentionClosureLimit; pub use closure_limit_error::RetentionClosureLimitError; pub use closure_limits::RetentionClosureLimits; +pub use generation_expectation::RetentionGenerationExpectation; pub use head::RetentionHead; pub use head_error::RetentionHeadError; pub use liveness_generation::LivenessGeneration; diff --git a/src/retention/root_generation.rs b/src/retention/root_generation.rs index 488565c..e1b3a4d 100644 --- a/src/retention/root_generation.rs +++ b/src/retention/root_generation.rs @@ -13,6 +13,9 @@ use super::RootGenerationError; pub struct RootGeneration(NonZeroU64); impl RootGeneration { + /// Initial published root generation. + pub const INITIAL: Self = Self(NonZeroU64::MIN); + /// Admits one positive root generation. /// /// # Errors diff --git a/tests/retention_transition.rs b/tests/retention_transition.rs new file mode 100644 index 0000000..a673e45 --- /dev/null +++ b/tests/retention_transition.rs @@ -0,0 +1,96 @@ +//! Storage-independent retention namespace transition laws. + +#[path = "retention_transition/refusal_laws.rs"] +mod refusal_laws; +mod support; + +use std::io; + +use keep::{ + AdmittedRetentionRoot, CanonicalRetentionRoot, RetentionGenerationExpectation, + RetentionNamespace, RetentionRoot, RetentionRootDigest, RetentionTransitionReadiness, + RootGeneration, plan_retention_transition, +}; + +const ONE_ANCHOR_ROOT: &str = include_str!("../conformance/segment-store/v2/one-anchor-root.hex"); + +#[test] +fn absent_namespace_admits_only_the_initial_candidate() -> Result<(), Box> { + let bytes = fixture_bytes()?; + let candidate = AdmittedRetentionRoot::decode(&bytes)?; + let readiness = + plan_retention_transition(RetentionGenerationExpectation::Absent, None, candidate)?; + assert!(matches!( + readiness, + RetentionTransitionReadiness::Publish { candidate } + if candidate.root().generation().get() == 1 + )); + Ok(()) +} + +#[test] +fn exact_successor_and_byte_identical_replay_have_distinct_readiness() +-> Result<(), Box> { + let initial_bytes = fixture_bytes()?; + let current = AdmittedRetentionRoot::decode(&initial_bytes)?; + let successor = successor(¤t)?; + let candidate = AdmittedRetentionRoot::decode(successor.encoded())?; + let readiness = plan_retention_transition( + RetentionGenerationExpectation::Current(current.root().generation()), + Some(¤t), + candidate, + )?; + assert!(matches!( + readiness, + RetentionTransitionReadiness::Publish { candidate } + if candidate.root().generation().get() == 2 + )); + + let published = AdmittedRetentionRoot::decode(successor.encoded())?; + let replay = AdmittedRetentionRoot::decode(successor.encoded())?; + let readiness = plan_retention_transition( + RetentionGenerationExpectation::Current(current.root().generation()), + Some(&published), + replay, + )?; + assert!(matches!( + readiness, + RetentionTransitionReadiness::AlreadyCommitted { candidate } + if candidate.encoded() == successor.encoded() + )); + Ok(()) +} + +fn successor( + current: &AdmittedRetentionRoot<'_>, +) -> Result> { + candidate( + current, + current.root().namespace().as_bytes(), + current.root().generation().successor()?, + Some(current.digest()), + ) +} + +fn candidate( + current: &AdmittedRetentionRoot<'_>, + namespace: &[u8], + generation: RootGeneration, + predecessor: Option, +) -> Result> { + let root = RetentionRoot::new( + RetentionNamespace::try_from(namespace)?, + generation, + keep::RetentionPolicy::new(current.root().profile(), current.root().limits()), + predecessor, + current.root().anchors().to_vec(), + )?; + CanonicalRetentionRoot::from_root(&root).map_err(Into::into) +} + +fn fixture_bytes() -> Result, io::Error> { + let encoded = ONE_ANCHOR_ROOT + .strip_suffix('\n') + .ok_or_else(|| io::Error::other("retention root fixture lacks final newline"))?; + support::decode_hex(encoded) +} diff --git a/tests/retention_transition/refusal_laws.rs b/tests/retention_transition/refusal_laws.rs new file mode 100644 index 0000000..3e4c14d --- /dev/null +++ b/tests/retention_transition/refusal_laws.rs @@ -0,0 +1,98 @@ +//! Exact stale, mismatch, and exhaustion transition refusals. + +use keep::{ + AdmittedRetentionRoot, RetentionGenerationExpectation, RetentionTransitionError, + RootGeneration, RootGenerationError, plan_retention_transition, +}; + +use super::{candidate, fixture_bytes}; + +#[test] +fn stale_expected_state_reports_both_coordinates() -> Result<(), Box> { + let bytes = fixture_bytes()?; + let candidate = AdmittedRetentionRoot::decode(&bytes)?; + let expected = RetentionGenerationExpectation::Current(RootGeneration::new(1)?); + assert!(matches!( + plan_retention_transition(expected, None, candidate), + Err(RetentionTransitionError::StaleGeneration { + expected: error_expected, + observed: None, + }) if error_expected == expected + )); + Ok(()) +} + +#[test] +fn successor_requires_the_same_namespace_generation_and_predecessor() +-> Result<(), Box> { + let bytes = fixture_bytes()?; + let current = AdmittedRetentionRoot::decode(&bytes)?; + let expected = RetentionGenerationExpectation::Current(current.root().generation()); + + let wrong_namespace = candidate( + ¤t, + b"different", + current.root().generation().successor()?, + Some(current.digest()), + )?; + let candidate_root = AdmittedRetentionRoot::decode(wrong_namespace.encoded())?; + assert!(matches!( + plan_retention_transition(expected, Some(¤t), candidate_root), + Err(RetentionTransitionError::NamespaceMismatch { .. }) + )); + + let wrong_generation = candidate( + ¤t, + current.root().namespace().as_bytes(), + current.root().generation().successor()?.successor()?, + Some(current.digest()), + )?; + let candidate_root = AdmittedRetentionRoot::decode(wrong_generation.encoded())?; + assert!(matches!( + plan_retention_transition(expected, Some(¤t), candidate_root), + Err(RetentionTransitionError::CandidateGeneration { + expected, + observed, + }) if expected.get() == 2 && observed.get() == 3 + )); + + let other_initial = candidate(¤t, b"other", RootGeneration::new(1)?, None)?; + let other = AdmittedRetentionRoot::decode(other_initial.encoded())?; + let wrong_predecessor = candidate( + ¤t, + current.root().namespace().as_bytes(), + current.root().generation().successor()?, + Some(other.digest()), + )?; + let candidate_root = AdmittedRetentionRoot::decode(wrong_predecessor.encoded())?; + assert!(matches!( + plan_retention_transition(expected, Some(¤t), candidate_root), + Err(RetentionTransitionError::CandidatePredecessor { .. }) + )); + Ok(()) +} + +#[test] +fn maximum_current_generation_has_no_transition() -> Result<(), Box> { + let bytes = fixture_bytes()?; + let initial = AdmittedRetentionRoot::decode(&bytes)?; + let maximum = candidate( + &initial, + initial.root().namespace().as_bytes(), + RootGeneration::new(u64::MAX)?, + Some(initial.digest()), + )?; + let current = AdmittedRetentionRoot::decode(maximum.encoded())?; + let candidate = AdmittedRetentionRoot::decode(&bytes)?; + assert!(matches!( + plan_retention_transition( + RetentionGenerationExpectation::Current(current.root().generation()), + Some(¤t), + candidate, + ), + Err(RetentionTransitionError::GenerationExhausted { + source: RootGenerationError::Exhausted { current: u64::MAX }, + }) + )); + Ok(()) +} From ca0b410f9bd5bcdc67c2d2d94994371b2683784b Mon Sep 17 00:00:00 2001 From: James Ross Date: Wed, 29 Jul 2026 21:57:53 -0700 Subject: [PATCH 013/111] Docs: Define retention closure accounting --- docs/formats/segment-store-v2/README.md | 4 +- docs/formats/segment-store-v2/closure.md | 180 ++++++++++++++++++ docs/formats/segment-store-v2/rationale.md | 10 + docs/formats/segment-store-v2/requirements.md | 2 +- docs/formats/segment-store-v2/retention.md | 19 +- .../retention_store_v2_protocol_contract.rs | 33 ++++ 6 files changed, 232 insertions(+), 16 deletions(-) create mode 100644 docs/formats/segment-store-v2/closure.md diff --git a/docs/formats/segment-store-v2/README.md b/docs/formats/segment-store-v2/README.md index 63e3731..59a14d2 100644 --- a/docs/formats/segment-store-v2/README.md +++ b/docs/formats/segment-store-v2/README.md @@ -39,7 +39,9 @@ Version 2 retains every version-1 physical law and adds these: The following pages form one protocol: - [Retention records and publication](retention.md) owns canonical namespace, - root-generation, manifest, retention-head, closure, and transition rules. + root-generation, manifest, retention-head, and transition rules. +- [Closure verification](closure.md) owns deterministic traversal, exact + resource accounting, authenticated reconstruction, and closure evidence. - [GC and disposition records](gc.md) owns the canonical planned intent, completion, and recovery-disposition byte grammars. - [Migration and recovery](recovery.md) owns the exact root namespace, diff --git a/docs/formats/segment-store-v2/closure.md b/docs/formats/segment-store-v2/closure.md new file mode 100644 index 0000000..8af1486 --- /dev/null +++ b/docs/formats/segment-store-v2/closure.md @@ -0,0 +1,180 @@ +# Closure Verification + +- Status: Normative version-2 protocol; production verifier planned in issue + [#19](https://github.com/flyingrobots/keep/issues/19) +- Format coordinate: `keep.segment-store/v2` +- Requirement: [`KEEP-RETENTION-005`](requirements.md#retention-transitions) +- Decision record: + [ADR-0009](../../adr/0009-retention-roots-release-and-gc-liveness.md) + +This page defines deterministic closure traversal, exact resource accounting, +authenticated reconstruction, and canonical closure evidence. The +[retention record specification](retention.md) owns the limits stored in each +root generation. The [format rationale](rationale.md) explains why evidence +cardinality and reconstruction work use separate counters. + +## Verification boundary + +Closure verification receives: + +- one admitted root generation with a canonical anchor sequence; +- its exact registered retention-realization profile; +- one pinned, completely verified catalog generation; and +- the root's admitted closure limits. + +Profile and limit admission completes before traversal. The verifier does not +read paths, enumerate a filesystem, consult a clock, invoke a caller callback, +or replace a missing witness. Version 2 selects the single record bound to each +logical identity by the pinned catalog. + +The catalog has already admitted each bound segment record's framing, checksum, +logical identity, and payload. Closure verification consumes those proofs, +decodes layouts again under the closure budget, and authenticates each complete +logical blob. + +## Deterministic traversal + +Anchors are visited in their canonical `BlobId`, then `LayoutId`, order. For +each anchor: + +1. Schedule the anchor's layout at depth `1`. +2. Resolve the exact layout record and charge its resource units. +3. Decode the canonical layout with the named `LayoutId` as an independent + expectation. +4. Require the layout target to equal the anchor's `BlobId`. +5. Visit layout entries in logical-offset order. +6. Schedule each entry's chunk at depth `2`, resolve its exact record, and + charge its resource units. +7. Reconstruct entries in layout order, replay the exact registered storage + profile, and authenticate the complete `BlobId`. + +Version-2 flat layouts cannot exceed depth `2`. The stored depth limit may be +larger so a successor layout grammar can be represented without weakening the +format ceiling. Version 2 refuses an unknown mandatory edge instead of +interpreting it as a deeper known node. + +The visited set is keyed by `SegmentRecordIdentity`. A node is inserted when +its identity is first scheduled, before catalog lookup. An anchor is not a +closure node because the root format bounds anchors separately. Each unique +`SegmentRecordIdentity` contributes one node even when layouts or chunks are +shared. Missing members still consume their scheduled node and depth budget +before the typed missing-member refusal. + +## Exact resource accounting + +Every counter starts at zero. Every increase uses checked addition before the +corresponding lookup, decode, record consumption, or reconstruction step. An +arithmetic overflow is a typed refusal, not an implied limit breach. + +### Nodes + +The node count is the number of unique catalog record identities first +scheduled across the complete root. It includes layout and chunk identities. +It excludes anchors, catalog entries not reached by an anchor, and a repeated +logical occurrence of an already visited identity. + +### Depth + +Depth is the number of catalog record identities on the active edge path. +The layout is depth `1`; one of its chunks is depth `2`. The verifier checks +the candidate depth before scheduling the identity. The observed depth in +successful evidence is the maximum reached across the complete root, or zero +for an empty anchor set. + +### Encoded bytes + +Encoded bytes count structured closure metadata decoded by the verifier. +Version 2 charges the canonical layout payload length once for each unique +layout identity, before decoding that payload. Chunk payloads, segment framing, +root bytes, manifest bytes, catalog bytes, and profile-definition bytes do not +contribute to this counter. + +### Physical bytes + +Physical bytes bound record-backed reconstruction work rather than unique +storage footprint. The verifier charges: + +- the complete segment-record length once when each layout is consumed; and +- the complete segment-record length for every chunk occurrence consumed in + layout order. + +A repeated logical occurrence therefore consumes physical bytes again even +though it does not add a node or another canonical closure-member entry. This +rule bounds replay and blob-authentication work for layouts that repeat one +small chunk many times. Shared physical evidence is not a license for +unbounded logical reconstruction. + +## Fail-closed order + +For one first-scheduled identity, checks occur in this order: + +1. admit the candidate depth; +2. checked-add and admit the node count; +3. resolve the identity from the pinned catalog; +4. checked-add and admit the complete segment-record length; +5. for a layout, checked-add and admit its canonical layout payload length; +6. consume the already admitted record proof; and +7. decode or reconstruct its semantic content. + +An already visited chunk skips steps 2, 3, 5, and canonical-member insertion, +but each logical occurrence repeats the physical-byte check in step 4 before +its bytes enter profile replay and blob authentication. + +The first failed check in deterministic traversal order is returned. Missing, +wrong-kind, unsupported-profile, limit, overflow, layout, anchor-target, +chunk, profile-boundary, and final-blob failures remain distinct typed errors +with expected and observed state where applicable. No failure yields partial +closure evidence. + +## Canonical closure digest + +Successful verification emits 96-byte closure-member entries, one for each +unique record identity: + +| Offset | Width | Field | Canonical value | +| ---: | ---: | --- | --- | +| 0 | 1 | record kind | `1` for chunk; `2` for layout | +| 1 | 3 | reserved | zero | +| 4 | 60 | identity slot | canonical encoding below | +| 64 | 32 | record checksum | exact admitted segment-record checksum | + +The chunk identity slot is its four-byte big-endian length, then its 32-byte +digest, then 24 zero bytes. The layout identity slot is the exact 60-byte +canonical binary `LayoutId`. Entries use canonical typed-identity order: +chunks by identity slot, followed by layouts by identity slot. There are no +duplicate entries. + +The closure digest is: + +```text +BLAKE3-256( + "keep.retention-closure/v2\0" || + profile-identity-u32 || + profile-version-u32 || + profile-definition-digest || + catalog-generation-u64 || + catalog-digest || + node-count-u64 || + maximum-depth-u16 || + six-zero-reserved-bytes || + encoded-bytes-u64 || + physical-bytes-u64 || + canonical-closure-member-entries +) +``` + +All integers are unsigned big-endian. `node-count-u64` is also the entry count. +The digest binds the exact profile, catalog, observed resource use, logical +member set, and record checksums. The transition receipt binds it beside the +root's separate anchor-set digest; neither digest substitutes for the other. + +## Evidence and nonclaims + +Successful evidence records the closure digest, all four observed counters, +the exact profile coordinate, and the exact catalog generation and digest. +It proves that every anchor reconstructed and authenticated at verification +time under those coordinates. + +It does not prove application meaning, future reachability after another +generation commits, unique physical ownership, retained byte count on disk, +secure erasure, or a faster verification path than the accounted traversal. diff --git a/docs/formats/segment-store-v2/rationale.md b/docs/formats/segment-store-v2/rationale.md index 315bee8..48213b1 100644 --- a/docs/formats/segment-store-v2/rationale.md +++ b/docs/formats/segment-store-v2/rationale.md @@ -63,6 +63,16 @@ Pretending to support multiple representation policies would add an unproved abstraction. The registered single-witness profile states the current law exactly; another profile requires a successor specification and evidence. +## Charge closure evidence and reconstruction work separately + +Counting unique closure members alone would let a layout repeat one small +chunk into an effectively unbounded reconstruction. Counting every repeated +identity as another node would misstate the canonical physical evidence. +Version 2 therefore deduplicates node, encoded-metadata, and member-digest +accounting by logical identity, while charging physical record length for +every chunk occurrence consumed during reconstruction. This keeps both the +evidence set and the work bound truthful. + ## Use a kernel reader fence A durable reader registry, lease, clock, and process liveness inference were diff --git a/docs/formats/segment-store-v2/requirements.md b/docs/formats/segment-store-v2/requirements.md index bba03ed..1ac7da9 100644 --- a/docs/formats/segment-store-v2/requirements.md +++ b/docs/formats/segment-store-v2/requirements.md @@ -13,7 +13,7 @@ case is not evidence. | `KEEP-RETENTION-002` | Root, manifest, and head codecs implement the exact canonical grammars and fixed bounds | independent golden corpus plus `tests/retention_root_encoding.rs`, `tests/retention_root_decoding.rs`, `tests/retention_manifest_codec.rs`, and `tests/retention_head_codec.rs` | Implemented | | `KEEP-RETENTION-003` | Every structural field, truncation boundary, ordering law, duplicate, overflow, flag, reserved byte, digest, checksum, and trailing byte has a precise refusal | corruption matrix and fuzz | Planned in #19 | | `KEEP-RETENTION-004` | Retain and release compare expected and observed generations and publish exact successors only | storage-independent planning in `tests/retention_transition.rs`; operation and publication evidence remains | In progress in #19 | -| `KEEP-RETENTION-005` | Closure derivation is deterministic, bounded, cycle-safe, fail-closed, and verifies complete blob reconstruction | property, corruption, and adversarial catalog tests | Planned in #19 | +| `KEEP-RETENTION-005` | Closure derivation is deterministic, bounded, cycle-safe, fail-closed, and verifies complete blob reconstruction | exact accounting in `closure.md`; property, corruption, and adversarial catalog tests remain | Specified; implementation planned in #19 | | `KEEP-RETENTION-006` | Publication follows the exact ordered durability protocol, including new namespace-directory admission and retention of fixed-stage evidence until head commit, and returns only after cleanup synchronization | operation-order and `KEEP-CRASH-036..=052` crash-injection tests | Planned in #19 | | `KEEP-RETENTION-007` | Restart resolves every fixed-stage crash prefix to one documented lawful state or typed ambiguity | debug and release crash matrix | Planned in #19 | | `KEEP-RETENTION-008` | Readers double-collect catalog and retention heads and bind one complete catalog, manifest, and root-generation view under a `ReaderFence` | immutable snapshot and concurrency tests | Planned in #19 | diff --git a/docs/formats/segment-store-v2/retention.md b/docs/formats/segment-store-v2/retention.md index 1601d98..e7e9529 100644 --- a/docs/formats/segment-store-v2/retention.md +++ b/docs/formats/segment-store-v2/retention.md @@ -240,20 +240,11 @@ The checksum domain is `keep.retention-head-checksum/v2\0`. ## Closure admission Before publication, Keep pins one completely verified catalog generation and -derives the complete closure for every anchor: - -1. Resolve and admit the exact layout record named by `LayoutId`. -2. Require its embedded `BlobId` to equal the anchor `BlobId`. -3. Resolve and admit every ordered chunk identity required by that layout. -4. Verify each physical record, identity, checksum, digest, and catalog - coordinate under the stored realization profile. -5. Enforce the stored limits with checked counters and a visited set. -6. Reconstruct and authenticate the complete blob identity. - -A missing or corrupt closure member, ambiguous catalog claim, unsupported -profile, limit breach, cycle, unknown mandatory edge, identity mismatch, or -ordering error refuses the entire transition. Keep never omits one failed -member and continues with a smaller live set. +applies the exact deterministic traversal, counter units, failure order, +authenticated reconstruction, and canonical digest defined by +[Closure verification](closure.md). Any closure failure refuses the entire +transition. Keep never omits one failed member and continues with a smaller +live set. Version-2 catalog publication holds the same writer authority and proves every current retained closure against its candidate catalog before replacing the diff --git a/xtask/tests/retention_store_v2_protocol_contract.rs b/xtask/tests/retention_store_v2_protocol_contract.rs index 34ebdcb..d413b2c 100644 --- a/xtask/tests/retention_store_v2_protocol_contract.rs +++ b/xtask/tests/retention_store_v2_protocol_contract.rs @@ -42,6 +42,7 @@ fn version_two_is_one_routed_protocol() -> Result<(), Box "`keep.segment-store/v2`", "successor to `keep.segment-store/v1`", "[Retention records and publication](retention.md)", + "[Closure verification](closure.md)", "[GC and disposition records](gc.md)", "[Migration and recovery](recovery.md)", "[Migration crash points](migration-crash.md)", @@ -93,6 +94,37 @@ fn retention_records_have_exact_canonical_grammars() -> Result<(), Box Result<(), Box> { + let closure = normalized(&read(&format!("{FORMAT_ROOT}/closure.md"))?); + + for required in [ + "one pinned, completely verified catalog generation", + "first scheduled", + "anchor is not a closure node", + "unique `SegmentRecordIdentity`", + "depth `1`", + "depth `2`", + "canonical layout payload length", + "complete segment-record length", + "checked addition before", + "repeated logical occurrence", + "replay the exact registered storage profile", + "authenticate the complete `BlobId`", + "keep.retention-closure/v2\\0", + "96-byte closure-member entries", + "canonical typed-identity order", + "Missing members still consume", + ] { + assert!( + closure.contains(required), + "segment-store v2 closure contract omits `{required}`" + ); + } + Ok(()) +} + #[test] fn migration_and_recovery_define_every_authority_boundary() -> Result<(), Box> { @@ -221,6 +253,7 @@ fn requirement_ledger_names_planned_and_executable_evidence() fn version_two_pages_stay_within_the_review_threshold() -> Result<(), Box> { for name in [ "README.md", + "closure.md", "gc.md", "migration-crash.md", "migration-inventory.md", From f2f910e7c516206f8068fa88f2692d00eeb9f16a Mon Sep 17 00:00:00 2001 From: James Ross Date: Wed, 29 Jul 2026 22:13:57 -0700 Subject: [PATCH 014/111] Add: Verify pinned retention closures --- CHANGELOG.md | 10 +- README.md | 9 +- docs/formats/segment-store-v2/README.md | 16 +- docs/formats/segment-store-v2/closure.md | 3 +- docs/formats/segment-store-v2/requirements.md | 2 +- src/adapters/retention.rs | 11 ++ src/adapters/retention/closure_accounting.rs | 110 +++++++++++ src/adapters/retention/closure_digest.rs | 39 ++++ src/adapters/retention/closure_error.rs | 129 +++++++++++++ .../retention/closure_error_display.rs | 123 ++++++++++++ src/adapters/retention/closure_member.rs | 38 ++++ .../retention/closure_profile_error.rs | 28 +++ src/adapters/retention/closure_verifier.rs | 182 ++++++++++++++++++ src/adapters/retention/verified_closure.rs | 60 ++++++ src/lib.rs | 17 +- .../boundary.rs} | 2 +- src/profile/mod.rs | 14 ++ src/profile/verification.rs | 102 ++++++++++ src/profile/verification_error.rs | 31 +++ src/reference/mod.rs | 3 +- src/reference/profile_verification.rs | 122 ++++-------- src/retention/closure_counter.rs | 27 +++ src/retention/closure_digest.rs | 18 ++ src/retention/closure_usage.rs | 51 +++++ src/retention/mod.rs | 6 + tests/retention_closure.rs | 150 +++++++++++++++ 26 files changed, 1191 insertions(+), 112 deletions(-) create mode 100644 src/adapters/retention/closure_accounting.rs create mode 100644 src/adapters/retention/closure_digest.rs create mode 100644 src/adapters/retention/closure_error.rs create mode 100644 src/adapters/retention/closure_error_display.rs create mode 100644 src/adapters/retention/closure_member.rs create mode 100644 src/adapters/retention/closure_profile_error.rs create mode 100644 src/adapters/retention/closure_verifier.rs create mode 100644 src/adapters/retention/verified_closure.rs rename src/{reference/profile_boundary.rs => profile/boundary.rs} (94%) create mode 100644 src/profile/verification.rs create mode 100644 src/profile/verification_error.rs create mode 100644 src/retention/closure_counter.rs create mode 100644 src/retention/closure_digest.rs create mode 100644 src/retention/closure_usage.rs create mode 100644 tests/retention_closure.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index 792cd93..74f5188 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -321,9 +321,13 @@ after its public API and format compatibility policies are established. transition planning now compares absent or exact-generation expectations, admits only same-namespace exact successors, preserves expected and observed stale coordinates, and distinguishes byte-identical already-committed - replay. Version-1 immutable bytes remain authoritative; production version-2 - writing remains unavailable until issue #19's executable evidence is - complete. + replay. Deterministic storage-independent closure verification now derives + unique catalog members, enforces exact node, depth, encoded-byte, and + physical-byte accounting, replays the registered storage profile, + authenticates each complete retained blob, and emits a catalog-bound + canonical closure digest. Version-1 immutable bytes remain authoritative; + production version-2 writing remains unavailable until issue #19's + executable evidence is complete. - Accepted ADR-0009 defines caller-supplied retention namespaces, `BlobId`/`LayoutId` reconstruction anchors, fail-closed canonical closure, generation-checked retention publication, immutable liveness snapshots, diff --git a/README.md b/README.md index 1b8c6a5..7194bed 100644 --- a/README.md +++ b/README.md @@ -116,10 +116,11 @@ recovery classifiers and immutable-artifact admission, and reconstructs the exact published generation and visible one-zero chunk when `HEAD` exists. This matrix proves application process-death behavior; it does not simulate host power loss. Version-2 retention values; canonical in-memory root, global -manifest, and retention-head codecs; and storage-independent expected-state -transition planning are implemented. Closure verification, publication, -recovery, compaction, and garbage collection remain planned. Presence in the -reference CAS does not claim retention, crash recovery, or durability. +manifest, and retention-head codecs; storage-independent expected-state +transition planning; and deterministic bounded closure verification against a +pinned catalog are implemented. Publication, recovery, compaction, and garbage +collection remain planned. Presence in the reference CAS does not claim +retention, crash recovery, or durability. Run the complete debug-profile matrix: diff --git a/docs/formats/segment-store-v2/README.md b/docs/formats/segment-store-v2/README.md index 59a14d2..8ba96bf 100644 --- a/docs/formats/segment-store-v2/README.md +++ b/docs/formats/segment-store-v2/README.md @@ -71,10 +71,12 @@ re-encode them. The format contract is frozen by ADR-0009 and this specification. Public core types now admit exact namespace bytes, namespace digests, root and liveness generations, registered realization profiles, bounded closure policies, -reconstruction anchors, and semantic roots. The canonical root encoder matches -the independent golden record. No production version-2 decoder, transition, -migration, or writer exists yet. Requirements that remain marked as planned or -in progress in issue #19 or issue #21 are not complete implementation evidence. -A store must refuse version-2 state until the relevant parser, corruption, -golden-format, model-based, crash-injection, recovery, and fuzz evidence is -implemented. +reconstruction anchors, and semantic roots. Canonical root, manifest, and head +codecs match their independent golden records. Storage-independent transition +planning and deterministic bounded closure verification against one pinned +catalog are available. Production filesystem retention publication, recovery, +migration, and garbage collection do not exist yet. Requirements that remain +planned or in progress in issue #19 or issue #21 are not complete +implementation evidence. A store must refuse unsupported version-2 state until +the relevant corruption, model-based, crash-injection, recovery, and fuzz +evidence is implemented. diff --git a/docs/formats/segment-store-v2/closure.md b/docs/formats/segment-store-v2/closure.md index 8af1486..0f88d6c 100644 --- a/docs/formats/segment-store-v2/closure.md +++ b/docs/formats/segment-store-v2/closure.md @@ -1,6 +1,7 @@ # Closure Verification -- Status: Normative version-2 protocol; production verifier planned in issue +- Status: Normative version-2 protocol; storage-independent verifier + implemented; publication integration planned in issue [#19](https://github.com/flyingrobots/keep/issues/19) - Format coordinate: `keep.segment-store/v2` - Requirement: [`KEEP-RETENTION-005`](requirements.md#retention-transitions) diff --git a/docs/formats/segment-store-v2/requirements.md b/docs/formats/segment-store-v2/requirements.md index 1ac7da9..fa09482 100644 --- a/docs/formats/segment-store-v2/requirements.md +++ b/docs/formats/segment-store-v2/requirements.md @@ -13,7 +13,7 @@ case is not evidence. | `KEEP-RETENTION-002` | Root, manifest, and head codecs implement the exact canonical grammars and fixed bounds | independent golden corpus plus `tests/retention_root_encoding.rs`, `tests/retention_root_decoding.rs`, `tests/retention_manifest_codec.rs`, and `tests/retention_head_codec.rs` | Implemented | | `KEEP-RETENTION-003` | Every structural field, truncation boundary, ordering law, duplicate, overflow, flag, reserved byte, digest, checksum, and trailing byte has a precise refusal | corruption matrix and fuzz | Planned in #19 | | `KEEP-RETENTION-004` | Retain and release compare expected and observed generations and publish exact successors only | storage-independent planning in `tests/retention_transition.rs`; operation and publication evidence remains | In progress in #19 | -| `KEEP-RETENTION-005` | Closure derivation is deterministic, bounded, cycle-safe, fail-closed, and verifies complete blob reconstruction | exact accounting in `closure.md`; property, corruption, and adversarial catalog tests remain | Specified; implementation planned in #19 | +| `KEEP-RETENTION-005` | Closure derivation is deterministic, bounded, cycle-safe, fail-closed, and verifies complete blob reconstruction | exact accounting in `closure.md` and one-anchor success in `tests/retention_closure.rs`; property, corruption, and adversarial catalog tests remain | In progress in #19 | | `KEEP-RETENTION-006` | Publication follows the exact ordered durability protocol, including new namespace-directory admission and retention of fixed-stage evidence until head commit, and returns only after cleanup synchronization | operation-order and `KEEP-CRASH-036..=052` crash-injection tests | Planned in #19 | | `KEEP-RETENTION-007` | Restart resolves every fixed-stage crash prefix to one documented lawful state or typed ambiguity | debug and release crash matrix | Planned in #19 | | `KEEP-RETENTION-008` | Readers double-collect catalog and retention heads and bind one complete catalog, manifest, and root-generation view under a `ReaderFence` | immutable snapshot and concurrency tests | Planned in #19 | diff --git a/src/adapters/retention.rs b/src/adapters/retention.rs index f2d09c2..74d5b77 100644 --- a/src/adapters/retention.rs +++ b/src/adapters/retention.rs @@ -6,6 +6,13 @@ mod canonical_head; mod canonical_manifest; mod canonical_root; mod checksummed_head; +mod closure_accounting; +mod closure_digest; +mod closure_error; +mod closure_error_display; +mod closure_member; +mod closure_profile_error; +mod closure_verifier; mod head_decode_error; mod head_decode_error_display; mod head_decoder; @@ -33,6 +40,7 @@ mod root_semantic_header; mod transition_error; mod transition_planner; mod transition_readiness; +mod verified_closure; pub use admitted_manifest::AdmittedRetentionManifest; pub use admitted_root::AdmittedRetentionRoot; @@ -40,6 +48,8 @@ pub use canonical_head::CanonicalRetentionHead; pub use canonical_manifest::CanonicalRetentionManifest; pub use canonical_root::CanonicalRetentionRoot; pub use checksummed_head::ChecksummedRetentionHead; +pub use closure_error::RetentionClosureVerificationError; +pub use closure_verifier::verify_retention_closure; pub use head_decode_error::RetentionHeadDecodeError; pub use manifest_decode_error::RetentionManifestDecodeError; pub use manifest_encode_error::RetentionManifestEncodeError; @@ -48,3 +58,4 @@ pub use root_encode_error::RetentionRootEncodeError; pub use transition_error::RetentionTransitionError; pub use transition_planner::plan_retention_transition; pub use transition_readiness::RetentionTransitionReadiness; +pub use verified_closure::VerifiedRetentionClosure; diff --git a/src/adapters/retention/closure_accounting.rs b/src/adapters/retention/closure_accounting.rs new file mode 100644 index 0000000..c5bb0ec --- /dev/null +++ b/src/adapters/retention/closure_accounting.rs @@ -0,0 +1,110 @@ +//! This module owns checked retention-closure resource accounting. + +use crate::{ + RetentionClosureCounter, RetentionClosureLimits, RetentionClosureUsage, + RetentionClosureVerificationError, +}; + +pub(super) struct ClosureAccounting { + limits: RetentionClosureLimits, + nodes: u64, + maximum_depth: u16, + encoded_bytes: u64, + physical_bytes: u64, +} + +impl ClosureAccounting { + pub(super) const fn new(limits: RetentionClosureLimits) -> Self { + Self { + limits, + nodes: 0, + maximum_depth: 0, + encoded_bytes: 0, + physical_bytes: 0, + } + } + + pub(super) fn admit_depth( + &mut self, + observed: u16, + ) -> Result<(), RetentionClosureVerificationError> { + let maximum = self.limits.depth(); + if observed > maximum { + return Err(RetentionClosureVerificationError::LimitExceeded { + counter: RetentionClosureCounter::Depth, + maximum: u64::from(maximum), + observed: u64::from(observed), + }); + } + self.maximum_depth = self.maximum_depth.max(observed); + Ok(()) + } + + pub(super) fn add_node(&mut self) -> Result<(), RetentionClosureVerificationError> { + self.nodes = checked_add( + RetentionClosureCounter::Nodes, + self.nodes, + 1, + self.limits.nodes(), + )?; + Ok(()) + } + + pub(super) fn add_encoded( + &mut self, + incoming: u64, + ) -> Result<(), RetentionClosureVerificationError> { + self.encoded_bytes = checked_add( + RetentionClosureCounter::EncodedBytes, + self.encoded_bytes, + incoming, + self.limits.encoded_bytes(), + )?; + Ok(()) + } + + pub(super) fn add_physical( + &mut self, + incoming: u64, + ) -> Result<(), RetentionClosureVerificationError> { + self.physical_bytes = checked_add( + RetentionClosureCounter::PhysicalBytes, + self.physical_bytes, + incoming, + self.limits.physical_bytes(), + )?; + Ok(()) + } + + pub(super) const fn usage(&self) -> RetentionClosureUsage { + RetentionClosureUsage::from_verified( + self.nodes, + self.maximum_depth, + self.encoded_bytes, + self.physical_bytes, + ) + } +} + +fn checked_add( + counter: RetentionClosureCounter, + current: u64, + incoming: u64, + maximum: u64, +) -> Result { + let observed = current.checked_add(incoming).ok_or( + RetentionClosureVerificationError::CounterOverflow { + counter, + current, + incoming, + }, + )?; + if observed > maximum { + return Err(RetentionClosureVerificationError::LimitExceeded { + counter, + maximum, + observed, + }); + } + Ok(observed) +} diff --git a/src/adapters/retention/closure_digest.rs b/src/adapters/retention/closure_digest.rs new file mode 100644 index 0000000..40e0f63 --- /dev/null +++ b/src/adapters/retention/closure_digest.rs @@ -0,0 +1,39 @@ +//! This module owns canonical retention-closure transcript hashing. + +use std::collections::BTreeMap; + +use blake3::Hasher; + +use crate::{ + AdmittedSegmentRecord, CatalogDigest, CatalogGeneration, RegisteredRetentionProfile, + RetentionClosureDigest, RetentionClosureUsage, SegmentRecordIdentity, +}; + +use super::closure_member::ClosureMember; + +const DOMAIN: &[u8] = b"keep.retention-closure/v2\0"; + +pub(super) fn calculate( + profile: RegisteredRetentionProfile, + catalog_generation: CatalogGeneration, + catalog_digest: CatalogDigest, + usage: RetentionClosureUsage, + records: &BTreeMap>, +) -> RetentionClosureDigest { + let mut hasher = Hasher::new(); + hasher.update(DOMAIN); + hasher.update(&profile.identity().to_be_bytes()); + hasher.update(&profile.version().to_be_bytes()); + hasher.update(profile.digest()); + hasher.update(&catalog_generation.get().to_be_bytes()); + hasher.update(catalog_digest.as_bytes()); + hasher.update(&usage.node_count().to_be_bytes()); + hasher.update(&usage.maximum_depth().to_be_bytes()); + hasher.update(&[0_u8; 6]); + hasher.update(&usage.encoded_bytes().to_be_bytes()); + hasher.update(&usage.physical_bytes().to_be_bytes()); + for (identity, record) in records { + hasher.update(ClosureMember::new(*identity, *record).as_bytes()); + } + RetentionClosureDigest::from_verified(*hasher.finalize().as_bytes()) +} diff --git a/src/adapters/retention/closure_error.rs b/src/adapters/retention/closure_error.rs new file mode 100644 index 0000000..34a66d4 --- /dev/null +++ b/src/adapters/retention/closure_error.rs @@ -0,0 +1,129 @@ +//! This module owns typed failures from retention-closure verification. + +use std::error::Error; +use std::fmt; + +use crate::{ + BlobHashError, BlobId, ChunkingError, LayoutDecodeError, LayoutEntryLimitError, LayoutId, + ProfileBoundary, RetentionClosureCounter, SegmentRecordIdentity, StorageProfileId, +}; + +/// Failure to derive and authenticate one complete retained root closure. +#[derive(Debug)] +pub enum RetentionClosureVerificationError { + /// A checked resource counter overflowed before the next operation. + CounterOverflow { + /// Counter whose addition failed. + counter: RetentionClosureCounter, + /// Value before the failed addition. + current: u64, + /// Requested increment. + incoming: u64, + }, + /// A candidate resource observation exceeds the stored admitted limit. + LimitExceeded { + /// Counter whose limit was exceeded. + counter: RetentionClosureCounter, + /// Stored admitted maximum. + maximum: u64, + /// Candidate observed value. + observed: u64, + }, + /// The admitted node limit could not become a host-independent entry cap. + LayoutEntryLimitHostWidth { + /// Admitted node limit that did not fit the layout cap width. + observed: u64, + }, + /// The derived layout entry cap violated the layout protocol bound. + LayoutEntryLimit { + /// Exact layout-bound refusal. + source: LayoutEntryLimitError, + }, + /// The pinned catalog omits a first-scheduled closure member. + MissingMember { + /// Exact missing logical record identity. + identity: SegmentRecordIdentity, + }, + /// A selected layout failed bounded canonical decoding. + LayoutDecode { + /// Layout named by the retained anchor. + layout: LayoutId, + /// Exact decoding refusal. + source: LayoutDecodeError, + }, + /// A selected layout names another logical blob. + AnchorTargetMismatch { + /// Layout named by the retained anchor. + layout: LayoutId, + /// Blob named by the anchor. + expected: BlobId, + /// Blob embedded in the admitted layout. + observed: BlobId, + }, + /// No replay verifier implements the layout's registered storage profile. + ProfileVerifierUnavailable { + /// Layout whose profile could not be replayed. + layout: LayoutId, + /// Registered profile without a verifier. + profile: StorageProfileId, + }, + /// Replaying the registered storage profile failed. + ProfileChunking { + /// Layout whose profile was replayed. + layout: LayoutId, + /// Exact detector failure. + source: ChunkingError, + }, + /// Replayed profile boundaries differ from the admitted layout. + ProfileBoundaryMismatch { + /// Layout whose profile was replayed. + layout: LayoutId, + /// Zero-based boundary index. + index: usize, + /// Boundary committed by the layout, or absence for an extra boundary. + expected: Option, + /// Replayed boundary, or absence for a missing boundary. + observed: Option, + }, + /// Complete logical identity calculation failed. + BlobHash { + /// Layout whose bytes were hashed. + layout: LayoutId, + /// Exact hashing failure. + source: BlobHashError, + }, + /// Reconstructed bytes do not authenticate as the retained blob. + BlobIdentityMismatch { + /// Layout whose complete stream was verified. + layout: LayoutId, + /// Blob named by the retained anchor. + expected: BlobId, + /// Blob calculated from the selected chunks. + observed: BlobId, + }, +} + +impl Error for RetentionClosureVerificationError { + fn source(&self) -> Option<&(dyn Error + 'static)> { + match self { + Self::LayoutEntryLimit { source } => Some(source), + Self::LayoutDecode { source, .. } => Some(source), + Self::ProfileChunking { source, .. } => Some(source), + Self::BlobHash { source, .. } => Some(source), + Self::CounterOverflow { .. } + | Self::LimitExceeded { .. } + | Self::LayoutEntryLimitHostWidth { .. } + | Self::MissingMember { .. } + | Self::AnchorTargetMismatch { .. } + | Self::ProfileVerifierUnavailable { .. } + | Self::ProfileBoundaryMismatch { .. } + | Self::BlobIdentityMismatch { .. } => None, + } + } +} + +impl fmt::Display for RetentionClosureVerificationError { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + super::closure_error_display::display(self, formatter) + } +} diff --git a/src/adapters/retention/closure_error_display.rs b/src/adapters/retention/closure_error_display.rs new file mode 100644 index 0000000..d36d38a --- /dev/null +++ b/src/adapters/retention/closure_error_display.rs @@ -0,0 +1,123 @@ +//! This module owns stable retention-closure verification diagnostics. + +use std::fmt; + +use super::RetentionClosureVerificationError; + +pub(super) fn display( + error: &RetentionClosureVerificationError, + formatter: &mut fmt::Formatter<'_>, +) -> fmt::Result { + match error { + RetentionClosureVerificationError::CounterOverflow { + counter, + current, + incoming, + } => write!( + formatter, + "{counter} overflowed while adding {incoming} to {current}" + ), + RetentionClosureVerificationError::LimitExceeded { + counter, + maximum, + observed, + } => write!( + formatter, + "{counter} limit {maximum} was exceeded by observed value {observed}" + ), + RetentionClosureVerificationError::LayoutEntryLimitHostWidth { observed } => write!( + formatter, + "closure node limit {observed} does not fit the layout entry-limit width" + ), + RetentionClosureVerificationError::LayoutEntryLimit { source } => { + write!( + formatter, + "closure-derived layout entry limit is invalid: {source}" + ) + } + RetentionClosureVerificationError::MissingMember { identity } => match identity { + crate::SegmentRecordIdentity::Chunk(chunk) => write!( + formatter, + "pinned catalog omits closure chunk length {} digest {}", + chunk.length(), + DigestHex(chunk.digest()) + ), + crate::SegmentRecordIdentity::Layout(layout) => { + write!(formatter, "pinned catalog omits closure layout {layout}") + } + }, + RetentionClosureVerificationError::LayoutDecode { layout, source } => { + write!( + formatter, + "retained layout {layout} is not admissible: {source}" + ) + } + RetentionClosureVerificationError::AnchorTargetMismatch { + layout, + expected, + observed, + } => write!( + formatter, + "retained layout {layout} names blob {observed}, not anchor blob {expected}" + ), + RetentionClosureVerificationError::ProfileVerifierUnavailable { layout, profile } => { + write!( + formatter, + "retained layout {layout} has no replay verifier for storage profile {profile}" + ) + } + RetentionClosureVerificationError::ProfileChunking { layout, source } => { + write!( + formatter, + "storage-profile replay failed for retained layout {layout}: {source}" + ) + } + RetentionClosureVerificationError::ProfileBoundaryMismatch { + layout, + index, + expected, + observed, + } => write!( + formatter, + "storage-profile boundary {index} for retained layout {layout} expected {} but observed {}", + BoundaryDisplay(*expected), + BoundaryDisplay(*observed) + ), + RetentionClosureVerificationError::BlobHash { layout, source } => { + write!( + formatter, + "blob hashing failed for retained layout {layout}: {source}" + ) + } + RetentionClosureVerificationError::BlobIdentityMismatch { + layout, + expected, + observed, + } => write!( + formatter, + "retained layout {layout} reconstructs {observed}, not anchor blob {expected}" + ), + } +} + +struct BoundaryDisplay(Option); + +impl fmt::Display for BoundaryDisplay { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + match self.0 { + Some(boundary) => write!(formatter, "{boundary}"), + None => formatter.write_str("no boundary"), + } + } +} + +struct DigestHex<'a>(&'a [u8; 32]); + +impl fmt::Display for DigestHex<'_> { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + for byte in self.0 { + write!(formatter, "{byte:02x}")?; + } + Ok(()) + } +} diff --git a/src/adapters/retention/closure_member.rs b/src/adapters/retention/closure_member.rs new file mode 100644 index 0000000..75c67bb --- /dev/null +++ b/src/adapters/retention/closure_member.rs @@ -0,0 +1,38 @@ +//! This module owns one canonical retention-closure member entry. + +use crate::{AdmittedSegmentRecord, SegmentRecordIdentity}; + +const ENTRY_LENGTH: usize = 96; +const CHUNK_KIND: u8 = 1; +const LAYOUT_KIND: u8 = 2; + +pub(super) struct ClosureMember([u8; ENTRY_LENGTH]); + +impl ClosureMember { + pub(super) const fn new( + identity: SegmentRecordIdentity, + record: AdmittedSegmentRecord<'_>, + ) -> Self { + let mut encoded = [0_u8; ENTRY_LENGTH]; + let (kind_slot, remainder) = encoded.split_at_mut(1); + kind_slot.copy_from_slice(&[kind(identity)]); + let (_reserved, remainder) = remainder.split_at_mut(3); + let (identity_slot, checksum_slot) = remainder.split_at_mut(60); + identity_slot.copy_from_slice(&crate::adapters::segment_record_identity_encoding::encode( + identity, + )); + checksum_slot.copy_from_slice(record.checksum().as_bytes()); + Self(encoded) + } + + pub(super) const fn as_bytes(&self) -> &[u8; ENTRY_LENGTH] { + &self.0 + } +} + +const fn kind(identity: SegmentRecordIdentity) -> u8 { + match identity { + SegmentRecordIdentity::Chunk(_) => CHUNK_KIND, + SegmentRecordIdentity::Layout(_) => LAYOUT_KIND, + } +} diff --git a/src/adapters/retention/closure_profile_error.rs b/src/adapters/retention/closure_profile_error.rs new file mode 100644 index 0000000..5e4d286 --- /dev/null +++ b/src/adapters/retention/closure_profile_error.rs @@ -0,0 +1,28 @@ +//! This module owns retention mapping for storage-profile replay failures. + +use crate::profile::StorageProfileVerificationError; +use crate::{LayoutId, RetentionClosureVerificationError}; + +pub(super) const fn map( + layout: LayoutId, + error: StorageProfileVerificationError, +) -> RetentionClosureVerificationError { + match error { + StorageProfileVerificationError::Unsupported { profile } => { + RetentionClosureVerificationError::ProfileVerifierUnavailable { layout, profile } + } + StorageProfileVerificationError::Chunking { source } => { + RetentionClosureVerificationError::ProfileChunking { layout, source } + } + StorageProfileVerificationError::BoundaryMismatch { + index, + expected, + observed, + } => RetentionClosureVerificationError::ProfileBoundaryMismatch { + layout, + index, + expected, + observed, + }, + } +} diff --git a/src/adapters/retention/closure_verifier.rs b/src/adapters/retention/closure_verifier.rs new file mode 100644 index 0000000..30d9dc1 --- /dev/null +++ b/src/adapters/retention/closure_verifier.rs @@ -0,0 +1,182 @@ +//! This module owns deterministic verification of one retained-root closure. + +use std::collections::BTreeMap; + +use crate::profile::StorageProfileVerifier; +use crate::{ + AdmittedLayout, AdmittedSegmentRecord, BlobHasher, CatalogSnapshot, LayoutDecodePolicy, + LayoutEntryLimit, RetentionAnchor, RetentionClosureVerificationError, RetentionRoot, + SegmentRecordIdentity, +}; + +use super::{ + VerifiedRetentionClosure, closure_accounting::ClosureAccounting, closure_digest, + closure_profile_error, +}; + +const LAYOUT_DEPTH: u16 = 1; +const CHUNK_DEPTH: u16 = 2; + +/// Verifies every anchor against one pinned admitted catalog. +/// +/// Verification performs no I/O. It allocates one bounded ordered record index +/// and one bounded decoded entry set per anchor. Every selected chunk is +/// scanned to replay its storage profile and authenticate the complete blob. +/// +/// # Errors +/// +/// Returns the first deterministic resource, catalog-member, layout, profile, +/// or reconstructed-identity refusal. No failure returns partial evidence. +pub fn verify_retention_closure( + root: &RetentionRoot, + catalog: &CatalogSnapshot<'_, '_, '_>, +) -> Result { + let entry_limit = layout_entry_limit(root)?; + let mut verifier = ClosureVerifier::new(root, catalog, entry_limit); + for anchor in root.anchors().iter().copied() { + verifier.verify_anchor(anchor)?; + } + Ok(verifier.finish()) +} + +struct ClosureVerifier<'snapshot, 'head, 'catalog, 'records> { + root: &'snapshot RetentionRoot, + catalog: &'snapshot CatalogSnapshot<'head, 'catalog, 'records>, + entry_limit: LayoutEntryLimit, + accounting: ClosureAccounting, + records: BTreeMap>, +} + +impl<'snapshot, 'head, 'catalog, 'records> ClosureVerifier<'snapshot, 'head, 'catalog, 'records> { + const fn new( + root: &'snapshot RetentionRoot, + catalog: &'snapshot CatalogSnapshot<'head, 'catalog, 'records>, + entry_limit: LayoutEntryLimit, + ) -> Self { + Self { + root, + catalog, + entry_limit, + accounting: ClosureAccounting::new(root.limits()), + records: BTreeMap::new(), + } + } + + fn verify_anchor( + &mut self, + anchor: RetentionAnchor, + ) -> Result<(), RetentionClosureVerificationError> { + let layout_id = anchor.layout_id(); + let identity = SegmentRecordIdentity::Layout(layout_id); + let (record, first_scheduled) = self.resolve(identity, LAYOUT_DEPTH)?; + self.accounting + .add_physical(record.header().record_length().get())?; + if first_scheduled { + self.accounting + .add_encoded(record.header().payload_length().get())?; + } + let policy = LayoutDecodePolicy::new(self.entry_limit).with_expected_id(layout_id); + let layout = AdmittedLayout::decode_record(record.payload(), policy).map_err(|source| { + RetentionClosureVerificationError::LayoutDecode { + layout: layout_id, + source, + } + })?; + require_anchor_target(anchor, &layout)?; + self.verify_reconstruction(anchor, &layout) + } + + fn verify_reconstruction( + &mut self, + anchor: RetentionAnchor, + layout: &AdmittedLayout, + ) -> Result<(), RetentionClosureVerificationError> { + let layout_id = anchor.layout_id(); + let mut profile = StorageProfileVerifier::new(layout) + .map_err(|error| closure_profile_error::map(layout_id, error))?; + let mut blob = BlobHasher::new(); + for entry in layout.entries().iter().copied() { + let identity = SegmentRecordIdentity::Chunk(entry.chunk_id()); + let (record, _first_scheduled) = self.resolve(identity, CHUNK_DEPTH)?; + self.accounting + .add_physical(record.header().record_length().get())?; + let bytes = record.payload(); + profile + .feed(bytes) + .map_err(|error| closure_profile_error::map(layout_id, error))?; + blob.update(bytes) + .map_err(|source| RetentionClosureVerificationError::BlobHash { + layout: layout_id, + source, + })?; + } + profile + .finish() + .map_err(|error| closure_profile_error::map(layout_id, error))?; + let observed = blob.finish(); + let expected = anchor.blob_id(); + if observed != expected { + return Err(RetentionClosureVerificationError::BlobIdentityMismatch { + layout: layout_id, + expected, + observed, + }); + } + Ok(()) + } + + fn resolve( + &mut self, + identity: SegmentRecordIdentity, + depth: u16, + ) -> Result<(AdmittedSegmentRecord<'records>, bool), RetentionClosureVerificationError> { + self.accounting.admit_depth(depth)?; + if let Some(record) = self.records.get(&identity).copied() { + return Ok((record, false)); + } + self.accounting.add_node()?; + let record = self + .catalog + .record(identity) + .ok_or(RetentionClosureVerificationError::MissingMember { identity })?; + self.records.insert(identity, record); + Ok((record, true)) + } + + fn finish(self) -> VerifiedRetentionClosure { + let profile = self.root.profile(); + let generation = self.catalog.generation(); + let catalog_digest = self.catalog.catalog_digest(); + let usage = self.accounting.usage(); + let digest = + closure_digest::calculate(profile, generation, catalog_digest, usage, &self.records); + VerifiedRetentionClosure::new(profile, generation, catalog_digest, usage, digest) + } +} + +fn layout_entry_limit( + root: &RetentionRoot, +) -> Result { + let observed = root.limits().nodes(); + let value = u32::try_from(observed).map_err(|_source| { + RetentionClosureVerificationError::LayoutEntryLimitHostWidth { observed } + })?; + LayoutEntryLimit::new(value) + .map_err(|source| RetentionClosureVerificationError::LayoutEntryLimit { source }) +} + +fn require_anchor_target( + anchor: RetentionAnchor, + layout: &AdmittedLayout, +) -> Result<(), RetentionClosureVerificationError> { + let expected = anchor.blob_id(); + let observed = layout.target(); + if observed == expected { + return Ok(()); + } + Err(RetentionClosureVerificationError::AnchorTargetMismatch { + layout: anchor.layout_id(), + expected, + observed, + }) +} diff --git a/src/adapters/retention/verified_closure.rs b/src/adapters/retention/verified_closure.rs new file mode 100644 index 0000000..ecae6ca --- /dev/null +++ b/src/adapters/retention/verified_closure.rs @@ -0,0 +1,60 @@ +//! This module owns successful retention-closure verification evidence. + +use crate::{ + CatalogDigest, CatalogGeneration, RegisteredRetentionProfile, RetentionClosureDigest, + RetentionClosureUsage, +}; + +/// Exact coordinates and accounting for one completely verified closure. +#[must_use = "verified closure evidence binds the transition's physical claim"] +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub struct VerifiedRetentionClosure { + profile: RegisteredRetentionProfile, + catalog_generation: CatalogGeneration, + catalog_digest: CatalogDigest, + usage: RetentionClosureUsage, + digest: RetentionClosureDigest, +} + +impl VerifiedRetentionClosure { + /// Returns the exact registered retention-realization profile. + pub const fn profile(self) -> RegisteredRetentionProfile { + self.profile + } + + /// Returns the pinned catalog generation used for verification. + pub const fn catalog_generation(self) -> CatalogGeneration { + self.catalog_generation + } + + /// Returns the pinned catalog digest used for verification. + pub const fn catalog_digest(self) -> CatalogDigest { + self.catalog_digest + } + + /// Returns the exact successful resource accounting. + pub const fn usage(self) -> RetentionClosureUsage { + self.usage + } + + /// Returns the canonical closure transcript digest. + pub const fn digest(self) -> RetentionClosureDigest { + self.digest + } + + pub(super) const fn new( + profile: RegisteredRetentionProfile, + catalog_generation: CatalogGeneration, + catalog_digest: CatalogDigest, + usage: RetentionClosureUsage, + digest: RetentionClosureDigest, + ) -> Self { + Self { + profile, + catalog_generation, + catalog_digest, + usage, + digest, + } + } +} diff --git a/src/lib.rs b/src/lib.rs index ee12705..12b90ed 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -23,9 +23,10 @@ //! a pinned writer-authorized filesystem adapter. Core retention namespaces, //! generations, realization policy, reconstruction anchors, and semantic roots //! are validated; canonical in-memory root, manifest, and head encoding and -//! decoding plus storage-independent expected-state transition planning are -//! available. Closure verification, retention publication, recovery, and -//! garbage collection remain intentionally absent. +//! decoding, storage-independent expected-state transition planning, and +//! deterministic bounded closure verification against a pinned catalog are +//! available. Retention publication, recovery, and garbage collection remain +//! intentionally absent. #[cfg(test)] extern crate self as keep; @@ -106,9 +107,10 @@ pub use adapters::{ pub use adapters::{ AdmittedRetentionManifest, AdmittedRetentionRoot, CanonicalRetentionHead, CanonicalRetentionManifest, CanonicalRetentionRoot, ChecksummedRetentionHead, - RetentionHeadDecodeError, RetentionManifestDecodeError, RetentionManifestEncodeError, - RetentionRootDecodeError, RetentionRootEncodeError, RetentionTransitionError, - RetentionTransitionReadiness, plan_retention_transition, + RetentionClosureVerificationError, RetentionHeadDecodeError, RetentionManifestDecodeError, + RetentionManifestEncodeError, RetentionRootDecodeError, RetentionRootEncodeError, + RetentionTransitionError, RetentionTransitionReadiness, VerifiedRetentionClosure, + plan_retention_transition, verify_retention_closure, }; pub use blob::{ BlobHashError, BlobHasher, BlobId, BlobLength, BlobReadError, ByteLength, ByteOffset, @@ -132,7 +134,8 @@ pub use reference::{ }; pub use retention::{ LivenessGeneration, LivenessGenerationError, RegisteredRetentionProfile, RetentionAnchor, - RetentionClosureLimit, RetentionClosureLimitError, RetentionClosureLimits, + RetentionClosureCounter, RetentionClosureDigest, RetentionClosureLimit, + RetentionClosureLimitError, RetentionClosureLimits, RetentionClosureUsage, RetentionGenerationExpectation, RetentionHead, RetentionHeadError, RetentionManifest, RetentionManifestDigest, RetentionManifestEntry, RetentionManifestError, RetentionManifestLength, RetentionManifestLengthError, RetentionNamespace, diff --git a/src/reference/profile_boundary.rs b/src/profile/boundary.rs similarity index 94% rename from src/reference/profile_boundary.rs rename to src/profile/boundary.rs index 5eabba9..16969fd 100644 --- a/src/reference/profile_boundary.rs +++ b/src/profile/boundary.rs @@ -1,4 +1,4 @@ -//! Compact semantic coordinate for profile-replay diagnostics. +//! This module owns one semantic storage-profile boundary coordinate. use std::fmt; diff --git a/src/profile/mod.rs b/src/profile/mod.rs index 4a6f5b5..05987fe 100644 --- a/src/profile/mod.rs +++ b/src/profile/mod.rs @@ -5,9 +5,23 @@ //! profile selection policy, storage, or application metadata. mod admission_error; +mod boundary; mod id; mod registered; +mod verification; +mod verification_error; pub use admission_error::StorageProfileAdmissionError; +pub use boundary::ProfileBoundary; pub use id::StorageProfileId; pub use registered::RegisteredStorageProfile; +#[allow( + clippy::redundant_pub_crate, + reason = "sibling adapters share profile replay without exposing it publicly" +)] +pub(crate) use verification::StorageProfileVerifier; +#[allow( + clippy::redundant_pub_crate, + reason = "sibling adapters map the same domain replay failures" +)] +pub(crate) use verification_error::StorageProfileVerificationError; diff --git a/src/profile/verification.rs b/src/profile/verification.rs new file mode 100644 index 0000000..3803e34 --- /dev/null +++ b/src/profile/verification.rs @@ -0,0 +1,102 @@ +//! This module owns streaming replay of one admitted storage profile. + +use crate::{AdmittedLayout, ChunkSpan, FastCdc, LayoutEntry, RegisteredStorageProfile}; + +use super::{ProfileBoundary, StorageProfileVerificationError}; + +/// Streaming verifier for the profile and boundaries bound by one layout. +#[allow( + clippy::redundant_pub_crate, + reason = "sibling adapters share profile replay without depending on each other" +)] +pub(crate) struct StorageProfileVerifier<'a> { + detector: FastCdc, + observation: BoundaryObservation<'a>, +} + +impl<'a> StorageProfileVerifier<'a> { + /// Starts replay for one already admitted layout. + pub(crate) fn new(layout: &'a AdmittedLayout) -> Result { + if layout.profile() != RegisteredStorageProfile::FAST_CDC_64K_V1 { + return Err(StorageProfileVerificationError::Unsupported { + profile: layout.profile().id(), + }); + } + Ok(Self { + detector: FastCdc::new(), + observation: BoundaryObservation::new(layout.entries()), + }) + } + + /// Feeds the next exact logical byte span in layout order. + pub(crate) fn feed(&mut self, bytes: &[u8]) -> Result<(), StorageProfileVerificationError> { + let observation = &mut self.observation; + self.detector + .feed(bytes, |span| observation.observe(span)) + .map_err(|source| StorageProfileVerificationError::Chunking { source })?; + observation.check() + } + + /// Finishes replay and requires the exact admitted boundary sequence. + pub(crate) fn finish(mut self) -> Result<(), StorageProfileVerificationError> { + let final_span = self + .detector + .finish() + .map_err(|source| StorageProfileVerificationError::Chunking { source })?; + if let Some(span) = final_span { + self.observation.observe(span); + } + self.observation.finish() + } +} + +struct BoundaryObservation<'a> { + expected: &'a [LayoutEntry], + next: usize, + mismatch: Option, +} + +impl<'a> BoundaryObservation<'a> { + const fn new(expected: &'a [LayoutEntry]) -> Self { + Self { + expected, + next: 0, + mismatch: None, + } + } + + fn observe(&mut self, span: ChunkSpan) { + if self.mismatch.is_some() { + return; + } + let observed = LayoutEntry::from(span); + let expected = self.expected.get(self.next).copied(); + if expected == Some(observed) + && let Some(accepted) = self.expected.get(..=self.next) + { + self.next = accepted.len(); + return; + } + self.mismatch = Some(StorageProfileVerificationError::BoundaryMismatch { + index: self.next, + expected: expected.map(ProfileBoundary::from), + observed: Some(ProfileBoundary::from(observed)), + }); + } + + fn check(&mut self) -> Result<(), StorageProfileVerificationError> { + self.mismatch.take().map_or(Ok(()), Err) + } + + fn finish(mut self) -> Result<(), StorageProfileVerificationError> { + self.check()?; + if let Some(expected) = self.expected.get(self.next).copied() { + return Err(StorageProfileVerificationError::BoundaryMismatch { + index: self.next, + expected: Some(ProfileBoundary::from(expected)), + observed: None, + }); + } + Ok(()) + } +} diff --git a/src/profile/verification_error.rs b/src/profile/verification_error.rs new file mode 100644 index 0000000..733d018 --- /dev/null +++ b/src/profile/verification_error.rs @@ -0,0 +1,31 @@ +//! This module owns storage-profile replay failures independent of adapters. + +use crate::{ChunkingError, ProfileBoundary, StorageProfileId}; + +/// Failure while replaying one admitted storage profile over logical bytes. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +#[allow( + clippy::redundant_pub_crate, + reason = "sibling adapters map the same domain replay failures" +)] +pub(crate) enum StorageProfileVerificationError { + /// No replay verifier implements the admitted profile. + Unsupported { + /// Registered profile without a verifier. + profile: StorageProfileId, + }, + /// The registered detector refused the supplied byte stream. + Chunking { + /// Exact detector failure. + source: ChunkingError, + }, + /// Replayed boundaries differ from the admitted layout. + BoundaryMismatch { + /// Zero-based boundary index. + index: usize, + /// Boundary committed by the layout, or absence for an extra boundary. + expected: Option, + /// Replayed boundary, or absence for a missing boundary. + observed: Option, + }, +} diff --git a/src/reference/mod.rs b/src/reference/mod.rs index 9ef043d..d612492 100644 --- a/src/reference/mod.rs +++ b/src/reference/mod.rs @@ -10,7 +10,6 @@ mod chunk_verification; mod ingestion; mod ingestion_error; mod output_write; -mod profile_boundary; mod profile_verification; mod publish_error; mod published_blob; @@ -27,9 +26,9 @@ mod reconstruction_receipt; mod staged_blob; mod store; +pub use crate::profile::ProfileBoundary; pub use capacity::ReferenceStoreCapacity; pub use ingestion_error::{IngestionAllocation, IngestionError}; -pub use profile_boundary::ProfileBoundary; pub use publish_error::PublishError; pub use published_blob::PublishedBlob; pub use range_read_error::RangeReadError; diff --git a/src/reference/profile_verification.rs b/src/reference/profile_verification.rs index 97b6357..e14e061 100644 --- a/src/reference/profile_verification.rs +++ b/src/reference/profile_verification.rs @@ -1,108 +1,58 @@ -//! Registered storage-profile replay during reconstruction. +//! Reference-adapter mapping for domain-owned storage-profile replay. -use crate::{AdmittedLayout, ChunkSpan, FastCdc, LayoutEntry, LayoutId, RegisteredStorageProfile}; +use crate::profile::{StorageProfileVerificationError, StorageProfileVerifier}; +use crate::{AdmittedLayout, LayoutId}; -use super::{ProfileBoundary, ReconstructionError}; +use super::ReconstructionError; pub(super) struct ProfileVerifier<'a> { - detector: FastCdc, - observation: BoundaryObservation<'a>, + layout: LayoutId, + verifier: StorageProfileVerifier<'a>, } impl<'a> ProfileVerifier<'a> { pub(super) fn new( - layout_id: LayoutId, - layout: &'a AdmittedLayout, + layout: LayoutId, + admitted: &'a AdmittedLayout, ) -> Result { - if layout.profile() != RegisteredStorageProfile::FAST_CDC_64K_V1 { - return Err(ReconstructionError::ProfileVerifierUnavailable { - layout: layout_id, - profile: layout.profile().id(), - }); - } - Ok(Self { - detector: FastCdc::new(), - observation: BoundaryObservation::new(layout_id, layout.entries()), - }) + let verifier = + StorageProfileVerifier::new(admitted).map_err(|error| map_error(layout, error))?; + Ok(Self { layout, verifier }) } pub(super) fn feed(&mut self, bytes: &[u8]) -> Result<(), ReconstructionError> { - let observation = &mut self.observation; - self.detector - .feed(bytes, |span| observation.observe(span)) - .map_err(|source| ReconstructionError::ProfileChunking { - layout: observation.layout, - source, - })?; - observation.check() + self.verifier + .feed(bytes) + .map_err(|error| map_error(self.layout, error)) } - pub(super) fn finish(mut self) -> Result<(), ReconstructionError> { - let final_span = - self.detector - .finish() - .map_err(|source| ReconstructionError::ProfileChunking { - layout: self.observation.layout, - source, - })?; - if let Some(span) = final_span { - self.observation.observe(span); - } - self.observation.finish() + pub(super) fn finish(self) -> Result<(), ReconstructionError> { + self.verifier + .finish() + .map_err(|error| map_error(self.layout, error)) } } -struct BoundaryObservation<'a> { +const fn map_error( layout: LayoutId, - expected: &'a [LayoutEntry], - next: usize, - mismatch: Option, -} - -impl<'a> BoundaryObservation<'a> { - const fn new(layout: LayoutId, expected: &'a [LayoutEntry]) -> Self { - Self { - layout, - expected, - next: 0, - mismatch: None, - } - } - - fn observe(&mut self, span: ChunkSpan) { - if self.mismatch.is_some() { - return; + error: StorageProfileVerificationError, +) -> ReconstructionError { + match error { + StorageProfileVerificationError::Unsupported { profile } => { + ReconstructionError::ProfileVerifierUnavailable { layout, profile } } - let observed = LayoutEntry::from(span); - let expected = self.expected.get(self.next).copied(); - if expected == Some(observed) - && let Some(accepted) = self.expected.get(..=self.next) - { - self.next = accepted.len(); - return; + StorageProfileVerificationError::Chunking { source } => { + ReconstructionError::ProfileChunking { layout, source } } - self.mismatch = Some(ReconstructionError::ProfileBoundaryMismatch { - layout: self.layout, - index: self.next, - expected: expected.map(ProfileBoundary::from), - observed: Some(ProfileBoundary::from(observed)), - }); - } - - fn check(&mut self) -> Result<(), ReconstructionError> { - self.mismatch.take().map_or(Ok(()), Err) - } - - fn finish(mut self) -> Result<(), ReconstructionError> { - self.check()?; - if let Some(expected) = self.expected.get(self.next).copied() { - return Err(ReconstructionError::ProfileBoundaryMismatch { - layout: self.layout, - index: self.next, - expected: Some(ProfileBoundary::from(expected)), - observed: None, - }); - } - Ok(()) + StorageProfileVerificationError::BoundaryMismatch { + index, + expected, + observed, + } => ReconstructionError::ProfileBoundaryMismatch { + layout, + index, + expected, + observed, + }, } } diff --git a/src/retention/closure_counter.rs b/src/retention/closure_counter.rs new file mode 100644 index 0000000..172ce0a --- /dev/null +++ b/src/retention/closure_counter.rs @@ -0,0 +1,27 @@ +//! This module owns typed retention-closure resource dimensions. + +use std::fmt; + +/// Resource dimension enforced during retention-closure verification. +#[derive(Clone, Copy, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)] +pub enum RetentionClosureCounter { + /// Unique first-scheduled catalog record identities. + Nodes, + /// Maximum catalog-record edge depth. + Depth, + /// Unique structured layout payload bytes decoded. + EncodedBytes, + /// Complete record bytes charged to reconstruction work. + PhysicalBytes, +} + +impl fmt::Display for RetentionClosureCounter { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter.write_str(match self { + Self::Nodes => "closure nodes", + Self::Depth => "closure depth", + Self::EncodedBytes => "encoded closure bytes", + Self::PhysicalBytes => "physical closure bytes", + }) + } +} diff --git a/src/retention/closure_digest.rs b/src/retention/closure_digest.rs new file mode 100644 index 0000000..d4ee12d --- /dev/null +++ b/src/retention/closure_digest.rs @@ -0,0 +1,18 @@ +//! This module owns one verified version-2 retention-closure digest. + +/// BLAKE3-256 digest of one canonical verified closure transcript. +#[must_use] +#[derive(Clone, Copy, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)] +pub struct RetentionClosureDigest([u8; 32]); + +impl RetentionClosureDigest { + /// Returns the exact digest bytes. + #[must_use] + pub const fn as_bytes(&self) -> &[u8; 32] { + &self.0 + } + + pub(crate) const fn from_verified(bytes: [u8; 32]) -> Self { + Self(bytes) + } +} diff --git a/src/retention/closure_usage.rs b/src/retention/closure_usage.rs new file mode 100644 index 0000000..06f015f --- /dev/null +++ b/src/retention/closure_usage.rs @@ -0,0 +1,51 @@ +//! This module owns observed resource use for one verified retention closure. + +/// Exact successful resource accounting for one complete retained root. +#[must_use] +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub struct RetentionClosureUsage { + node_count: u64, + maximum_depth: u16, + encoded_bytes: u64, + physical_bytes: u64, +} + +impl RetentionClosureUsage { + /// Returns the unique first-scheduled record count. + #[must_use] + pub const fn node_count(self) -> u64 { + self.node_count + } + + /// Returns the maximum catalog-record edge depth reached. + #[must_use] + pub const fn maximum_depth(self) -> u16 { + self.maximum_depth + } + + /// Returns the unique structured layout payload bytes decoded. + #[must_use] + pub const fn encoded_bytes(self) -> u64 { + self.encoded_bytes + } + + /// Returns complete record bytes charged to reconstruction work. + #[must_use] + pub const fn physical_bytes(self) -> u64 { + self.physical_bytes + } + + pub(crate) const fn from_verified( + node_count: u64, + maximum_depth: u16, + encoded_bytes: u64, + physical_bytes: u64, + ) -> Self { + Self { + node_count, + maximum_depth, + encoded_bytes, + physical_bytes, + } + } +} diff --git a/src/retention/mod.rs b/src/retention/mod.rs index 72567e3..a9da928 100644 --- a/src/retention/mod.rs +++ b/src/retention/mod.rs @@ -6,9 +6,12 @@ //! collection. mod anchor; +mod closure_counter; +mod closure_digest; mod closure_limit; mod closure_limit_error; mod closure_limits; +mod closure_usage; mod generation_expectation; mod head; mod head_error; @@ -33,9 +36,12 @@ mod root_generation; mod root_generation_error; pub use anchor::RetentionAnchor; +pub use closure_counter::RetentionClosureCounter; +pub use closure_digest::RetentionClosureDigest; pub use closure_limit::RetentionClosureLimit; pub use closure_limit_error::RetentionClosureLimitError; pub use closure_limits::RetentionClosureLimits; +pub use closure_usage::RetentionClosureUsage; pub use generation_expectation::RetentionGenerationExpectation; pub use head::RetentionHead; pub use head_error::RetentionHeadError; diff --git a/tests/retention_closure.rs b/tests/retention_closure.rs new file mode 100644 index 0000000..4a6bf37 --- /dev/null +++ b/tests/retention_closure.rs @@ -0,0 +1,150 @@ +//! Deterministic retention-closure verification laws. + +mod support; + +use std::error::Error; + +use blake3::Hasher; +use keep::{ + AdmittedCatalog, AdmittedSegment, BlobId, CatalogSnapshot, ChecksummedCatalog, + ChecksummedPublicationHead, LayoutEntryLimit, LayoutId, RegisteredRetentionProfile, + RetentionAnchor, RetentionClosureLimits, RetentionNamespace, RetentionPolicy, RetentionRoot, + RootGeneration, SegmentReadPolicy, SegmentRecordLimit, verify_retention_closure, +}; +use support::decode_hex; + +const BUNDLE_SEGMENT_HEX: &str = + include_str!("../conformance/segment-store/v1/one-zero-bundle-segment.hex"); +const BUNDLE_CATALOG_HEX: &str = + include_str!("../conformance/segment-store/v1/one-zero-bundle-catalog.hex"); +const BUNDLE_HEAD_HEX: &str = + include_str!("../conformance/segment-store/v1/one-zero-bundle-head.hex"); +const ONE_ZERO_BLOB: &str = concat!( + "keep:blob:v1:blake3-256:1:", + "1cfb8fa9e917aba15a1f592095f377ff180755fe1212b0d7d2ec750bd128b606" +); +const ONE_ZERO_LAYOUT: &str = concat!( + "keep:layout:v1:flat-chunks-v1:blake3-256:220:", + "887da23f1a7483359a78fc9a7fde80030ec2c4690603803f0ab7d0edb56575b8" +); +const CHUNK_DIGEST_HEX: &str = "9b9c9a42912a0efdcd41e83ea024d72f10f2627d239e4eb240dd53f39ce0ff62"; +const CHUNK_RECORD_CHECKSUM_HEX: &str = + "becb46b35120723210798a47e26144b8214d5ea65d28806e0ba941d2aa66bbfa"; +const LAYOUT_RECORD_CHECKSUM_HEX: &str = + "c498a9c3cc24142926857d778fee7fd622b8b03312318a2360a68be3461168d6"; +const CLOSURE_DOMAIN: &[u8] = b"keep.retention-closure/v2\0"; + +#[test] +fn one_anchor_closure_binds_exact_evidence_and_authenticated_bytes() -> Result<(), Box> { + let segment_bytes = fixture(BUNDLE_SEGMENT_HEX)?; + let catalog_bytes = fixture(BUNDLE_CATALOG_HEX)?; + let head_bytes = fixture(BUNDLE_HEAD_HEX)?; + let segment = AdmittedSegment::decode(&segment_bytes, maximum_policy())?; + let segments = [segment]; + let catalog = admitted_catalog(&catalog_bytes, &segments)?; + let head = ChecksummedPublicationHead::decode(&head_bytes)?; + let snapshot = head.admit(catalog)?; + let root = one_anchor_root()?; + + let evidence = verify_retention_closure(&root, &snapshot)?; + + assert_eq!(evidence.profile(), root.profile()); + assert_eq!(evidence.catalog_generation(), snapshot.generation()); + assert_eq!(evidence.catalog_digest(), snapshot.catalog_digest()); + assert_eq!(evidence.usage().node_count(), 2); + assert_eq!(evidence.usage().maximum_depth(), 2); + assert_eq!(evidence.usage().encoded_bytes(), 220); + assert_eq!(evidence.usage().physical_bytes(), 509); + assert_eq!(evidence.digest().as_bytes(), &expected_digest(&snapshot)?); + Ok(()) +} + +fn admitted_catalog<'catalog, 'records>( + catalog_bytes: &'catalog [u8], + segments: &'records [AdmittedSegment<'records>], +) -> Result, Box> { + ChecksummedCatalog::decode(catalog_bytes)? + .admit(segments) + .map_err(Into::into) +} + +fn one_anchor_root() -> Result> { + let blob: BlobId = ONE_ZERO_BLOB.parse()?; + let layout: LayoutId = ONE_ZERO_LAYOUT.parse()?; + Ok(RetentionRoot::new( + RetentionNamespace::try_from(b"contract".as_slice())?, + RootGeneration::new(1)?, + RetentionPolicy::new( + RegisteredRetentionProfile::SINGLE_CANONICAL_WITNESS_V1, + RetentionClosureLimits::new(2, 2, 220, 509)?, + ), + None, + vec![RetentionAnchor::new(blob, layout)], + )?) +} + +fn expected_digest(snapshot: &CatalogSnapshot<'_, '_, '_>) -> Result<[u8; 32], Box> { + let profile = RegisteredRetentionProfile::SINGLE_CANONICAL_WITNESS_V1; + let mut hasher = Hasher::new(); + hasher.update(CLOSURE_DOMAIN); + hasher.update(&profile.identity().to_be_bytes()); + hasher.update(&profile.version().to_be_bytes()); + hasher.update(profile.digest()); + hasher.update(&snapshot.generation().get().to_be_bytes()); + hasher.update(snapshot.catalog_digest().as_bytes()); + hasher.update(&2_u64.to_be_bytes()); + hasher.update(&2_u16.to_be_bytes()); + hasher.update(&[0_u8; 6]); + hasher.update(&220_u64.to_be_bytes()); + hasher.update(&509_u64.to_be_bytes()); + hasher.update(&chunk_member()?); + hasher.update(&layout_member()?); + Ok(*hasher.finalize().as_bytes()) +} + +fn chunk_member() -> Result<[u8; 96], Box> { + let mut entry = [0_u8; 96]; + *entry.first_mut().ok_or("closure member has no kind byte")? = 1; + entry + .get_mut(4..8) + .ok_or("closure member lacks chunk length")? + .copy_from_slice(&1_u32.to_be_bytes()); + entry + .get_mut(8..40) + .ok_or("closure member lacks chunk digest")? + .copy_from_slice(&digest(CHUNK_DIGEST_HEX)?); + entry + .get_mut(64..96) + .ok_or("closure member lacks checksum")? + .copy_from_slice(&digest(CHUNK_RECORD_CHECKSUM_HEX)?); + Ok(entry) +} + +fn layout_member() -> Result<[u8; 96], Box> { + let layout: LayoutId = ONE_ZERO_LAYOUT.parse()?; + let mut entry = [0_u8; 96]; + *entry.first_mut().ok_or("closure member has no kind byte")? = 2; + entry + .get_mut(4..64) + .ok_or("closure member lacks layout identity")? + .copy_from_slice(&layout.encode_binary()); + entry + .get_mut(64..96) + .ok_or("closure member lacks checksum")? + .copy_from_slice(&digest(LAYOUT_RECORD_CHECKSUM_HEX)?); + Ok(entry) +} + +fn digest(hex: &str) -> Result<[u8; 32], Box> { + decode_hex(hex)? + .try_into() + .map_err(|_source| "digest fixture is not 32 bytes".into()) +} + +const fn maximum_policy() -> SegmentReadPolicy { + SegmentReadPolicy::new(SegmentRecordLimit::MAXIMUM, LayoutEntryLimit::MAXIMUM) +} + +fn fixture(hex: &str) -> Result, Box> { + decode_hex(hex.strip_suffix('\n').ok_or("fixture must end in one LF")?).map_err(Into::into) +} From 96241455b0f5305b055792a3ebafa2efaa071c1b Mon Sep 17 00:00:00 2001 From: James Ross Date: Wed, 29 Jul 2026 22:25:55 -0700 Subject: [PATCH 015/111] Fix: Decouple repeated closure entries from nodes --- docs/formats/segment-store-v2/requirements.md | 2 +- src/adapters/retention/closure_error.rs | 16 +- .../retention/closure_error_display.rs | 10 - src/adapters/retention/closure_verifier.rs | 19 +- tests/retention_closure.rs | 2 + tests/retention_closure/repeated_chunk_law.rs | 180 ++++++++++++++++++ 6 files changed, 187 insertions(+), 42 deletions(-) create mode 100644 tests/retention_closure/repeated_chunk_law.rs diff --git a/docs/formats/segment-store-v2/requirements.md b/docs/formats/segment-store-v2/requirements.md index fa09482..ccd9c81 100644 --- a/docs/formats/segment-store-v2/requirements.md +++ b/docs/formats/segment-store-v2/requirements.md @@ -13,7 +13,7 @@ case is not evidence. | `KEEP-RETENTION-002` | Root, manifest, and head codecs implement the exact canonical grammars and fixed bounds | independent golden corpus plus `tests/retention_root_encoding.rs`, `tests/retention_root_decoding.rs`, `tests/retention_manifest_codec.rs`, and `tests/retention_head_codec.rs` | Implemented | | `KEEP-RETENTION-003` | Every structural field, truncation boundary, ordering law, duplicate, overflow, flag, reserved byte, digest, checksum, and trailing byte has a precise refusal | corruption matrix and fuzz | Planned in #19 | | `KEEP-RETENTION-004` | Retain and release compare expected and observed generations and publish exact successors only | storage-independent planning in `tests/retention_transition.rs`; operation and publication evidence remains | In progress in #19 | -| `KEEP-RETENTION-005` | Closure derivation is deterministic, bounded, cycle-safe, fail-closed, and verifies complete blob reconstruction | exact accounting in `closure.md` and one-anchor success in `tests/retention_closure.rs`; property, corruption, and adversarial catalog tests remain | In progress in #19 | +| `KEEP-RETENTION-005` | Closure derivation is deterministic, bounded, cycle-safe, fail-closed, and verifies complete blob reconstruction | exact accounting plus one-anchor and repeated-chunk laws in `tests/retention_closure.rs`; property, corruption, and adversarial catalog tests remain | In progress in #19 | | `KEEP-RETENTION-006` | Publication follows the exact ordered durability protocol, including new namespace-directory admission and retention of fixed-stage evidence until head commit, and returns only after cleanup synchronization | operation-order and `KEEP-CRASH-036..=052` crash-injection tests | Planned in #19 | | `KEEP-RETENTION-007` | Restart resolves every fixed-stage crash prefix to one documented lawful state or typed ambiguity | debug and release crash matrix | Planned in #19 | | `KEEP-RETENTION-008` | Readers double-collect catalog and retention heads and bind one complete catalog, manifest, and root-generation view under a `ReaderFence` | immutable snapshot and concurrency tests | Planned in #19 | diff --git a/src/adapters/retention/closure_error.rs b/src/adapters/retention/closure_error.rs index 34a66d4..11b1fde 100644 --- a/src/adapters/retention/closure_error.rs +++ b/src/adapters/retention/closure_error.rs @@ -4,8 +4,8 @@ use std::error::Error; use std::fmt; use crate::{ - BlobHashError, BlobId, ChunkingError, LayoutDecodeError, LayoutEntryLimitError, LayoutId, - ProfileBoundary, RetentionClosureCounter, SegmentRecordIdentity, StorageProfileId, + BlobHashError, BlobId, ChunkingError, LayoutDecodeError, LayoutId, ProfileBoundary, + RetentionClosureCounter, SegmentRecordIdentity, StorageProfileId, }; /// Failure to derive and authenticate one complete retained root closure. @@ -29,16 +29,6 @@ pub enum RetentionClosureVerificationError { /// Candidate observed value. observed: u64, }, - /// The admitted node limit could not become a host-independent entry cap. - LayoutEntryLimitHostWidth { - /// Admitted node limit that did not fit the layout cap width. - observed: u64, - }, - /// The derived layout entry cap violated the layout protocol bound. - LayoutEntryLimit { - /// Exact layout-bound refusal. - source: LayoutEntryLimitError, - }, /// The pinned catalog omits a first-scheduled closure member. MissingMember { /// Exact missing logical record identity. @@ -106,13 +96,11 @@ pub enum RetentionClosureVerificationError { impl Error for RetentionClosureVerificationError { fn source(&self) -> Option<&(dyn Error + 'static)> { match self { - Self::LayoutEntryLimit { source } => Some(source), Self::LayoutDecode { source, .. } => Some(source), Self::ProfileChunking { source, .. } => Some(source), Self::BlobHash { source, .. } => Some(source), Self::CounterOverflow { .. } | Self::LimitExceeded { .. } - | Self::LayoutEntryLimitHostWidth { .. } | Self::MissingMember { .. } | Self::AnchorTargetMismatch { .. } | Self::ProfileVerifierUnavailable { .. } diff --git a/src/adapters/retention/closure_error_display.rs b/src/adapters/retention/closure_error_display.rs index d36d38a..0aa8672 100644 --- a/src/adapters/retention/closure_error_display.rs +++ b/src/adapters/retention/closure_error_display.rs @@ -25,16 +25,6 @@ pub(super) fn display( formatter, "{counter} limit {maximum} was exceeded by observed value {observed}" ), - RetentionClosureVerificationError::LayoutEntryLimitHostWidth { observed } => write!( - formatter, - "closure node limit {observed} does not fit the layout entry-limit width" - ), - RetentionClosureVerificationError::LayoutEntryLimit { source } => { - write!( - formatter, - "closure-derived layout entry limit is invalid: {source}" - ) - } RetentionClosureVerificationError::MissingMember { identity } => match identity { crate::SegmentRecordIdentity::Chunk(chunk) => write!( formatter, diff --git a/src/adapters/retention/closure_verifier.rs b/src/adapters/retention/closure_verifier.rs index 30d9dc1..31e93f7 100644 --- a/src/adapters/retention/closure_verifier.rs +++ b/src/adapters/retention/closure_verifier.rs @@ -31,8 +31,7 @@ pub fn verify_retention_closure( root: &RetentionRoot, catalog: &CatalogSnapshot<'_, '_, '_>, ) -> Result { - let entry_limit = layout_entry_limit(root)?; - let mut verifier = ClosureVerifier::new(root, catalog, entry_limit); + let mut verifier = ClosureVerifier::new(root, catalog); for anchor in root.anchors().iter().copied() { verifier.verify_anchor(anchor)?; } @@ -42,7 +41,6 @@ pub fn verify_retention_closure( struct ClosureVerifier<'snapshot, 'head, 'catalog, 'records> { root: &'snapshot RetentionRoot, catalog: &'snapshot CatalogSnapshot<'head, 'catalog, 'records>, - entry_limit: LayoutEntryLimit, accounting: ClosureAccounting, records: BTreeMap>, } @@ -51,12 +49,10 @@ impl<'snapshot, 'head, 'catalog, 'records> ClosureVerifier<'snapshot, 'head, 'ca const fn new( root: &'snapshot RetentionRoot, catalog: &'snapshot CatalogSnapshot<'head, 'catalog, 'records>, - entry_limit: LayoutEntryLimit, ) -> Self { Self { root, catalog, - entry_limit, accounting: ClosureAccounting::new(root.limits()), records: BTreeMap::new(), } @@ -75,7 +71,7 @@ impl<'snapshot, 'head, 'catalog, 'records> ClosureVerifier<'snapshot, 'head, 'ca self.accounting .add_encoded(record.header().payload_length().get())?; } - let policy = LayoutDecodePolicy::new(self.entry_limit).with_expected_id(layout_id); + let policy = LayoutDecodePolicy::new(LayoutEntryLimit::MAXIMUM).with_expected_id(layout_id); let layout = AdmittedLayout::decode_record(record.payload(), policy).map_err(|source| { RetentionClosureVerificationError::LayoutDecode { layout: layout_id, @@ -154,17 +150,6 @@ impl<'snapshot, 'head, 'catalog, 'records> ClosureVerifier<'snapshot, 'head, 'ca } } -fn layout_entry_limit( - root: &RetentionRoot, -) -> Result { - let observed = root.limits().nodes(); - let value = u32::try_from(observed).map_err(|_source| { - RetentionClosureVerificationError::LayoutEntryLimitHostWidth { observed } - })?; - LayoutEntryLimit::new(value) - .map_err(|source| RetentionClosureVerificationError::LayoutEntryLimit { source }) -} - fn require_anchor_target( anchor: RetentionAnchor, layout: &AdmittedLayout, diff --git a/tests/retention_closure.rs b/tests/retention_closure.rs index 4a6bf37..ee6dd77 100644 --- a/tests/retention_closure.rs +++ b/tests/retention_closure.rs @@ -1,5 +1,7 @@ //! Deterministic retention-closure verification laws. +#[path = "retention_closure/repeated_chunk_law.rs"] +mod repeated_chunk_law; mod support; use std::error::Error; diff --git a/tests/retention_closure/repeated_chunk_law.rs b/tests/retention_closure/repeated_chunk_law.rs new file mode 100644 index 0000000..7c0985c --- /dev/null +++ b/tests/retention_closure/repeated_chunk_law.rs @@ -0,0 +1,180 @@ +//! Repeated logical chunk accounting law. + +use std::cell::RefCell; +use std::error::Error; +use std::io::{self, Write}; +use std::rc::Rc; + +use keep::{ + AdmittedLayout, AdmittedSegment, AdmittedSegmentRecord, BlobId, CanonicalCatalog, + CanonicalPublicationHead, CatalogGeneration, ChecksummedPublicationHead, FastCdc, + LayoutEntryLimit, RegisteredRetentionProfile, RegisteredStorageProfile, RetentionAnchor, + RetentionClosureLimits, RetentionNamespace, RetentionPolicy, RetentionRoot, RootGeneration, + SegmentReadPolicy, SegmentRecordLimit, SegmentStage, StagedSegment, verify_retention_closure, +}; + +const REPETITIONS: usize = 3; +const RECORD_OVERHEAD: u64 = 144; + +#[test] +fn repeated_chunk_occurrences_consume_physical_bytes_not_unique_nodes() -> Result<(), Box> +{ + let repeated = repeated_source()?; + let source = repeated.bytes; + let spans = repeated.spans; + let blob = BlobId::hash_bytes(&source)?; + let layout = AdmittedLayout::from_spans( + blob, + RegisteredStorageProfile::FAST_CDC_64K_V1, + spans, + LayoutEntryLimit::MAXIMUM, + )?; + let canonical_layout = layout.encode_record()?; + let chunk_length = source + .len() + .checked_div(REPETITIONS) + .ok_or("repetition count is zero")?; + let chunk = source + .get(..chunk_length) + .ok_or("repeated source omits its first chunk")?; + let (stage, probe) = MemoryStage::new(); + let staged = StagedSegment::begin(stage, SegmentRecordLimit::MAXIMUM)?; + let staged = staged.append(AdmittedSegmentRecord::for_chunk(chunk)?)?; + let staged = staged.append(AdmittedSegmentRecord::for_layout(&canonical_layout)?)?; + let _sealed = staged.seal()?; + let segment_bytes = probe.bytes(); + let segment = AdmittedSegment::decode(&segment_bytes, maximum_policy())?; + let segments = [segment]; + let canonical_catalog = + CanonicalCatalog::from_segments(CatalogGeneration::new(1)?, None, &segments)?; + let canonical_head = CanonicalPublicationHead::for_catalog(canonical_catalog.checksummed()); + let catalog = canonical_catalog.checksummed().admit(&segments)?; + let head = ChecksummedPublicationHead::decode(canonical_head.encoded())?; + let snapshot = head.admit(catalog)?; + let encoded_bytes = u64::try_from(canonical_layout.bytes().len())?; + let chunk_bytes = u64::try_from(chunk.len())?; + let repetitions = u64::try_from(REPETITIONS)?; + let physical_bytes = RECORD_OVERHEAD + .checked_add(encoded_bytes) + .and_then(|layout_record| { + RECORD_OVERHEAD + .checked_add(chunk_bytes) + .and_then(|chunk_record| chunk_record.checked_mul(repetitions)) + .and_then(|chunks| layout_record.checked_add(chunks)) + }) + .ok_or("physical-byte oracle overflowed")?; + let root = RetentionRoot::new( + RetentionNamespace::try_from(b"repeated".as_slice())?, + RootGeneration::new(1)?, + RetentionPolicy::new( + RegisteredRetentionProfile::SINGLE_CANONICAL_WITNESS_V1, + RetentionClosureLimits::new(2, 2, encoded_bytes, physical_bytes)?, + ), + None, + vec![RetentionAnchor::new(blob, canonical_layout.id())], + )?; + + let evidence = verify_retention_closure(&root, &snapshot)?; + + assert_eq!(evidence.usage().node_count(), 2); + assert_eq!(evidence.usage().maximum_depth(), 2); + assert_eq!(evidence.usage().encoded_bytes(), encoded_bytes); + assert_eq!(evidence.usage().physical_bytes(), physical_bytes); + Ok(()) +} + +struct RepeatedSource { + bytes: Vec, + spans: Vec, +} + +fn repeated_source() -> Result> { + let candidate_length = usize::try_from( + RegisteredStorageProfile::FAST_CDC_64K_V1 + .maximum_chunk_length() + .get(), + )?; + let candidate = vec![0_u8; candidate_length]; + let first_pass = detect(&candidate)?; + let first = first_pass + .first() + .copied() + .ok_or("profile emitted no candidate chunk")?; + let chunk_length = usize::try_from(first.length().get())?; + let chunk = candidate + .get(..chunk_length) + .ok_or("candidate omits its first emitted chunk")?; + let source_length = chunk_length + .checked_mul(REPETITIONS) + .ok_or("repeated source length overflowed")?; + let mut source = Vec::new(); + source.try_reserve_exact(source_length)?; + for _index in 0..REPETITIONS { + source.extend_from_slice(chunk); + } + let spans = detect(&source)?; + if spans.len() != REPETITIONS || !spans.iter().all(|span| span.id() == first.id()) { + return Err("repeated source did not reproduce one exact chunk identity".into()); + } + Ok(RepeatedSource { + bytes: source, + spans, + }) +} + +fn detect(bytes: &[u8]) -> Result, Box> { + let mut spans = Vec::new(); + let mut detector = FastCdc::new(); + detector.feed(bytes, |span| spans.push(span))?; + if let Some(span) = detector.finish()? { + spans.push(span); + } + Ok(spans) +} + +struct MemoryStage { + bytes: Rc>>, +} + +struct MemoryProbe { + bytes: Rc>>, +} + +impl MemoryStage { + fn new() -> (Self, MemoryProbe) { + let bytes = Rc::new(RefCell::new(Vec::new())); + ( + Self { + bytes: Rc::clone(&bytes), + }, + MemoryProbe { bytes }, + ) + } +} + +impl Write for MemoryStage { + fn write(&mut self, bytes: &[u8]) -> io::Result { + self.bytes.borrow_mut().extend_from_slice(bytes); + Ok(bytes.len()) + } + + fn flush(&mut self) -> io::Result<()> { + Ok(()) + } +} + +impl SegmentStage for MemoryStage { + fn synchronize(&mut self) -> io::Result<()> { + Ok(()) + } +} + +impl MemoryProbe { + fn bytes(&self) -> Vec { + self.bytes.borrow().clone() + } +} + +const fn maximum_policy() -> SegmentReadPolicy { + SegmentReadPolicy::new(SegmentRecordLimit::MAXIMUM, LayoutEntryLimit::MAXIMUM) +} From 163cfa1a6227e1be8c494ff9b9c988f0476f8a9c Mon Sep 17 00:00:00 2001 From: James Ross Date: Wed, 29 Jul 2026 22:36:49 -0700 Subject: [PATCH 016/111] Test: Prove closure refusal precedence --- docs/formats/segment-store-v2/closure.md | 17 +++ docs/formats/segment-store-v2/requirements.md | 2 +- tests/retention_closure.rs | 8 ++ .../adversarial_catalog_laws.rs | 116 ++++++++++++++++++ .../limit_precedence_laws.rs | 64 ++++++++++ tests/retention_closure/memory_stage.rs | 49 ++++++++ tests/retention_closure/one_zero_bundle.rs | 63 ++++++++++ tests/retention_closure/repeated_chunk_law.rs | 61 ++------- 8 files changed, 326 insertions(+), 54 deletions(-) create mode 100644 tests/retention_closure/adversarial_catalog_laws.rs create mode 100644 tests/retention_closure/limit_precedence_laws.rs create mode 100644 tests/retention_closure/memory_stage.rs create mode 100644 tests/retention_closure/one_zero_bundle.rs diff --git a/docs/formats/segment-store-v2/closure.md b/docs/formats/segment-store-v2/closure.md index 0f88d6c..599f4dc 100644 --- a/docs/formats/segment-store-v2/closure.md +++ b/docs/formats/segment-store-v2/closure.md @@ -169,6 +169,23 @@ The digest binds the exact profile, catalog, observed resource use, logical member set, and record checksums. The transition receipt binds it beside the root's separate anchor-set digest; neither digest substitutes for the other. +## Executable evidence + +- The [one-anchor closure law](../../../tests/retention_closure.rs) freezes the + exact counters, canonical member transcript, closure digest, and authenticated + reconstruction result. +- The + [repeated-chunk law](../../../tests/retention_closure/repeated_chunk_law.rs) + proves that logical reconstruction work and unique-node evidence remain + separate. +- The + [adversarial-catalog laws](../../../tests/retention_closure/adversarial_catalog_laws.rs) + prove exact missing-member refusal and target-mismatch precedence. +- The + [limit-precedence laws](../../../tests/retention_closure/limit_precedence_laws.rs) + prove the documented depth, node, physical-byte, and encoded-byte refusal + order. + ## Evidence and nonclaims Successful evidence records the closure digest, all four observed counters, diff --git a/docs/formats/segment-store-v2/requirements.md b/docs/formats/segment-store-v2/requirements.md index ccd9c81..708271a 100644 --- a/docs/formats/segment-store-v2/requirements.md +++ b/docs/formats/segment-store-v2/requirements.md @@ -13,7 +13,7 @@ case is not evidence. | `KEEP-RETENTION-002` | Root, manifest, and head codecs implement the exact canonical grammars and fixed bounds | independent golden corpus plus `tests/retention_root_encoding.rs`, `tests/retention_root_decoding.rs`, `tests/retention_manifest_codec.rs`, and `tests/retention_head_codec.rs` | Implemented | | `KEEP-RETENTION-003` | Every structural field, truncation boundary, ordering law, duplicate, overflow, flag, reserved byte, digest, checksum, and trailing byte has a precise refusal | corruption matrix and fuzz | Planned in #19 | | `KEEP-RETENTION-004` | Retain and release compare expected and observed generations and publish exact successors only | storage-independent planning in `tests/retention_transition.rs`; operation and publication evidence remains | In progress in #19 | -| `KEEP-RETENTION-005` | Closure derivation is deterministic, bounded, cycle-safe, fail-closed, and verifies complete blob reconstruction | exact accounting plus one-anchor and repeated-chunk laws in `tests/retention_closure.rs`; property, corruption, and adversarial catalog tests remain | In progress in #19 | +| `KEEP-RETENTION-005` | Closure derivation is deterministic, bounded, cycle-safe, fail-closed, and verifies complete blob reconstruction | exact accounting plus one-anchor, repeated-chunk, and adversarial-catalog laws in `tests/retention_closure.rs`; property and corruption tests remain | In progress in #19 | | `KEEP-RETENTION-006` | Publication follows the exact ordered durability protocol, including new namespace-directory admission and retention of fixed-stage evidence until head commit, and returns only after cleanup synchronization | operation-order and `KEEP-CRASH-036..=052` crash-injection tests | Planned in #19 | | `KEEP-RETENTION-007` | Restart resolves every fixed-stage crash prefix to one documented lawful state or typed ambiguity | debug and release crash matrix | Planned in #19 | | `KEEP-RETENTION-008` | Readers double-collect catalog and retention heads and bind one complete catalog, manifest, and root-generation view under a `ReaderFence` | immutable snapshot and concurrency tests | Planned in #19 | diff --git a/tests/retention_closure.rs b/tests/retention_closure.rs index ee6dd77..7510d11 100644 --- a/tests/retention_closure.rs +++ b/tests/retention_closure.rs @@ -1,5 +1,13 @@ //! Deterministic retention-closure verification laws. +#[path = "retention_closure/adversarial_catalog_laws.rs"] +mod adversarial_catalog_laws; +#[path = "retention_closure/limit_precedence_laws.rs"] +mod limit_precedence_laws; +#[path = "retention_closure/memory_stage.rs"] +mod memory_stage; +#[path = "retention_closure/one_zero_bundle.rs"] +mod one_zero_bundle; #[path = "retention_closure/repeated_chunk_law.rs"] mod repeated_chunk_law; mod support; diff --git a/tests/retention_closure/adversarial_catalog_laws.rs b/tests/retention_closure/adversarial_catalog_laws.rs new file mode 100644 index 0000000..9f89dc8 --- /dev/null +++ b/tests/retention_closure/adversarial_catalog_laws.rs @@ -0,0 +1,116 @@ +//! Adversarial catalog and first-refusal ordering laws. + +use std::error::Error; + +use keep::{ + AdmittedLayout, AdmittedSegment, AdmittedSegmentRecord, BlobId, CanonicalCatalog, + CanonicalPublicationHead, CatalogGeneration, ChecksummedPublicationHead, LayoutDecodePolicy, + LayoutEntryLimit, RetentionClosureLimits, RetentionClosureVerificationError, RetentionRoot, + SegmentRecordIdentity, VerifiedRetentionClosure, verify_retention_closure, +}; + +use super::{ + ONE_ZERO_BLOB, maximum_policy, + memory_stage::segment_bytes, + one_zero_bundle::{root_with_limits, verify_fixture}, + support::{layout_record_bytes, require_error}, +}; + +const CHUNK_CATALOG_HEX: &str = + include_str!("../../conformance/segment-store/v1/one-zero-catalog.hex"); +const CHUNK_HEAD_HEX: &str = include_str!("../../conformance/segment-store/v1/one-zero-head.hex"); +const CHUNK_SEGMENT_HEX: &str = + include_str!("../../conformance/segment-store/v1/one-zero-segment.hex"); + +#[test] +fn missing_layout_is_an_exact_first_scheduled_member_refusal() -> Result<(), Box> { + let root = root_with_limits(RetentionClosureLimits::new(2, 2, 220, 509)?, None)?; + let error = require_error( + verify_fixture(&root, CHUNK_SEGMENT_HEX, CHUNK_CATALOG_HEX, CHUNK_HEAD_HEX)?, + "chunk-only catalog unexpectedly satisfied a retained layout", + )?; + let expected = root + .anchors() + .first() + .copied() + .ok_or("retention root omits its required anchor")? + .layout_id(); + + assert!(matches!( + error, + RetentionClosureVerificationError::MissingMember { + identity: SegmentRecordIdentity::Layout(layout) + } if layout == expected + )); + Ok(()) +} + +#[test] +fn missing_chunk_is_an_exact_logical_occurrence_refusal() -> Result<(), Box> { + let root = root_with_limits(RetentionClosureLimits::new(2, 2, 220, 509)?, None)?; + let error = require_error( + verify_layout_only(&root)?, + "layout-only catalog unexpectedly reconstructed a retained blob", + )?; + let expected = expected_chunk_identity()?; + + assert!(matches!( + error, + RetentionClosureVerificationError::MissingMember { identity } if identity == expected + )); + Ok(()) +} + +#[test] +fn anchor_target_mismatch_precedes_chunk_traversal() -> Result<(), Box> { + let expected = BlobId::hash_bytes(b"adversarial anchor")?; + let root = root_with_limits(RetentionClosureLimits::new(2, 2, 220, 509)?, Some(expected))?; + let error = require_error( + verify_layout_only(&root)?, + "mismatched anchor target unexpectedly verified", + )?; + let observed: BlobId = ONE_ZERO_BLOB.parse()?; + + assert!(matches!( + error, + RetentionClosureVerificationError::AnchorTargetMismatch { + expected: actual_expected, + observed: actual_observed, + .. + } if actual_expected == expected && actual_observed == observed + )); + Ok(()) +} + +fn verify_layout_only( + root: &RetentionRoot, +) -> Result, Box> { + let layout = one_zero_layout()?; + let canonical_layout = layout.encode_record()?; + let records = [AdmittedSegmentRecord::for_layout(&canonical_layout)?]; + let segment_bytes = segment_bytes(&records)?; + let segment = AdmittedSegment::decode(&segment_bytes, maximum_policy())?; + let segments = [segment]; + let catalog = CanonicalCatalog::from_segments(CatalogGeneration::new(1)?, None, &segments)?; + let head = CanonicalPublicationHead::for_catalog(catalog.checksummed()); + let admitted = catalog.checksummed().admit(&segments)?; + let snapshot = ChecksummedPublicationHead::decode(head.encoded())?.admit(admitted)?; + Ok(verify_retention_closure(root, &snapshot)) +} + +fn expected_chunk_identity() -> Result> { + let layout = one_zero_layout()?; + let entry = layout + .entries() + .first() + .copied() + .ok_or("one-zero layout omits its chunk entry")?; + Ok(SegmentRecordIdentity::Chunk(entry.chunk_id())) +} + +fn one_zero_layout() -> Result> { + Ok(AdmittedLayout::decode_record( + &layout_record_bytes("one-zero")?, + LayoutDecodePolicy::new(LayoutEntryLimit::MAXIMUM), + )?) +} diff --git a/tests/retention_closure/limit_precedence_laws.rs b/tests/retention_closure/limit_precedence_laws.rs new file mode 100644 index 0000000..e11ee93 --- /dev/null +++ b/tests/retention_closure/limit_precedence_laws.rs @@ -0,0 +1,64 @@ +//! Closure resource-limit precedence laws. + +use std::error::Error; + +use keep::{ + RetentionClosureCounter, RetentionClosureLimits, RetentionClosureVerificationError, + RetentionRoot, +}; + +use super::{ + one_zero_bundle::{root_with_limits, verify_bundle}, + support::require_error, +}; + +#[test] +fn depth_refusal_precedes_second_node_admission() -> Result<(), Box> { + let root = root_with_limits(RetentionClosureLimits::new(2, 1, 220, 509)?, None)?; + + assert_limit(&root, RetentionClosureCounter::Depth, 1, 2) +} + +#[test] +fn node_refusal_follows_depth_admission_and_precedes_chunk_lookup() -> Result<(), Box> { + let root = root_with_limits(RetentionClosureLimits::new(1, 2, 220, 509)?, None)?; + + assert_limit(&root, RetentionClosureCounter::Nodes, 1, 2) +} + +#[test] +fn physical_byte_refusal_precedes_layout_decoding() -> Result<(), Box> { + let root = root_with_limits(RetentionClosureLimits::new(2, 2, 220, 363)?, None)?; + + assert_limit(&root, RetentionClosureCounter::PhysicalBytes, 363, 364) +} + +#[test] +fn encoded_byte_refusal_follows_layout_record_charge() -> Result<(), Box> { + let root = root_with_limits(RetentionClosureLimits::new(2, 2, 219, 509)?, None)?; + + assert_limit(&root, RetentionClosureCounter::EncodedBytes, 219, 220) +} + +fn assert_limit( + root: &RetentionRoot, + counter: RetentionClosureCounter, + maximum: u64, + observed: u64, +) -> Result<(), Box> { + let error = require_error( + verify_bundle(root)?, + "resource-constrained closure unexpectedly verified", + )?; + assert!(matches!( + error, + RetentionClosureVerificationError::LimitExceeded { + counter: actual_counter, + maximum: actual_maximum, + observed: actual_observed, + } if actual_counter == counter + && actual_maximum == maximum + && actual_observed == observed + )); + Ok(()) +} diff --git a/tests/retention_closure/memory_stage.rs b/tests/retention_closure/memory_stage.rs new file mode 100644 index 0000000..e0bde89 --- /dev/null +++ b/tests/retention_closure/memory_stage.rs @@ -0,0 +1,49 @@ +//! In-memory segment-stage support for closure integration laws. +#![allow( + clippy::redundant_pub_crate, + reason = "private integration-test siblings share this segment fixture" +)] + +use std::cell::RefCell; +use std::io::{self, Write}; +use std::rc::Rc; + +use keep::{ + AdmittedSegmentRecord, SegmentRecordLimit, SegmentStage, SegmentWriteError, StagedSegment, +}; + +pub(super) fn segment_bytes( + records: &[AdmittedSegmentRecord<'_>], +) -> Result, SegmentWriteError> { + let bytes = Rc::new(RefCell::new(Vec::new())); + let stage = MemoryStage { + bytes: Rc::clone(&bytes), + }; + let mut staged = StagedSegment::begin(stage, SegmentRecordLimit::MAXIMUM)?; + for record in records { + staged = staged.append(*record)?; + } + let _sealed = staged.seal()?; + Ok(bytes.borrow().clone()) +} + +struct MemoryStage { + bytes: Rc>>, +} + +impl Write for MemoryStage { + fn write(&mut self, bytes: &[u8]) -> io::Result { + self.bytes.borrow_mut().extend_from_slice(bytes); + Ok(bytes.len()) + } + + fn flush(&mut self) -> io::Result<()> { + Ok(()) + } +} + +impl SegmentStage for MemoryStage { + fn synchronize(&mut self) -> io::Result<()> { + Ok(()) + } +} diff --git a/tests/retention_closure/one_zero_bundle.rs b/tests/retention_closure/one_zero_bundle.rs new file mode 100644 index 0000000..c6267ca --- /dev/null +++ b/tests/retention_closure/one_zero_bundle.rs @@ -0,0 +1,63 @@ +//! One-zero closure fixture construction and verification. +#![allow( + clippy::redundant_pub_crate, + reason = "private integration-test siblings share this closure fixture" +)] + +use std::error::Error; + +use keep::{ + BlobId, RetentionAnchor, RetentionClosureLimits, RetentionClosureVerificationError, + RetentionNamespace, RetentionPolicy, RetentionRoot, RootGeneration, VerifiedRetentionClosure, + verify_retention_closure, +}; + +use super::{ + BUNDLE_CATALOG_HEX, BUNDLE_HEAD_HEX, BUNDLE_SEGMENT_HEX, ONE_ZERO_BLOB, ONE_ZERO_LAYOUT, + admitted_catalog, fixture, maximum_policy, +}; + +pub(super) fn root_with_limits( + limits: RetentionClosureLimits, + target: Option, +) -> Result> { + let blob = target.map_or_else(|| ONE_ZERO_BLOB.parse(), Ok)?; + Ok(RetentionRoot::new( + RetentionNamespace::try_from(b"adversarial".as_slice())?, + RootGeneration::new(1)?, + RetentionPolicy::new( + keep::RegisteredRetentionProfile::SINGLE_CANONICAL_WITNESS_V1, + limits, + ), + None, + vec![RetentionAnchor::new(blob, ONE_ZERO_LAYOUT.parse()?)], + )?) +} + +pub(super) fn verify_bundle( + root: &RetentionRoot, +) -> Result, Box> { + verify_fixture( + root, + BUNDLE_SEGMENT_HEX, + BUNDLE_CATALOG_HEX, + BUNDLE_HEAD_HEX, + ) +} + +pub(super) fn verify_fixture( + root: &RetentionRoot, + segment_hex: &str, + catalog_hex: &str, + head_hex: &str, +) -> Result, Box> { + let segment_bytes = fixture(segment_hex)?; + let catalog_bytes = fixture(catalog_hex)?; + let head_bytes = fixture(head_hex)?; + let segment = keep::AdmittedSegment::decode(&segment_bytes, maximum_policy())?; + let segments = [segment]; + let catalog = admitted_catalog(&catalog_bytes, &segments)?; + let head = keep::ChecksummedPublicationHead::decode(&head_bytes)?; + let snapshot = head.admit(catalog)?; + Ok(verify_retention_closure(root, &snapshot)) +} diff --git a/tests/retention_closure/repeated_chunk_law.rs b/tests/retention_closure/repeated_chunk_law.rs index 7c0985c..ba06dc1 100644 --- a/tests/retention_closure/repeated_chunk_law.rs +++ b/tests/retention_closure/repeated_chunk_law.rs @@ -1,18 +1,17 @@ //! Repeated logical chunk accounting law. -use std::cell::RefCell; use std::error::Error; -use std::io::{self, Write}; -use std::rc::Rc; use keep::{ AdmittedLayout, AdmittedSegment, AdmittedSegmentRecord, BlobId, CanonicalCatalog, CanonicalPublicationHead, CatalogGeneration, ChecksummedPublicationHead, FastCdc, LayoutEntryLimit, RegisteredRetentionProfile, RegisteredStorageProfile, RetentionAnchor, RetentionClosureLimits, RetentionNamespace, RetentionPolicy, RetentionRoot, RootGeneration, - SegmentReadPolicy, SegmentRecordLimit, SegmentStage, StagedSegment, verify_retention_closure, + SegmentReadPolicy, SegmentRecordLimit, verify_retention_closure, }; +use super::memory_stage::segment_bytes; + const REPETITIONS: usize = 3; const RECORD_OVERHEAD: u64 = 144; @@ -37,12 +36,11 @@ fn repeated_chunk_occurrences_consume_physical_bytes_not_unique_nodes() -> Resul let chunk = source .get(..chunk_length) .ok_or("repeated source omits its first chunk")?; - let (stage, probe) = MemoryStage::new(); - let staged = StagedSegment::begin(stage, SegmentRecordLimit::MAXIMUM)?; - let staged = staged.append(AdmittedSegmentRecord::for_chunk(chunk)?)?; - let staged = staged.append(AdmittedSegmentRecord::for_layout(&canonical_layout)?)?; - let _sealed = staged.seal()?; - let segment_bytes = probe.bytes(); + let records = [ + AdmittedSegmentRecord::for_chunk(chunk)?, + AdmittedSegmentRecord::for_layout(&canonical_layout)?, + ]; + let segment_bytes = segment_bytes(&records)?; let segment = AdmittedSegment::decode(&segment_bytes, maximum_policy())?; let segments = [segment]; let canonical_catalog = @@ -132,49 +130,6 @@ fn detect(bytes: &[u8]) -> Result, Box> { Ok(spans) } -struct MemoryStage { - bytes: Rc>>, -} - -struct MemoryProbe { - bytes: Rc>>, -} - -impl MemoryStage { - fn new() -> (Self, MemoryProbe) { - let bytes = Rc::new(RefCell::new(Vec::new())); - ( - Self { - bytes: Rc::clone(&bytes), - }, - MemoryProbe { bytes }, - ) - } -} - -impl Write for MemoryStage { - fn write(&mut self, bytes: &[u8]) -> io::Result { - self.bytes.borrow_mut().extend_from_slice(bytes); - Ok(bytes.len()) - } - - fn flush(&mut self) -> io::Result<()> { - Ok(()) - } -} - -impl SegmentStage for MemoryStage { - fn synchronize(&mut self) -> io::Result<()> { - Ok(()) - } -} - -impl MemoryProbe { - fn bytes(&self) -> Vec { - self.bytes.borrow().clone() - } -} - const fn maximum_policy() -> SegmentReadPolicy { SegmentReadPolicy::new(SegmentRecordLimit::MAXIMUM, LayoutEntryLimit::MAXIMUM) } From d9b1771849ba35ba49b2f4847fc5f02bd27abba4 Mon Sep 17 00:00:00 2001 From: James Ross Date: Wed, 29 Jul 2026 22:40:51 -0700 Subject: [PATCH 017/111] Test: Model closure resource boundaries --- docs/formats/segment-store-v2/closure.md | 28 ++--- docs/formats/segment-store-v2/requirements.md | 2 +- tests/retention_closure.rs | 2 + tests/retention_closure/closure_model_laws.rs | 102 ++++++++++++++++++ 4 files changed, 119 insertions(+), 15 deletions(-) create mode 100644 tests/retention_closure/closure_model_laws.rs diff --git a/docs/formats/segment-store-v2/closure.md b/docs/formats/segment-store-v2/closure.md index 599f4dc..b4bb152 100644 --- a/docs/formats/segment-store-v2/closure.md +++ b/docs/formats/segment-store-v2/closure.md @@ -171,20 +171,20 @@ root's separate anchor-set digest; neither digest substitutes for the other. ## Executable evidence -- The [one-anchor closure law](../../../tests/retention_closure.rs) freezes the - exact counters, canonical member transcript, closure digest, and authenticated - reconstruction result. -- The - [repeated-chunk law](../../../tests/retention_closure/repeated_chunk_law.rs) - proves that logical reconstruction work and unique-node evidence remain - separate. -- The - [adversarial-catalog laws](../../../tests/retention_closure/adversarial_catalog_laws.rs) - prove exact missing-member refusal and target-mismatch precedence. -- The - [limit-precedence laws](../../../tests/retention_closure/limit_precedence_laws.rs) - prove the documented depth, node, physical-byte, and encoded-byte refusal - order. +- The [one-anchor law](../../../tests/retention_closure.rs) freezes exact + counters, the member transcript, the digest, and reconstruction. +- The [repeated-chunk + law](../../../tests/retention_closure/repeated_chunk_law.rs) separates + reconstruction work from unique-node evidence. +- The [adversarial-catalog + laws](../../../tests/retention_closure/adversarial_catalog_laws.rs) prove exact + missing-member and target-mismatch precedence. +- The [limit-precedence + laws](../../../tests/retention_closure/limit_precedence_laws.rs) prove the + depth, node, physical-byte, and encoded-byte refusal order. +- The [closure model + laws](../../../tests/retention_closure/closure_model_laws.rs) compare all + `3 × 3 × 3 × 5 = 135` one-zero boundary policies with a boring model. ## Evidence and nonclaims diff --git a/docs/formats/segment-store-v2/requirements.md b/docs/formats/segment-store-v2/requirements.md index 708271a..5dda56e 100644 --- a/docs/formats/segment-store-v2/requirements.md +++ b/docs/formats/segment-store-v2/requirements.md @@ -13,7 +13,7 @@ case is not evidence. | `KEEP-RETENTION-002` | Root, manifest, and head codecs implement the exact canonical grammars and fixed bounds | independent golden corpus plus `tests/retention_root_encoding.rs`, `tests/retention_root_decoding.rs`, `tests/retention_manifest_codec.rs`, and `tests/retention_head_codec.rs` | Implemented | | `KEEP-RETENTION-003` | Every structural field, truncation boundary, ordering law, duplicate, overflow, flag, reserved byte, digest, checksum, and trailing byte has a precise refusal | corruption matrix and fuzz | Planned in #19 | | `KEEP-RETENTION-004` | Retain and release compare expected and observed generations and publish exact successors only | storage-independent planning in `tests/retention_transition.rs`; operation and publication evidence remains | In progress in #19 | -| `KEEP-RETENTION-005` | Closure derivation is deterministic, bounded, cycle-safe, fail-closed, and verifies complete blob reconstruction | exact accounting plus one-anchor, repeated-chunk, and adversarial-catalog laws in `tests/retention_closure.rs`; property and corruption tests remain | In progress in #19 | +| `KEEP-RETENTION-005` | Closure derivation is deterministic, bounded, cycle-safe, fail-closed, and verifies complete blob reconstruction | exact accounting plus one-anchor, repeated-chunk, adversarial-catalog, and exhaustive model laws in `tests/retention_closure.rs`; corruption tests remain | In progress in #19 | | `KEEP-RETENTION-006` | Publication follows the exact ordered durability protocol, including new namespace-directory admission and retention of fixed-stage evidence until head commit, and returns only after cleanup synchronization | operation-order and `KEEP-CRASH-036..=052` crash-injection tests | Planned in #19 | | `KEEP-RETENTION-007` | Restart resolves every fixed-stage crash prefix to one documented lawful state or typed ambiguity | debug and release crash matrix | Planned in #19 | | `KEEP-RETENTION-008` | Readers double-collect catalog and retention heads and bind one complete catalog, manifest, and root-generation view under a `ReaderFence` | immutable snapshot and concurrency tests | Planned in #19 | diff --git a/tests/retention_closure.rs b/tests/retention_closure.rs index 7510d11..0ed1b3d 100644 --- a/tests/retention_closure.rs +++ b/tests/retention_closure.rs @@ -2,6 +2,8 @@ #[path = "retention_closure/adversarial_catalog_laws.rs"] mod adversarial_catalog_laws; +#[path = "retention_closure/closure_model_laws.rs"] +mod closure_model_laws; #[path = "retention_closure/limit_precedence_laws.rs"] mod limit_precedence_laws; #[path = "retention_closure/memory_stage.rs"] diff --git a/tests/retention_closure/closure_model_laws.rs b/tests/retention_closure/closure_model_laws.rs new file mode 100644 index 0000000..12fc915 --- /dev/null +++ b/tests/retention_closure/closure_model_laws.rs @@ -0,0 +1,102 @@ +//! Exhaustive closure-accounting outcomes against a boring model. + +use std::error::Error; + +use keep::{RetentionClosureCounter, RetentionClosureLimits, RetentionClosureVerificationError}; + +use super::one_zero_bundle::{root_with_limits, verify_bundle}; + +const NODE_LIMITS: [u64; 3] = [1, 2, 3]; +const DEPTH_LIMITS: [u16; 3] = [1, 2, 3]; +const ENCODED_LIMITS: [u64; 3] = [219, 220, 221]; +const PHYSICAL_LIMITS: [u64; 5] = [363, 364, 508, 509, 510]; + +#[test] +fn exhaustive_boundary_policies_agree_with_the_boring_model() -> Result<(), Box> { + for nodes in NODE_LIMITS { + for depth in DEPTH_LIMITS { + for encoded in ENCODED_LIMITS { + for physical in PHYSICAL_LIMITS { + let limits = RetentionClosureLimits::new(nodes, depth, encoded, physical)?; + let root = root_with_limits(limits, None)?; + let observed = classify(verify_bundle(&root)?)?; + let expected = model(nodes, depth, encoded, physical); + + assert_eq!( + observed, expected, + "nodes={nodes} depth={depth} encoded={encoded} physical={physical}" + ); + } + } + } + } + Ok(()) +} + +fn model(nodes: u64, depth: u16, encoded: u64, physical: u64) -> Outcome { + if physical < 364 { + return Outcome::limit(RetentionClosureCounter::PhysicalBytes, physical, 364); + } + if encoded < 220 { + return Outcome::limit(RetentionClosureCounter::EncodedBytes, encoded, 220); + } + if depth < 2 { + return Outcome::limit(RetentionClosureCounter::Depth, u64::from(depth), 2); + } + if nodes < 2 { + return Outcome::limit(RetentionClosureCounter::Nodes, nodes, 2); + } + if physical < 509 { + return Outcome::limit(RetentionClosureCounter::PhysicalBytes, physical, 509); + } + Outcome::Verified { + nodes: 2, + depth: 2, + encoded: 220, + physical: 509, + } +} + +fn classify( + result: Result, +) -> Result> { + match result { + Ok(evidence) => Ok(Outcome::Verified { + nodes: evidence.usage().node_count(), + depth: evidence.usage().maximum_depth(), + encoded: evidence.usage().encoded_bytes(), + physical: evidence.usage().physical_bytes(), + }), + Err(RetentionClosureVerificationError::LimitExceeded { + counter, + maximum, + observed, + }) => Ok(Outcome::limit(counter, maximum, observed)), + Err(error) => Err(error.into()), + } +} + +#[derive(Debug, Eq, PartialEq)] +enum Outcome { + Limit { + counter: RetentionClosureCounter, + maximum: u64, + observed: u64, + }, + Verified { + nodes: u64, + depth: u16, + encoded: u64, + physical: u64, + }, +} + +impl Outcome { + const fn limit(counter: RetentionClosureCounter, maximum: u64, observed: u64) -> Self { + Self::Limit { + counter, + maximum, + observed, + } + } +} From e44bfd78a4d1e4ce40fba063573fd0c945e11b3d Mon Sep 17 00:00:00 2001 From: James Ross Date: Wed, 29 Jul 2026 22:47:42 -0700 Subject: [PATCH 018/111] Docs: Define closure corruption boundary --- docs/formats/segment-store-v2/README.md | 2 + .../segment-store-v2/closure-corruption.md | 69 +++++++++++ docs/formats/segment-store-v2/closure.md | 8 +- docs/formats/segment-store-v2/requirements.md | 2 +- .../retention_store_v2_protocol_contract.rs | 111 ++---------------- .../closure_contract_laws.rs | 42 +++++++ .../migration_contract_laws.rs | 76 ++++++++++++ 7 files changed, 201 insertions(+), 109 deletions(-) create mode 100644 docs/formats/segment-store-v2/closure-corruption.md create mode 100644 xtask/tests/retention_store_v2_protocol_contract/closure_contract_laws.rs create mode 100644 xtask/tests/retention_store_v2_protocol_contract/migration_contract_laws.rs diff --git a/docs/formats/segment-store-v2/README.md b/docs/formats/segment-store-v2/README.md index 8ba96bf..103e379 100644 --- a/docs/formats/segment-store-v2/README.md +++ b/docs/formats/segment-store-v2/README.md @@ -42,6 +42,8 @@ The following pages form one protocol: root-generation, manifest, retention-head, and transition rules. - [Closure verification](closure.md) owns deterministic traversal, exact resource accounting, authenticated reconstruction, and closure evidence. +- [Closure corruption boundary](closure-corruption.md) owns the admitted-record + ingress proof and its exact refusal evidence. - [GC and disposition records](gc.md) owns the canonical planned intent, completion, and recovery-disposition byte grammars. - [Migration and recovery](recovery.md) owns the exact root namespace, diff --git a/docs/formats/segment-store-v2/closure-corruption.md b/docs/formats/segment-store-v2/closure-corruption.md new file mode 100644 index 0000000..8e31924 --- /dev/null +++ b/docs/formats/segment-store-v2/closure-corruption.md @@ -0,0 +1,69 @@ +# Closure Corruption Boundary + +- Status: Normative version-2 protocol; executable ingress evidence implemented +- Format coordinate: `keep.segment-store/v2` +- Requirement: [`KEEP-RETENTION-005`](requirements.md#retention-transitions) +- Parent contract: [Closure verification](closure.md) + +This page defines where corrupt closure-member bytes refuse. Its primary job is +to keep untrusted byte admission separate from deterministic closure traversal. + +## Trust boundary + +`verify_retention_closure` accepts a validated `RetentionRoot` and an immutable +`CatalogSnapshot`. It does not accept untrusted bytes, raw segment records, +paths, readers, or caller lookup callbacks. + +A record reaches that snapshot only through this proof chain: + +1. `ChecksummedSegmentRecord::decode` admits exact framing and its checksum. +2. `ChecksummedSegmentRecord::admit` recomputes the chunk or layout identity + from the payload and returns an `AdmittedSegmentRecord`. +3. `AdmittedSegment::decode` admits every complete nested record and the + segment seal and digest. +4. `ChecksummedCatalog::admit` binds each catalog entry to the exact admitted + record identity, checksum, and top-level location. +5. `ChecksummedPublicationHead::admit` binds the admitted catalog's generation, + length, and digest into a `CatalogSnapshot`. + +Failure at any step makes the next type unconstructible through the public API. +Closure verification therefore has no corruption fallback and never +reinterprets a malformed record as a missing member. + +## Exact refusal ownership + +The inherited version-1 boundaries retain their typed errors: + +- malformed record framing or checksum returns `SegmentRecordDecodeError`; +- chunk payload identity disagreement or malformed layout payload returns + `SegmentRecordAdmissionError`; +- complete-segment corruption returns `SegmentReadError`; +- catalog location, identity, checksum, or segment disagreement returns + `CatalogAdmissionError`; and +- publication-head disagreement returns `CatalogSnapshotError`. + +`RetentionClosureVerificationError::MissingMember` means the pinned, +fully admitted catalog has no binding for the scheduled logical identity. It +does not mean bytes were present but corrupt. + +## Executable evidence + +- The [segment-record framing + laws](../../../tests/segment_record/framing_laws.rs) cover checksum and + framing corruption. +- The [segment-record admission + laws](../../../tests/segment_record/admission_laws.rs) cover content-valid + checksums whose chunk or layout payload does not match its declared identity. +- The [segment corruption-localization + laws](../../../tests/segment/identity_laws.rs) prove record refusal precedes + the outer segment digest. +- The [`segment_format` fuzz + target](../../../fuzz/fuzz_targets/segment_format.rs) reaches record decoding, + record admission, and complete-segment admission from deterministic canonical + seeds owned by the [Rust seed-corpus + task](../../../xtask/src/fuzz_seed_corpus/segment_seeds.rs). + +These proofs establish ingress safety. They do not prove that a future +retention publication adapter preserves the original source chain when it maps +these failures into an operation-level error; that obligation remains with +`KEEP-RETENTION-006`. diff --git a/docs/formats/segment-store-v2/closure.md b/docs/formats/segment-store-v2/closure.md index b4bb152..6e1f0e1 100644 --- a/docs/formats/segment-store-v2/closure.md +++ b/docs/formats/segment-store-v2/closure.md @@ -28,10 +28,10 @@ read paths, enumerate a filesystem, consult a clock, invoke a caller callback, or replace a missing witness. Version 2 selects the single record bound to each logical identity by the pinned catalog. -The catalog has already admitted each bound segment record's framing, checksum, -logical identity, and payload. Closure verification consumes those proofs, -decodes layouts again under the closure budget, and authenticates each complete -logical blob. +The [corruption boundary](closure-corruption.md) defines how each bound record +earns framing, checksum, logical-identity, and payload proofs before it enters a +`CatalogSnapshot`. Closure verification consumes those proofs, decodes layouts +under the closure budget, and authenticates each complete logical blob. ## Deterministic traversal diff --git a/docs/formats/segment-store-v2/requirements.md b/docs/formats/segment-store-v2/requirements.md index 5dda56e..62419a8 100644 --- a/docs/formats/segment-store-v2/requirements.md +++ b/docs/formats/segment-store-v2/requirements.md @@ -13,7 +13,7 @@ case is not evidence. | `KEEP-RETENTION-002` | Root, manifest, and head codecs implement the exact canonical grammars and fixed bounds | independent golden corpus plus `tests/retention_root_encoding.rs`, `tests/retention_root_decoding.rs`, `tests/retention_manifest_codec.rs`, and `tests/retention_head_codec.rs` | Implemented | | `KEEP-RETENTION-003` | Every structural field, truncation boundary, ordering law, duplicate, overflow, flag, reserved byte, digest, checksum, and trailing byte has a precise refusal | corruption matrix and fuzz | Planned in #19 | | `KEEP-RETENTION-004` | Retain and release compare expected and observed generations and publish exact successors only | storage-independent planning in `tests/retention_transition.rs`; operation and publication evidence remains | In progress in #19 | -| `KEEP-RETENTION-005` | Closure derivation is deterministic, bounded, cycle-safe, fail-closed, and verifies complete blob reconstruction | exact accounting plus one-anchor, repeated-chunk, adversarial-catalog, and exhaustive model laws in `tests/retention_closure.rs`; corruption tests remain | In progress in #19 | +| `KEEP-RETENTION-005` | Closure derivation is deterministic, bounded, cycle-safe, fail-closed, and verifies complete blob reconstruction | exact accounting, reconstruction, adversarial-catalog, and exhaustive model laws in `tests/retention_closure.rs`; corrupt members refuse through the inherited segment-record admission laws and seeded `segment_format` fuzz target routed by `closure-corruption.md` | Implemented | | `KEEP-RETENTION-006` | Publication follows the exact ordered durability protocol, including new namespace-directory admission and retention of fixed-stage evidence until head commit, and returns only after cleanup synchronization | operation-order and `KEEP-CRASH-036..=052` crash-injection tests | Planned in #19 | | `KEEP-RETENTION-007` | Restart resolves every fixed-stage crash prefix to one documented lawful state or typed ambiguity | debug and release crash matrix | Planned in #19 | | `KEEP-RETENTION-008` | Readers double-collect catalog and retention heads and bind one complete catalog, manifest, and root-generation view under a `ReaderFence` | immutable snapshot and concurrency tests | Planned in #19 | diff --git a/xtask/tests/retention_store_v2_protocol_contract.rs b/xtask/tests/retention_store_v2_protocol_contract.rs index d413b2c..d2dc2ae 100644 --- a/xtask/tests/retention_store_v2_protocol_contract.rs +++ b/xtask/tests/retention_store_v2_protocol_contract.rs @@ -2,6 +2,11 @@ #![cfg(feature = "repository-tasks")] +#[path = "retention_store_v2_protocol_contract/closure_contract_laws.rs"] +mod closure_contract_laws; +#[path = "retention_store_v2_protocol_contract/migration_contract_laws.rs"] +mod migration_contract_laws; + use std::fs; use std::io; use std::path::{Path, PathBuf}; @@ -43,6 +48,7 @@ fn version_two_is_one_routed_protocol() -> Result<(), Box "successor to `keep.segment-store/v1`", "[Retention records and publication](retention.md)", "[Closure verification](closure.md)", + "[Closure corruption boundary](closure-corruption.md)", "[GC and disposition records](gc.md)", "[Migration and recovery](recovery.md)", "[Migration crash points](migration-crash.md)", @@ -94,110 +100,6 @@ fn retention_records_have_exact_canonical_grammars() -> Result<(), Box Result<(), Box> { - let closure = normalized(&read(&format!("{FORMAT_ROOT}/closure.md"))?); - - for required in [ - "one pinned, completely verified catalog generation", - "first scheduled", - "anchor is not a closure node", - "unique `SegmentRecordIdentity`", - "depth `1`", - "depth `2`", - "canonical layout payload length", - "complete segment-record length", - "checked addition before", - "repeated logical occurrence", - "replay the exact registered storage profile", - "authenticate the complete `BlobId`", - "keep.retention-closure/v2\\0", - "96-byte closure-member entries", - "canonical typed-identity order", - "Missing members still consume", - ] { - assert!( - closure.contains(required), - "segment-store v2 closure contract omits `{required}`" - ); - } - Ok(()) -} - -#[test] -fn migration_and_recovery_define_every_authority_boundary() -> Result<(), Box> -{ - let recovery = normalized(&read(&format!("{FORMAT_ROOT}/recovery.md"))?); - - for required in [ - "one-way explicit migration", - "`migration.intent`", - "`migration.intent.next`", - "`migration.receipt`", - "`migration.receipt.next`", - "`FORMAT.next`", - "`migration.intent` is exactly 256 bytes", - "`migration.receipt` is exactly 256 bytes", - "catalog generation, length, and digest", - "`definition.tsv`", - "migration inventory entry is exactly 56 bytes", - "2,097,152", - "keep.store-migration-intent/v2\\0", - "keep.store-format-marker/v2\\0", - "deterministically derived store identifier", - "absence of `retention/HEAD` is the canonical empty retention state", - "pre-effect incomplete stage", - "keep.initial-retention-state/v2\\0", - "keep.initial-gc-state/v2\\0", - "keep.empty-disposition-set/v2\\0", - "root.next` is durable before a new namespace directory", - "`KEEP-CRASH-036`", - "`KEEP-CRASH-073`", - "partial migration", - "Version-1 admission refuses", - "`reader.lock`", - "`GcRetirementIntent`", - "`GcRetirementReceipt`", - "`RecoveryDispositionReceipt`", - "unknown entry", - "unrecoverable ambiguity", - "idempotent", - "process death", - ] { - assert!( - recovery.contains(required), - "segment-store v2 recovery contract omits `{required}`" - ); - } - Ok(()) -} - -#[test] -fn migration_never_writes_canonical_fixed_names_in_place() -> Result<(), Box> -{ - let migration = normalized(&read(&format!("{FORMAT_ROOT}/migration-crash.md"))?); - - for required in [ - "never writes canonical fixed names in place", - "`migration.intent.next`", - "`FORMAT.next`", - "`migration.receipt.next`", - "linked without replacement", - "pre-effect incomplete stage", - "`KEEP-CRASH-053`", - "`KEEP-CRASH-073`", - "`0x00000000000003ff`", - "before, during, and after process-death evidence", - ] { - assert!( - migration.contains(required), - "segment-store v2 migration crash protocol omits `{required}`" - ); - } - Ok(()) -} - #[test] fn gc_records_are_bounded_before_their_implementation() -> Result<(), Box> { let gc = normalized(&read(&format!("{FORMAT_ROOT}/gc.md"))?); @@ -253,6 +155,7 @@ fn requirement_ledger_names_planned_and_executable_evidence() fn version_two_pages_stay_within_the_review_threshold() -> Result<(), Box> { for name in [ "README.md", + "closure-corruption.md", "closure.md", "gc.md", "migration-crash.md", diff --git a/xtask/tests/retention_store_v2_protocol_contract/closure_contract_laws.rs b/xtask/tests/retention_store_v2_protocol_contract/closure_contract_laws.rs new file mode 100644 index 0000000..34fe451 --- /dev/null +++ b/xtask/tests/retention_store_v2_protocol_contract/closure_contract_laws.rs @@ -0,0 +1,42 @@ +//! Closure verification and corruption-boundary contract laws. + +use super::{FORMAT_ROOT, normalized, read}; + +#[test] +fn closure_accounting_has_exact_units_and_canonical_evidence() +-> Result<(), Box> { + let closure = format!( + "{} {}", + normalized(&read(&format!("{FORMAT_ROOT}/closure.md"))?), + normalized(&read(&format!("{FORMAT_ROOT}/closure-corruption.md"))?) + ); + + for required in [ + "one pinned, completely verified catalog generation", + "first scheduled", + "anchor is not a closure node", + "unique `SegmentRecordIdentity`", + "depth `1`", + "depth `2`", + "canonical layout payload length", + "complete segment-record length", + "checked addition before", + "repeated logical occurrence", + "replay the exact registered storage profile", + "authenticate the complete `BlobId`", + "keep.retention-closure/v2\\0", + "96-byte closure-member entries", + "canonical typed-identity order", + "Missing members still consume", + "`CatalogSnapshot`", + "does not accept untrusted bytes", + "segment-record admission laws", + "`segment_format` fuzz target", + ] { + assert!( + closure.contains(required), + "segment-store v2 closure contract omits `{required}`" + ); + } + Ok(()) +} diff --git a/xtask/tests/retention_store_v2_protocol_contract/migration_contract_laws.rs b/xtask/tests/retention_store_v2_protocol_contract/migration_contract_laws.rs new file mode 100644 index 0000000..1a479cf --- /dev/null +++ b/xtask/tests/retention_store_v2_protocol_contract/migration_contract_laws.rs @@ -0,0 +1,76 @@ +//! Migration and recovery written-contract laws. + +use super::{FORMAT_ROOT, normalized, read}; + +#[test] +fn migration_and_recovery_define_every_authority_boundary() -> Result<(), Box> +{ + let recovery = normalized(&read(&format!("{FORMAT_ROOT}/recovery.md"))?); + + for required in [ + "one-way explicit migration", + "`migration.intent`", + "`migration.intent.next`", + "`migration.receipt`", + "`migration.receipt.next`", + "`FORMAT.next`", + "`migration.intent` is exactly 256 bytes", + "`migration.receipt` is exactly 256 bytes", + "catalog generation, length, and digest", + "`definition.tsv`", + "migration inventory entry is exactly 56 bytes", + "2,097,152", + "keep.store-migration-intent/v2\\0", + "keep.store-format-marker/v2\\0", + "deterministically derived store identifier", + "absence of `retention/HEAD` is the canonical empty retention state", + "pre-effect incomplete stage", + "keep.initial-retention-state/v2\\0", + "keep.initial-gc-state/v2\\0", + "keep.empty-disposition-set/v2\\0", + "root.next` is durable before a new namespace directory", + "`KEEP-CRASH-036`", + "`KEEP-CRASH-073`", + "partial migration", + "Version-1 admission refuses", + "`reader.lock`", + "`GcRetirementIntent`", + "`GcRetirementReceipt`", + "`RecoveryDispositionReceipt`", + "unknown entry", + "unrecoverable ambiguity", + "idempotent", + "process death", + ] { + assert!( + recovery.contains(required), + "segment-store v2 recovery contract omits `{required}`" + ); + } + Ok(()) +} + +#[test] +fn migration_never_writes_canonical_fixed_names_in_place() -> Result<(), Box> +{ + let migration = normalized(&read(&format!("{FORMAT_ROOT}/migration-crash.md"))?); + + for required in [ + "never writes canonical fixed names in place", + "`migration.intent.next`", + "`FORMAT.next`", + "`migration.receipt.next`", + "linked without replacement", + "pre-effect incomplete stage", + "`KEEP-CRASH-053`", + "`KEEP-CRASH-073`", + "`0x00000000000003ff`", + "before, during, and after process-death evidence", + ] { + assert!( + migration.contains(required), + "segment-store v2 migration crash protocol omits `{required}`" + ); + } + Ok(()) +} From b82b17ca8b8cd8aab99e3ef88c897b57cdc4ac3b Mon Sep 17 00:00:00 2001 From: James Ross Date: Wed, 29 Jul 2026 22:59:49 -0700 Subject: [PATCH 019/111] Add: Preflight retention transitions --- CHANGELOG.md | 3 + README.md | 9 +- docs/formats/segment-store-v2/README.md | 14 +- docs/formats/segment-store-v2/requirements.md | 2 +- docs/formats/segment-store-v2/retention.md | 4 + src/adapters/retention.rs | 4 + .../retention/transition_preflight.rs | 80 ++++++++ .../retention/transition_preflight_error.rs | 39 ++++ src/lib.rs | 13 +- tests/retention_preflight.rs | 179 ++++++++++++++++++ 10 files changed, 329 insertions(+), 18 deletions(-) create mode 100644 src/adapters/retention/transition_preflight.rs create mode 100644 src/adapters/retention/transition_preflight_error.rs create mode 100644 tests/retention_preflight.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index 74f5188..3ad25fc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,9 @@ after its public API and format compatibility policies are established. ### Changed +- Retention transition preflight now combines exact expected-generation + planning with deterministic closure verification against one pinned catalog + before any future publication storage call. - Repository crash-matrix execution now terminates isolated writer process groups at all 105 canonical before/during/after coordinates, retains open writer and stage authority until termination, executes production diff --git a/README.md b/README.md index 7194bed..b1da614 100644 --- a/README.md +++ b/README.md @@ -117,10 +117,11 @@ exact published generation and visible one-zero chunk when `HEAD` exists. This matrix proves application process-death behavior; it does not simulate host power loss. Version-2 retention values; canonical in-memory root, global manifest, and retention-head codecs; storage-independent expected-state -transition planning; and deterministic bounded closure verification against a -pinned catalog are implemented. Publication, recovery, compaction, and garbage -collection remain planned. Presence in the reference CAS does not claim -retention, crash recovery, or durability. +transition planning; deterministic bounded closure verification against a +pinned catalog; and a combined transition preflight proof are implemented. +Publication, recovery, compaction, and garbage collection remain planned. +Presence in the reference CAS does not claim retention, crash recovery, or +durability. Run the complete debug-profile matrix: diff --git a/docs/formats/segment-store-v2/README.md b/docs/formats/segment-store-v2/README.md index 103e379..3c12a8e 100644 --- a/docs/formats/segment-store-v2/README.md +++ b/docs/formats/segment-store-v2/README.md @@ -75,10 +75,10 @@ types now admit exact namespace bytes, namespace digests, root and liveness generations, registered realization profiles, bounded closure policies, reconstruction anchors, and semantic roots. Canonical root, manifest, and head codecs match their independent golden records. Storage-independent transition -planning and deterministic bounded closure verification against one pinned -catalog are available. Production filesystem retention publication, recovery, -migration, and garbage collection do not exist yet. Requirements that remain -planned or in progress in issue #19 or issue #21 are not complete -implementation evidence. A store must refuse unsupported version-2 state until -the relevant corruption, model-based, crash-injection, recovery, and fuzz -evidence is implemented. +planning, deterministic bounded closure verification against one pinned +catalog, and their combined preflight proof are available. Production +filesystem retention publication, recovery, migration, and garbage collection +do not exist yet. Requirements that remain planned or in progress in issue #19 +or issue #21 are not complete implementation evidence. A store must refuse +unsupported version-2 state until the relevant corruption, model-based, +crash-injection, recovery, and fuzz evidence is implemented. diff --git a/docs/formats/segment-store-v2/requirements.md b/docs/formats/segment-store-v2/requirements.md index 62419a8..abbeb89 100644 --- a/docs/formats/segment-store-v2/requirements.md +++ b/docs/formats/segment-store-v2/requirements.md @@ -12,7 +12,7 @@ case is not evidence. | `KEEP-RETENTION-001` | `RetentionNamespace`, `RootGeneration`, `LivenessGeneration`, profile coordinates, limits, anchors, and digests are validated typed values | `tests/retention_values.rs` and `tests/retention_root_encoding.rs` | Implemented | | `KEEP-RETENTION-002` | Root, manifest, and head codecs implement the exact canonical grammars and fixed bounds | independent golden corpus plus `tests/retention_root_encoding.rs`, `tests/retention_root_decoding.rs`, `tests/retention_manifest_codec.rs`, and `tests/retention_head_codec.rs` | Implemented | | `KEEP-RETENTION-003` | Every structural field, truncation boundary, ordering law, duplicate, overflow, flag, reserved byte, digest, checksum, and trailing byte has a precise refusal | corruption matrix and fuzz | Planned in #19 | -| `KEEP-RETENTION-004` | Retain and release compare expected and observed generations and publish exact successors only | storage-independent planning in `tests/retention_transition.rs`; operation and publication evidence remains | In progress in #19 | +| `KEEP-RETENTION-004` | Retain and release compare expected and observed generations and publish exact successors only | storage-independent planning and generation-before-closure preflight in `tests/retention_transition.rs` and `tests/retention_preflight.rs`; publication evidence remains | In progress in #19 | | `KEEP-RETENTION-005` | Closure derivation is deterministic, bounded, cycle-safe, fail-closed, and verifies complete blob reconstruction | exact accounting, reconstruction, adversarial-catalog, and exhaustive model laws in `tests/retention_closure.rs`; corrupt members refuse through the inherited segment-record admission laws and seeded `segment_format` fuzz target routed by `closure-corruption.md` | Implemented | | `KEEP-RETENTION-006` | Publication follows the exact ordered durability protocol, including new namespace-directory admission and retention of fixed-stage evidence until head commit, and returns only after cleanup synchronization | operation-order and `KEEP-CRASH-036..=052` crash-injection tests | Planned in #19 | | `KEEP-RETENTION-007` | Restart resolves every fixed-stage crash prefix to one documented lawful state or typed ambiguity | debug and release crash matrix | Planned in #19 | diff --git a/docs/formats/segment-store-v2/retention.md b/docs/formats/segment-store-v2/retention.md index e7e9529..f7241a9 100644 --- a/docs/formats/segment-store-v2/retention.md +++ b/docs/formats/segment-store-v2/retention.md @@ -246,6 +246,10 @@ authenticated reconstruction, and canonical digest defined by transition. Keep never omits one failed member and continues with a smaller live set. +`preflight_retention_transition` now combines steps 3 and 4 below without I/O. +It returns a consequential publish or already-committed proof only after exact +generation planning and complete closure verification succeed in that order. + Version-2 catalog publication holds the same writer authority and proves every current retained closure against its candidate catalog before replacing the catalog `HEAD`. diff --git a/src/adapters/retention.rs b/src/adapters/retention.rs index 74d5b77..7471b67 100644 --- a/src/adapters/retention.rs +++ b/src/adapters/retention.rs @@ -39,6 +39,8 @@ mod root_integrity; mod root_semantic_header; mod transition_error; mod transition_planner; +mod transition_preflight; +mod transition_preflight_error; mod transition_readiness; mod verified_closure; @@ -57,5 +59,7 @@ pub use root_decode_error::RetentionRootDecodeError; pub use root_encode_error::RetentionRootEncodeError; pub use transition_error::RetentionTransitionError; pub use transition_planner::plan_retention_transition; +pub use transition_preflight::{RetentionTransitionPreflight, preflight_retention_transition}; +pub use transition_preflight_error::RetentionTransitionPreflightError; pub use transition_readiness::RetentionTransitionReadiness; pub use verified_closure::VerifiedRetentionClosure; diff --git a/src/adapters/retention/transition_preflight.rs b/src/adapters/retention/transition_preflight.rs new file mode 100644 index 0000000..06992b6 --- /dev/null +++ b/src/adapters/retention/transition_preflight.rs @@ -0,0 +1,80 @@ +//! This boundary module owns complete retention transition preflight. + +use super::{ + AdmittedRetentionRoot, RetentionTransitionPreflightError, RetentionTransitionReadiness, + VerifiedRetentionClosure, plan_retention_transition, verify_retention_closure, +}; +use crate::CatalogSnapshot; +use crate::retention::RetentionGenerationExpectation; + +/// Complete storage-independent proof required before retention publication. +#[must_use = "retention preflight must be consumed by publication or handled explicitly"] +#[derive(Debug)] +pub enum RetentionTransitionPreflight<'encoded> { + /// The candidate is an exact successor whose verified closure must publish. + Publish { + /// Fully admitted canonical candidate root. + candidate: AdmittedRetentionRoot<'encoded>, + /// Closure proof against the exact pinned catalog. + closure: VerifiedRetentionClosure, + }, + /// The exact candidate is current and its closure still verifies. + AlreadyCommitted { + /// Fully admitted byte-identical current root. + candidate: AdmittedRetentionRoot<'encoded>, + /// Current closure proof against the exact pinned catalog. + closure: VerifiedRetentionClosure, + }, +} + +impl<'encoded> RetentionTransitionPreflight<'encoded> { + /// Borrows the fully admitted candidate root. + pub const fn candidate(&self) -> &AdmittedRetentionRoot<'encoded> { + match self { + Self::Publish { candidate, .. } | Self::AlreadyCommitted { candidate, .. } => candidate, + } + } + + /// Returns the complete verified closure evidence. + pub const fn closure(&self) -> VerifiedRetentionClosure { + match self { + Self::Publish { closure, .. } | Self::AlreadyCommitted { closure, .. } => *closure, + } + } +} + +/// Proves generation and closure invariants before retention storage mutation. +/// +/// Generation planning completes before closure traversal. Exact replay still +/// requires the current closure to verify against the pinned catalog. The +/// function performs no I/O and inherits closure verification's root-bounded +/// record index and per-layout entry allocation. +/// +/// # Errors +/// +/// Returns [`RetentionTransitionPreflightError::Transition`] for generation or +/// successor refusal, then [`RetentionTransitionPreflightError::Closure`] for +/// the first deterministic closure refusal. +pub fn preflight_retention_transition<'encoded>( + expected: RetentionGenerationExpectation, + current: Option<&AdmittedRetentionRoot<'_>>, + candidate: AdmittedRetentionRoot<'encoded>, + catalog: &CatalogSnapshot<'_, '_, '_>, +) -> Result, RetentionTransitionPreflightError> { + let readiness = plan_retention_transition(expected, current, candidate) + .map_err(|source| RetentionTransitionPreflightError::Transition { source })?; + let closure = + verify_retention_closure(readiness.candidate().root(), catalog).map_err(|source| { + RetentionTransitionPreflightError::Closure { + source: Box::new(source), + } + })?; + Ok(match readiness { + RetentionTransitionReadiness::Publish { candidate } => { + RetentionTransitionPreflight::Publish { candidate, closure } + } + RetentionTransitionReadiness::AlreadyCommitted { candidate } => { + RetentionTransitionPreflight::AlreadyCommitted { candidate, closure } + } + }) +} diff --git a/src/adapters/retention/transition_preflight_error.rs b/src/adapters/retention/transition_preflight_error.rs new file mode 100644 index 0000000..61113b1 --- /dev/null +++ b/src/adapters/retention/transition_preflight_error.rs @@ -0,0 +1,39 @@ +//! This boundary module owns retention transition preflight failures. + +use std::error::Error; +use std::fmt; + +use super::{RetentionClosureVerificationError, RetentionTransitionError}; + +/// Failure before a retention transition may invoke publication storage. +#[derive(Debug)] +pub enum RetentionTransitionPreflightError { + /// Generation or exact-successor planning refused the candidate. + Transition { + /// Preserved transition-planning refusal. + source: RetentionTransitionError, + }, + /// The candidate closure failed against the pinned catalog. + Closure { + /// Preserved deterministic closure refusal. + source: Box, + }, +} + +impl fmt::Display for RetentionTransitionPreflightError { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::Transition { .. } => formatter.write_str("retention transition planning failed"), + Self::Closure { .. } => formatter.write_str("retention closure verification failed"), + } + } +} + +impl Error for RetentionTransitionPreflightError { + fn source(&self) -> Option<&(dyn Error + 'static)> { + match self { + Self::Transition { source } => Some(source), + Self::Closure { source } => Some(source.as_ref()), + } + } +} diff --git a/src/lib.rs b/src/lib.rs index 12b90ed..ed115fb 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -23,10 +23,10 @@ //! a pinned writer-authorized filesystem adapter. Core retention namespaces, //! generations, realization policy, reconstruction anchors, and semantic roots //! are validated; canonical in-memory root, manifest, and head encoding and -//! decoding, storage-independent expected-state transition planning, and -//! deterministic bounded closure verification against a pinned catalog are -//! available. Retention publication, recovery, and garbage collection remain -//! intentionally absent. +//! decoding, storage-independent expected-state transition planning, +//! deterministic bounded closure verification against a pinned catalog, and a +//! combined transition preflight proof are available. Retention publication, +//! recovery, and garbage collection remain intentionally absent. #[cfg(test)] extern crate self as keep; @@ -109,8 +109,9 @@ pub use adapters::{ CanonicalRetentionManifest, CanonicalRetentionRoot, ChecksummedRetentionHead, RetentionClosureVerificationError, RetentionHeadDecodeError, RetentionManifestDecodeError, RetentionManifestEncodeError, RetentionRootDecodeError, RetentionRootEncodeError, - RetentionTransitionError, RetentionTransitionReadiness, VerifiedRetentionClosure, - plan_retention_transition, verify_retention_closure, + RetentionTransitionError, RetentionTransitionPreflight, RetentionTransitionPreflightError, + RetentionTransitionReadiness, VerifiedRetentionClosure, plan_retention_transition, + preflight_retention_transition, verify_retention_closure, }; pub use blob::{ BlobHashError, BlobHasher, BlobId, BlobLength, BlobReadError, ByteLength, ByteOffset, diff --git a/tests/retention_preflight.rs b/tests/retention_preflight.rs new file mode 100644 index 0000000..7eee6b5 --- /dev/null +++ b/tests/retention_preflight.rs @@ -0,0 +1,179 @@ +//! Retention transition preflight laws. + +mod support; + +use std::error::Error; + +use keep::{ + AdmittedCatalog, AdmittedRetentionRoot, AdmittedSegment, CatalogSnapshot, ChecksummedCatalog, + ChecksummedPublicationHead, LayoutEntryLimit, RetentionClosureVerificationError, + RetentionGenerationExpectation, RetentionTransitionError, RetentionTransitionPreflight, + RetentionTransitionPreflightError, RootGeneration, SegmentReadPolicy, SegmentRecordIdentity, + SegmentRecordLimit, preflight_retention_transition, +}; +use support::{decode_hex, require_error}; + +const ROOT_HEX: &str = include_str!("../conformance/segment-store/v2/one-anchor-root.hex"); +const BUNDLE_SEGMENT_HEX: &str = + include_str!("../conformance/segment-store/v1/one-zero-bundle-segment.hex"); +const BUNDLE_CATALOG_HEX: &str = + include_str!("../conformance/segment-store/v1/one-zero-bundle-catalog.hex"); +const BUNDLE_HEAD_HEX: &str = + include_str!("../conformance/segment-store/v1/one-zero-bundle-head.hex"); +const CHUNK_SEGMENT_HEX: &str = + include_str!("../conformance/segment-store/v1/one-zero-segment.hex"); +const CHUNK_CATALOG_HEX: &str = + include_str!("../conformance/segment-store/v1/one-zero-catalog.hex"); +const CHUNK_HEAD_HEX: &str = include_str!("../conformance/segment-store/v1/one-zero-head.hex"); + +#[test] +fn publish_preflight_binds_generation_and_closure_proofs() -> Result<(), Box> { + let root_bytes = fixture(ROOT_HEX)?; + let candidate = AdmittedRetentionRoot::decode(&root_bytes)?; + let preflight = with_snapshot( + BUNDLE_SEGMENT_HEX, + BUNDLE_CATALOG_HEX, + BUNDLE_HEAD_HEX, + |snapshot| { + preflight_retention_transition( + RetentionGenerationExpectation::Absent, + None, + candidate, + snapshot, + ) + }, + )??; + + assert!(matches!( + preflight, + RetentionTransitionPreflight::Publish { + candidate, + closure, + } if candidate.root().generation() == RootGeneration::INITIAL + && closure.usage().node_count() == 2 + )); + Ok(()) +} + +#[test] +fn stale_generation_refuses_before_missing_closure_evidence() -> Result<(), Box> { + let root_bytes = fixture(ROOT_HEX)?; + let candidate = AdmittedRetentionRoot::decode(&root_bytes)?; + let result = with_snapshot( + CHUNK_SEGMENT_HEX, + CHUNK_CATALOG_HEX, + CHUNK_HEAD_HEX, + |snapshot| { + preflight_retention_transition( + RetentionGenerationExpectation::Current(RootGeneration::INITIAL), + None, + candidate, + snapshot, + ) + }, + )?; + let error = require_error(result, "stale generation reached closure verification")?; + + assert!(matches!( + error, + RetentionTransitionPreflightError::Transition { + source: RetentionTransitionError::StaleGeneration { + expected: RetentionGenerationExpectation::Current(expected), + observed: None, + }, + } if expected == RootGeneration::INITIAL + )); + Ok(()) +} + +#[test] +fn valid_generation_preserves_missing_closure_member() -> Result<(), Box> { + let root_bytes = fixture(ROOT_HEX)?; + let candidate = AdmittedRetentionRoot::decode(&root_bytes)?; + let result = with_snapshot( + CHUNK_SEGMENT_HEX, + CHUNK_CATALOG_HEX, + CHUNK_HEAD_HEX, + |snapshot| { + preflight_retention_transition( + RetentionGenerationExpectation::Absent, + None, + candidate, + snapshot, + ) + }, + )?; + let error = require_error(result, "missing closure member passed preflight")?; + let RetentionTransitionPreflightError::Closure { source } = error else { + return Err("missing closure member reached the wrong preflight boundary".into()); + }; + + assert!(matches!( + *source, + RetentionClosureVerificationError::MissingMember { + identity: SegmentRecordIdentity::Layout(_), + } + )); + Ok(()) +} + +#[test] +fn exact_retry_still_returns_current_closure_evidence() -> Result<(), Box> { + let root_bytes = fixture(ROOT_HEX)?; + let current = AdmittedRetentionRoot::decode(&root_bytes)?; + let candidate = AdmittedRetentionRoot::decode(&root_bytes)?; + let preflight = with_snapshot( + BUNDLE_SEGMENT_HEX, + BUNDLE_CATALOG_HEX, + BUNDLE_HEAD_HEX, + |snapshot| { + preflight_retention_transition( + RetentionGenerationExpectation::Absent, + Some(¤t), + candidate, + snapshot, + ) + }, + )??; + + assert!(matches!( + preflight, + RetentionTransitionPreflight::AlreadyCommitted { closure, .. } + if closure.usage().physical_bytes() == 509 + )); + Ok(()) +} + +fn with_snapshot( + segment_hex: &str, + catalog_hex: &str, + head_hex: &str, + operation: impl FnOnce(&CatalogSnapshot<'_, '_, '_>) -> Result, +) -> Result, Box> { + let segment_bytes = fixture(segment_hex)?; + let catalog_bytes = fixture(catalog_hex)?; + let head_bytes = fixture(head_hex)?; + let segment = AdmittedSegment::decode(&segment_bytes, maximum_policy())?; + let segments = [segment]; + let catalog = admitted_catalog(&catalog_bytes, &segments)?; + let head = ChecksummedPublicationHead::decode(&head_bytes)?; + let snapshot = head.admit(catalog)?; + Ok(operation(&snapshot)) +} + +fn admitted_catalog<'catalog, 'records>( + catalog_bytes: &'catalog [u8], + segments: &'records [AdmittedSegment<'records>], +) -> Result, Box> { + ChecksummedCatalog::decode(catalog_bytes)? + .admit(segments) + .map_err(Into::into) +} + +const fn maximum_policy() -> SegmentReadPolicy { + SegmentReadPolicy::new(SegmentRecordLimit::MAXIMUM, LayoutEntryLimit::MAXIMUM) +} + +fn fixture(hex: &str) -> Result, Box> { + decode_hex(hex.strip_suffix('\n').ok_or("fixture must end in one LF")?).map_err(Into::into) +} From 50eb23caa0bcaa970f3b758852a4ce1b9ee969bb Mon Sep 17 00:00:00 2001 From: James Ross Date: Wed, 29 Jul 2026 23:09:20 -0700 Subject: [PATCH 020/111] Add: Name retention publication phases --- CHANGELOG.md | 3 +- README.md | 8 +- docs/formats/segment-store-v2/README.md | 13 +-- docs/formats/segment-store-v2/recovery.md | 3 + docs/formats/segment-store-v2/requirements.md | 2 +- src/adapters/retention.rs | 2 + src/adapters/retention/publication_phase.rs | 92 +++++++++++++++++++ src/lib.rs | 13 +-- tests/retention_publication_phase.rs | 83 +++++++++++++++++ 9 files changed, 201 insertions(+), 18 deletions(-) create mode 100644 src/adapters/retention/publication_phase.rs create mode 100644 tests/retention_publication_phase.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index 3ad25fc..7b9474b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,7 +12,8 @@ after its public API and format compatibility policies are established. - Retention transition preflight now combines exact expected-generation planning with deterministic closure verification against one pinned catalog - before any future publication storage call. + before any future publication storage call. A typed 17-phase vocabulary + freezes the planned durability and crash-boundary order. - Repository crash-matrix execution now terminates isolated writer process groups at all 105 canonical before/during/after coordinates, retains open writer and stage authority until termination, executes production diff --git a/README.md b/README.md index b1da614..39d7afe 100644 --- a/README.md +++ b/README.md @@ -118,10 +118,10 @@ matrix proves application process-death behavior; it does not simulate host power loss. Version-2 retention values; canonical in-memory root, global manifest, and retention-head codecs; storage-independent expected-state transition planning; deterministic bounded closure verification against a -pinned catalog; and a combined transition preflight proof are implemented. -Publication, recovery, compaction, and garbage collection remain planned. -Presence in the reference CAS does not claim retention, crash recovery, or -durability. +pinned catalog; a combined transition preflight proof; and the exact 17-phase +publication vocabulary are implemented. Publication execution, recovery, +compaction, and garbage collection remain planned. Presence in the reference +CAS does not claim retention, crash recovery, or durability. Run the complete debug-profile matrix: diff --git a/docs/formats/segment-store-v2/README.md b/docs/formats/segment-store-v2/README.md index 3c12a8e..12001af 100644 --- a/docs/formats/segment-store-v2/README.md +++ b/docs/formats/segment-store-v2/README.md @@ -76,9 +76,10 @@ generations, registered realization profiles, bounded closure policies, reconstruction anchors, and semantic roots. Canonical root, manifest, and head codecs match their independent golden records. Storage-independent transition planning, deterministic bounded closure verification against one pinned -catalog, and their combined preflight proof are available. Production -filesystem retention publication, recovery, migration, and garbage collection -do not exist yet. Requirements that remain planned or in progress in issue #19 -or issue #21 are not complete implementation evidence. A store must refuse -unsupported version-2 state until the relevant corruption, model-based, -crash-injection, recovery, and fuzz evidence is implemented. +catalog, their combined preflight proof, and the exact 17-phase publication +vocabulary are available. Production filesystem retention publication, +recovery, migration, and garbage collection do not exist yet. Requirements +that remain planned or in progress in issue #19 or issue #21 are not complete +implementation evidence. A store must refuse unsupported version-2 state until +the relevant corruption, model-based, crash-injection, recovery, and fuzz +evidence is implemented. diff --git a/docs/formats/segment-store-v2/recovery.md b/docs/formats/segment-store-v2/recovery.md index dfd29dc..604d0e3 100644 --- a/docs/formats/segment-store-v2/recovery.md +++ b/docs/formats/segment-store-v2/recovery.md @@ -277,6 +277,9 @@ The retention crash points are: | `KEEP-CRASH-051` | retained manifest-stage removal | | `KEEP-CRASH-052` | retention cleanup synchronization | +`RetentionPublicationPhase::ALL` freezes this exact order as a typed public +vocabulary. Storage execution and process-death evidence remain unimplemented. + Each point requires before, during, and after process-death evidence. Restart must establish exact catalog visibility, retention head, namespace generation, orphan classification, stage disposition, and recovery report. diff --git a/docs/formats/segment-store-v2/requirements.md b/docs/formats/segment-store-v2/requirements.md index abbeb89..fa450bd 100644 --- a/docs/formats/segment-store-v2/requirements.md +++ b/docs/formats/segment-store-v2/requirements.md @@ -14,7 +14,7 @@ case is not evidence. | `KEEP-RETENTION-003` | Every structural field, truncation boundary, ordering law, duplicate, overflow, flag, reserved byte, digest, checksum, and trailing byte has a precise refusal | corruption matrix and fuzz | Planned in #19 | | `KEEP-RETENTION-004` | Retain and release compare expected and observed generations and publish exact successors only | storage-independent planning and generation-before-closure preflight in `tests/retention_transition.rs` and `tests/retention_preflight.rs`; publication evidence remains | In progress in #19 | | `KEEP-RETENTION-005` | Closure derivation is deterministic, bounded, cycle-safe, fail-closed, and verifies complete blob reconstruction | exact accounting, reconstruction, adversarial-catalog, and exhaustive model laws in `tests/retention_closure.rs`; corrupt members refuse through the inherited segment-record admission laws and seeded `segment_format` fuzz target routed by `closure-corruption.md` | Implemented | -| `KEEP-RETENTION-006` | Publication follows the exact ordered durability protocol, including new namespace-directory admission and retention of fixed-stage evidence until head commit, and returns only after cleanup synchronization | operation-order and `KEEP-CRASH-036..=052` crash-injection tests | Planned in #19 | +| `KEEP-RETENTION-006` | Publication follows the exact ordered durability protocol, including new namespace-directory admission and retention of fixed-stage evidence until head commit, and returns only after cleanup synchronization | typed 17-phase vocabulary in `tests/retention_publication_phase.rs`; storage execution, operation-order, and `KEEP-CRASH-036..=052` crash-injection tests remain | In progress in #19 | | `KEEP-RETENTION-007` | Restart resolves every fixed-stage crash prefix to one documented lawful state or typed ambiguity | debug and release crash matrix | Planned in #19 | | `KEEP-RETENTION-008` | Readers double-collect catalog and retention heads and bind one complete catalog, manifest, and root-generation view under a `ReaderFence` | immutable snapshot and concurrency tests | Planned in #19 | | `KEEP-RETENTION-009` | Exact already-committed retry is idempotent only while its successor remains current | byte-identical readiness and stale-state planning in `tests/retention_transition.rs`; publication retry remains | In progress in #19 | diff --git a/src/adapters/retention.rs b/src/adapters/retention.rs index 7471b67..cc00ceb 100644 --- a/src/adapters/retention.rs +++ b/src/adapters/retention.rs @@ -27,6 +27,7 @@ mod manifest_field_decoder; mod manifest_header_decoder; mod manifest_integrity; mod manifest_semantic_header; +mod publication_phase; mod root_anchor_decoder; mod root_decode_error; mod root_decode_error_display; @@ -55,6 +56,7 @@ pub use closure_verifier::verify_retention_closure; pub use head_decode_error::RetentionHeadDecodeError; pub use manifest_decode_error::RetentionManifestDecodeError; pub use manifest_encode_error::RetentionManifestEncodeError; +pub use publication_phase::RetentionPublicationPhase; pub use root_decode_error::RetentionRootDecodeError; pub use root_encode_error::RetentionRootEncodeError; pub use transition_error::RetentionTransitionError; diff --git a/src/adapters/retention/publication_phase.rs b/src/adapters/retention/publication_phase.rs new file mode 100644 index 0000000..130bc77 --- /dev/null +++ b/src/adapters/retention/publication_phase.rs @@ -0,0 +1,92 @@ +//! This boundary module owns exact retention publication durability phases. + +use std::fmt; + +/// Storage transition attempted by retention namespace publication. +/// +/// [`Self::ALL`] corresponds in order to `KEEP-CRASH-036` through +/// `KEEP-CRASH-052`. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum RetentionPublicationPhase { + /// Write the complete canonical `root.next`. + WriteRootStage, + /// Synchronize `root.next`. + SynchronizeRootStage, + /// Create or exactly admit the digest-named root namespace. + AdmitRootNamespace, + /// Synchronize `retention/roots` after namespace admission. + SynchronizeRootsAfterNamespace, + /// Link the synchronized root stage into its immutable namespace. + LinkRoot, + /// Synchronize the digest-named root namespace after linking. + SynchronizeRootNamespace, + /// Write the complete canonical `manifest.next`. + WriteManifestStage, + /// Synchronize `manifest.next`. + SynchronizeManifestStage, + /// Link the synchronized manifest into its immutable pool. + LinkManifest, + /// Synchronize the immutable manifest pool. + SynchronizeManifestPool, + /// Write the complete canonical retention `head.next`. + WriteHeadStage, + /// Synchronize the retention `head.next`. + SynchronizeHeadStage, + /// Atomically replace the retention `HEAD`. + ReplaceHead, + /// Synchronize `retention` after head replacement. + SynchronizeRetentionNamespace, + /// Remove the retained `root.next`. + RemoveRootStage, + /// Remove the retained `manifest.next`. + RemoveManifestStage, + /// Synchronize `retention` after stage cleanup. + SynchronizeCleanup, +} + +impl RetentionPublicationPhase { + /// Every publication phase in normative crash-boundary order. + pub const ALL: [Self; 17] = [ + Self::WriteRootStage, + Self::SynchronizeRootStage, + Self::AdmitRootNamespace, + Self::SynchronizeRootsAfterNamespace, + Self::LinkRoot, + Self::SynchronizeRootNamespace, + Self::WriteManifestStage, + Self::SynchronizeManifestStage, + Self::LinkManifest, + Self::SynchronizeManifestPool, + Self::WriteHeadStage, + Self::SynchronizeHeadStage, + Self::ReplaceHead, + Self::SynchronizeRetentionNamespace, + Self::RemoveRootStage, + Self::RemoveManifestStage, + Self::SynchronizeCleanup, + ]; +} + +impl fmt::Display for RetentionPublicationPhase { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter.write_str(match self { + Self::WriteRootStage => "root-stage write", + Self::SynchronizeRootStage => "root-stage synchronization", + Self::AdmitRootNamespace => "root-namespace admission", + Self::SynchronizeRootsAfterNamespace => "post-namespace roots synchronization", + Self::LinkRoot => "immutable root link", + Self::SynchronizeRootNamespace => "root-namespace synchronization", + Self::WriteManifestStage => "manifest-stage write", + Self::SynchronizeManifestStage => "manifest-stage synchronization", + Self::LinkManifest => "immutable manifest link", + Self::SynchronizeManifestPool => "manifest-pool synchronization", + Self::WriteHeadStage => "retention-head-stage write", + Self::SynchronizeHeadStage => "retention-head-stage synchronization", + Self::ReplaceHead => "retention-head replacement", + Self::SynchronizeRetentionNamespace => "retention-namespace synchronization", + Self::RemoveRootStage => "retained root-stage removal", + Self::RemoveManifestStage => "retained manifest-stage removal", + Self::SynchronizeCleanup => "retention cleanup synchronization", + }) + } +} diff --git a/src/lib.rs b/src/lib.rs index ed115fb..52eabd1 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -25,8 +25,9 @@ //! are validated; canonical in-memory root, manifest, and head encoding and //! decoding, storage-independent expected-state transition planning, //! deterministic bounded closure verification against a pinned catalog, and a -//! combined transition preflight proof are available. Retention publication, -//! recovery, and garbage collection remain intentionally absent. +//! combined transition preflight proof and exact publication phase vocabulary +//! are available. Retention publication execution, recovery, and garbage +//! collection remain intentionally absent. #[cfg(test)] extern crate self as keep; @@ -108,10 +109,10 @@ pub use adapters::{ AdmittedRetentionManifest, AdmittedRetentionRoot, CanonicalRetentionHead, CanonicalRetentionManifest, CanonicalRetentionRoot, ChecksummedRetentionHead, RetentionClosureVerificationError, RetentionHeadDecodeError, RetentionManifestDecodeError, - RetentionManifestEncodeError, RetentionRootDecodeError, RetentionRootEncodeError, - RetentionTransitionError, RetentionTransitionPreflight, RetentionTransitionPreflightError, - RetentionTransitionReadiness, VerifiedRetentionClosure, plan_retention_transition, - preflight_retention_transition, verify_retention_closure, + RetentionManifestEncodeError, RetentionPublicationPhase, RetentionRootDecodeError, + RetentionRootEncodeError, RetentionTransitionError, RetentionTransitionPreflight, + RetentionTransitionPreflightError, RetentionTransitionReadiness, VerifiedRetentionClosure, + plan_retention_transition, preflight_retention_transition, verify_retention_closure, }; pub use blob::{ BlobHashError, BlobHasher, BlobId, BlobLength, BlobReadError, ByteLength, ByteOffset, diff --git a/tests/retention_publication_phase.rs b/tests/retention_publication_phase.rs new file mode 100644 index 0000000..6fb343e --- /dev/null +++ b/tests/retention_publication_phase.rs @@ -0,0 +1,83 @@ +//! Retention publication phase vocabulary laws. + +use keep::RetentionPublicationPhase; + +#[test] +fn publication_phases_are_complete_ordered_and_stably_named() { + let expected = [ + ( + RetentionPublicationPhase::WriteRootStage, + "root-stage write", + ), + ( + RetentionPublicationPhase::SynchronizeRootStage, + "root-stage synchronization", + ), + ( + RetentionPublicationPhase::AdmitRootNamespace, + "root-namespace admission", + ), + ( + RetentionPublicationPhase::SynchronizeRootsAfterNamespace, + "post-namespace roots synchronization", + ), + (RetentionPublicationPhase::LinkRoot, "immutable root link"), + ( + RetentionPublicationPhase::SynchronizeRootNamespace, + "root-namespace synchronization", + ), + ( + RetentionPublicationPhase::WriteManifestStage, + "manifest-stage write", + ), + ( + RetentionPublicationPhase::SynchronizeManifestStage, + "manifest-stage synchronization", + ), + ( + RetentionPublicationPhase::LinkManifest, + "immutable manifest link", + ), + ( + RetentionPublicationPhase::SynchronizeManifestPool, + "manifest-pool synchronization", + ), + ( + RetentionPublicationPhase::WriteHeadStage, + "retention-head-stage write", + ), + ( + RetentionPublicationPhase::SynchronizeHeadStage, + "retention-head-stage synchronization", + ), + ( + RetentionPublicationPhase::ReplaceHead, + "retention-head replacement", + ), + ( + RetentionPublicationPhase::SynchronizeRetentionNamespace, + "retention-namespace synchronization", + ), + ( + RetentionPublicationPhase::RemoveRootStage, + "retained root-stage removal", + ), + ( + RetentionPublicationPhase::RemoveManifestStage, + "retained manifest-stage removal", + ), + ( + RetentionPublicationPhase::SynchronizeCleanup, + "retention cleanup synchronization", + ), + ]; + + assert_eq!( + RetentionPublicationPhase::ALL, + expected.map(|entry| entry.0) + ); + assert_eq!( + RetentionPublicationPhase::ALL.map(|phase| phase.to_string()), + expected.map(|entry| entry.1.to_owned()) + ); +} From 261a8a65395d0860dcb15f944fc62615ae224d48 Mon Sep 17 00:00:00 2001 From: James Ross Date: Wed, 29 Jul 2026 23:28:06 -0700 Subject: [PATCH 021/111] Add: Define retention publication storage --- CHANGELOG.md | 2 +- README.md | 3 +- docs/formats/segment-store-v2/README.md | 13 +- docs/formats/segment-store-v2/requirements.md | 2 +- docs/formats/segment-store-v2/retention.md | 8 +- src/adapters/retention.rs | 4 + src/adapters/retention/namespace_admission.rs | 11 ++ src/adapters/retention/publication_storage.rs | 139 ++++++++++++++++++ src/lib.rs | 12 +- tests/retention_publication_storage.rs | 60 ++++++++ .../recording_storage.rs | 122 +++++++++++++++ 11 files changed, 359 insertions(+), 17 deletions(-) create mode 100644 src/adapters/retention/namespace_admission.rs create mode 100644 src/adapters/retention/publication_storage.rs create mode 100644 tests/retention_publication_storage.rs create mode 100644 tests/retention_publication_storage/recording_storage.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index 7b9474b..5a25d50 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,7 +13,7 @@ after its public API and format compatibility policies are established. - Retention transition preflight now combines exact expected-generation planning with deterministic closure verification against one pinned catalog before any future publication storage call. A typed 17-phase vocabulary - freezes the planned durability and crash-boundary order. + and blocking storage port freeze the durability and crash-boundary contract. - Repository crash-matrix execution now terminates isolated writer process groups at all 105 canonical before/during/after coordinates, retains open writer and stage authority until termination, executes production diff --git a/README.md b/README.md index 39d7afe..c10200e 100644 --- a/README.md +++ b/README.md @@ -119,7 +119,8 @@ power loss. Version-2 retention values; canonical in-memory root, global manifest, and retention-head codecs; storage-independent expected-state transition planning; deterministic bounded closure verification against a pinned catalog; a combined transition preflight proof; and the exact 17-phase -publication vocabulary are implemented. Publication execution, recovery, +publication vocabulary with a blocking storage capability port are +implemented. Publication orchestration, filesystem execution, recovery, compaction, and garbage collection remain planned. Presence in the reference CAS does not claim retention, crash recovery, or durability. diff --git a/docs/formats/segment-store-v2/README.md b/docs/formats/segment-store-v2/README.md index 12001af..f1c627a 100644 --- a/docs/formats/segment-store-v2/README.md +++ b/docs/formats/segment-store-v2/README.md @@ -77,9 +77,10 @@ reconstruction anchors, and semantic roots. Canonical root, manifest, and head codecs match their independent golden records. Storage-independent transition planning, deterministic bounded closure verification against one pinned catalog, their combined preflight proof, and the exact 17-phase publication -vocabulary are available. Production filesystem retention publication, -recovery, migration, and garbage collection do not exist yet. Requirements -that remain planned or in progress in issue #19 or issue #21 are not complete -implementation evidence. A store must refuse unsupported version-2 state until -the relevant corruption, model-based, crash-injection, recovery, and fuzz -evidence is implemented. +vocabulary with a blocking storage capability port are available. Publication +orchestration and production filesystem retention publication, recovery, +migration, and garbage collection do not exist yet. Requirements that remain +planned or in progress in issue #19 or issue #21 are not complete implementation +evidence. A store must refuse unsupported version-2 state until the relevant +corruption, model-based, crash-injection, recovery, and fuzz evidence is +implemented. diff --git a/docs/formats/segment-store-v2/requirements.md b/docs/formats/segment-store-v2/requirements.md index fa450bd..298d6f9 100644 --- a/docs/formats/segment-store-v2/requirements.md +++ b/docs/formats/segment-store-v2/requirements.md @@ -14,7 +14,7 @@ case is not evidence. | `KEEP-RETENTION-003` | Every structural field, truncation boundary, ordering law, duplicate, overflow, flag, reserved byte, digest, checksum, and trailing byte has a precise refusal | corruption matrix and fuzz | Planned in #19 | | `KEEP-RETENTION-004` | Retain and release compare expected and observed generations and publish exact successors only | storage-independent planning and generation-before-closure preflight in `tests/retention_transition.rs` and `tests/retention_preflight.rs`; publication evidence remains | In progress in #19 | | `KEEP-RETENTION-005` | Closure derivation is deterministic, bounded, cycle-safe, fail-closed, and verifies complete blob reconstruction | exact accounting, reconstruction, adversarial-catalog, and exhaustive model laws in `tests/retention_closure.rs`; corrupt members refuse through the inherited segment-record admission laws and seeded `segment_format` fuzz target routed by `closure-corruption.md` | Implemented | -| `KEEP-RETENTION-006` | Publication follows the exact ordered durability protocol, including new namespace-directory admission and retention of fixed-stage evidence until head commit, and returns only after cleanup synchronization | typed 17-phase vocabulary in `tests/retention_publication_phase.rs`; storage execution, operation-order, and `KEEP-CRASH-036..=052` crash-injection tests remain | In progress in #19 | +| `KEEP-RETENTION-006` | Publication follows the exact ordered durability protocol, including new namespace-directory admission and retention of fixed-stage evidence until head commit, and returns only after cleanup synchronization | typed 17-phase vocabulary in `tests/retention_publication_phase.rs` and blocking capability port in `tests/retention_publication_storage.rs`; ordered execution and `KEEP-CRASH-036..=052` crash-injection tests remain | In progress in #19 | | `KEEP-RETENTION-007` | Restart resolves every fixed-stage crash prefix to one documented lawful state or typed ambiguity | debug and release crash matrix | Planned in #19 | | `KEEP-RETENTION-008` | Readers double-collect catalog and retention heads and bind one complete catalog, manifest, and root-generation view under a `ReaderFence` | immutable snapshot and concurrency tests | Planned in #19 | | `KEEP-RETENTION-009` | Exact already-committed retry is idempotent only while its successor remains current | byte-identical readiness and stale-state planning in `tests/retention_transition.rs`; publication retry remains | In progress in #19 | diff --git a/docs/formats/segment-store-v2/retention.md b/docs/formats/segment-store-v2/retention.md index f7241a9..c55f051 100644 --- a/docs/formats/segment-store-v2/retention.md +++ b/docs/formats/segment-store-v2/retention.md @@ -160,9 +160,11 @@ retention/roots// ``` Names with alternate width, case, suffix, generation, or digest refuse. Keep -implements validated in-memory root, manifest, and head codecs plus -storage-independent expected-state transition planning. Closure verification, -filesystem publication, recovery, and garbage collection remain absent. +implements validated in-memory root, manifest, and head codecs, +storage-independent expected-state transition planning, deterministic closure +verification, and a blocking publication storage capability port. Publication +orchestration, filesystem execution, recovery, and garbage collection remain +absent. ## Global retention manifest diff --git a/src/adapters/retention.rs b/src/adapters/retention.rs index cc00ceb..73a63f5 100644 --- a/src/adapters/retention.rs +++ b/src/adapters/retention.rs @@ -27,7 +27,9 @@ mod manifest_field_decoder; mod manifest_header_decoder; mod manifest_integrity; mod manifest_semantic_header; +mod namespace_admission; mod publication_phase; +mod publication_storage; mod root_anchor_decoder; mod root_decode_error; mod root_decode_error_display; @@ -56,7 +58,9 @@ pub use closure_verifier::verify_retention_closure; pub use head_decode_error::RetentionHeadDecodeError; pub use manifest_decode_error::RetentionManifestDecodeError; pub use manifest_encode_error::RetentionManifestEncodeError; +pub use namespace_admission::RetentionNamespaceAdmission; pub use publication_phase::RetentionPublicationPhase; +pub use publication_storage::RetentionPublicationStorage; pub use root_decode_error::RetentionRootDecodeError; pub use root_encode_error::RetentionRootEncodeError; pub use transition_error::RetentionTransitionError; diff --git a/src/adapters/retention/namespace_admission.rs b/src/adapters/retention/namespace_admission.rs new file mode 100644 index 0000000..4d9b1f7 --- /dev/null +++ b/src/adapters/retention/namespace_admission.rs @@ -0,0 +1,11 @@ +//! This boundary module owns digest-named retention namespace admission outcomes. + +/// Result of exact retention root-namespace admission. +#[must_use] +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum RetentionNamespaceAdmission { + /// The exact digest-named directory already existed and was admitted. + Existing, + /// The exact digest-named directory was created and admitted. + Created, +} diff --git a/src/adapters/retention/publication_storage.rs b/src/adapters/retention/publication_storage.rs new file mode 100644 index 0000000..cbef5d8 --- /dev/null +++ b/src/adapters/retention/publication_storage.rs @@ -0,0 +1,139 @@ +//! This boundary module owns blocking retention publication durability capabilities. + +use std::io; + +use super::{ + AdmittedRetentionRoot, CanonicalRetentionHead, CanonicalRetentionManifest, + RetentionNamespaceAdmission, +}; + +/// Blocking storage capabilities for one writer-locked retention publication. +/// +/// An implementation must retain exclusive writer authority and one pinned +/// store root for the complete operation. Each method corresponds to one +/// [`RetentionPublicationPhase`](super::RetentionPublicationPhase) and must not +/// report success before the named durability and verification obligations are +/// complete. +pub trait RetentionPublicationStorage { + /// Exclusively creates and completely writes the canonical root stage. + /// + /// # Errors + /// + /// Returns the exact creation, write, or flush failure. + fn write_root_stage(&mut self, root: &AdmittedRetentionRoot<'_>) -> io::Result<()>; + + /// Synchronizes the complete root stage. + /// + /// # Errors + /// + /// Returns the exact file-synchronization failure. + fn synchronize_root_stage(&mut self) -> io::Result<()>; + + /// Creates or exactly admits the candidate's digest-named root namespace. + /// + /// # Errors + /// + /// Returns the exact namespace creation or admission failure. + fn admit_root_namespace( + &mut self, + root: &AdmittedRetentionRoot<'_>, + ) -> io::Result; + + /// Synchronizes `retention/roots` after namespace creation. + /// + /// # Errors + /// + /// Returns the exact directory-synchronization failure. + fn synchronize_roots_after_namespace(&mut self) -> io::Result<()>; + + /// Links and completely verifies the immutable root-pool entry. + /// + /// # Errors + /// + /// Returns the exact link, reopen, or verification failure. + fn link_root(&mut self, root: &AdmittedRetentionRoot<'_>) -> io::Result<()>; + + /// Synchronizes the candidate's digest-named root namespace. + /// + /// # Errors + /// + /// Returns the exact directory-synchronization failure. + fn synchronize_root_namespace(&mut self, root: &AdmittedRetentionRoot<'_>) -> io::Result<()>; + + /// Exclusively creates and completely writes the canonical manifest stage. + /// + /// # Errors + /// + /// Returns the exact creation, write, or flush failure. + fn write_manifest_stage(&mut self, manifest: &CanonicalRetentionManifest) -> io::Result<()>; + + /// Synchronizes the complete manifest stage. + /// + /// # Errors + /// + /// Returns the exact file-synchronization failure. + fn synchronize_manifest_stage(&mut self) -> io::Result<()>; + + /// Links and completely verifies the immutable manifest-pool entry. + /// + /// # Errors + /// + /// Returns the exact link, reopen, or verification failure. + fn link_manifest(&mut self, manifest: &CanonicalRetentionManifest) -> io::Result<()>; + + /// Synchronizes the immutable manifest pool. + /// + /// # Errors + /// + /// Returns the exact directory-synchronization failure. + fn synchronize_manifest_pool(&mut self) -> io::Result<()>; + + /// Exclusively creates and completely writes the canonical head stage. + /// + /// # Errors + /// + /// Returns the exact creation, write, or flush failure. + fn write_head_stage(&mut self, head: &CanonicalRetentionHead) -> io::Result<()>; + + /// Synchronizes the complete head stage. + /// + /// # Errors + /// + /// Returns the exact file-synchronization failure. + fn synchronize_head_stage(&mut self) -> io::Result<()>; + + /// Atomically replaces `retention/HEAD` with the synchronized head stage. + /// + /// # Errors + /// + /// Returns the exact replacement failure. + fn replace_head(&mut self) -> io::Result<()>; + + /// Synchronizes `retention` after head replacement. + /// + /// # Errors + /// + /// Returns the exact directory-synchronization failure. + fn synchronize_retention_namespace(&mut self) -> io::Result<()>; + + /// Removes only the retained root stage. + /// + /// # Errors + /// + /// Returns the exact removal failure. + fn remove_root_stage(&mut self) -> io::Result<()>; + + /// Removes only the retained manifest stage. + /// + /// # Errors + /// + /// Returns the exact removal failure. + fn remove_manifest_stage(&mut self) -> io::Result<()>; + + /// Synchronizes `retention` after both stage removals. + /// + /// # Errors + /// + /// Returns the exact directory-synchronization failure. + fn synchronize_cleanup(&mut self) -> io::Result<()>; +} diff --git a/src/lib.rs b/src/lib.rs index 52eabd1..bda7161 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -26,7 +26,8 @@ //! decoding, storage-independent expected-state transition planning, //! deterministic bounded closure verification against a pinned catalog, and a //! combined transition preflight proof and exact publication phase vocabulary -//! are available. Retention publication execution, recovery, and garbage +//! with a blocking storage capability port are available. Retention +//! publication orchestration, filesystem execution, recovery, and garbage //! collection remain intentionally absent. #[cfg(test)] @@ -109,10 +110,11 @@ pub use adapters::{ AdmittedRetentionManifest, AdmittedRetentionRoot, CanonicalRetentionHead, CanonicalRetentionManifest, CanonicalRetentionRoot, ChecksummedRetentionHead, RetentionClosureVerificationError, RetentionHeadDecodeError, RetentionManifestDecodeError, - RetentionManifestEncodeError, RetentionPublicationPhase, RetentionRootDecodeError, - RetentionRootEncodeError, RetentionTransitionError, RetentionTransitionPreflight, - RetentionTransitionPreflightError, RetentionTransitionReadiness, VerifiedRetentionClosure, - plan_retention_transition, preflight_retention_transition, verify_retention_closure, + RetentionManifestEncodeError, RetentionNamespaceAdmission, RetentionPublicationPhase, + RetentionPublicationStorage, RetentionRootDecodeError, RetentionRootEncodeError, + RetentionTransitionError, RetentionTransitionPreflight, RetentionTransitionPreflightError, + RetentionTransitionReadiness, VerifiedRetentionClosure, plan_retention_transition, + preflight_retention_transition, verify_retention_closure, }; pub use blob::{ BlobHashError, BlobHasher, BlobId, BlobLength, BlobReadError, ByteLength, ByteOffset, diff --git a/tests/retention_publication_storage.rs b/tests/retention_publication_storage.rs new file mode 100644 index 0000000..900a85b --- /dev/null +++ b/tests/retention_publication_storage.rs @@ -0,0 +1,60 @@ +//! Retention publication storage-port laws. + +#[path = "retention_publication_storage/recording_storage.rs"] +pub mod recording_storage; +mod support; + +use std::error::Error; + +use keep::{ + AdmittedRetentionManifest, AdmittedRetentionRoot, CanonicalRetentionHead, + CanonicalRetentionManifest, ChecksummedRetentionHead, RetentionNamespaceAdmission, + RetentionPublicationPhase, RetentionPublicationStorage, +}; +use recording_storage::RecordingStorage; +use support::decode_hex; + +const ROOT_HEX: &str = include_str!("../conformance/segment-store/v2/one-anchor-root.hex"); +const MANIFEST_HEX: &str = include_str!("../conformance/segment-store/v2/one-root-manifest.hex"); +const HEAD_HEX: &str = include_str!("../conformance/segment-store/v2/one-root-head.hex"); + +#[test] +fn storage_port_names_one_capability_per_publication_phase() -> Result<(), Box> { + let root_bytes = fixture(ROOT_HEX)?; + let manifest_bytes = fixture(MANIFEST_HEX)?; + let head_bytes = fixture(HEAD_HEX)?; + let root = AdmittedRetentionRoot::decode(&root_bytes)?; + let admitted_manifest = AdmittedRetentionManifest::decode(&manifest_bytes)?; + let manifest = CanonicalRetentionManifest::from_manifest(admitted_manifest.manifest())?; + let checksummed_head = ChecksummedRetentionHead::decode(&head_bytes)?; + let head = CanonicalRetentionHead::from_head(checksummed_head.head()); + let mut storage = RecordingStorage::new(); + + storage.write_root_stage(&root)?; + storage.synchronize_root_stage()?; + assert_eq!( + storage.admit_root_namespace(&root)?, + RetentionNamespaceAdmission::Created + ); + storage.synchronize_roots_after_namespace()?; + storage.link_root(&root)?; + storage.synchronize_root_namespace(&root)?; + storage.write_manifest_stage(&manifest)?; + storage.synchronize_manifest_stage()?; + storage.link_manifest(&manifest)?; + storage.synchronize_manifest_pool()?; + storage.write_head_stage(&head)?; + storage.synchronize_head_stage()?; + storage.replace_head()?; + storage.synchronize_retention_namespace()?; + storage.remove_root_stage()?; + storage.remove_manifest_stage()?; + storage.synchronize_cleanup()?; + + assert_eq!(storage.observed(), RetentionPublicationPhase::ALL); + Ok(()) +} + +fn fixture(hex: &str) -> Result, Box> { + decode_hex(hex.strip_suffix('\n').ok_or("fixture must end in one LF")?).map_err(Into::into) +} diff --git a/tests/retention_publication_storage/recording_storage.rs b/tests/retention_publication_storage/recording_storage.rs new file mode 100644 index 0000000..51047dc --- /dev/null +++ b/tests/retention_publication_storage/recording_storage.rs @@ -0,0 +1,122 @@ +//! Deterministic retention publication storage recorder. + +use std::io; + +use keep::{ + AdmittedRetentionRoot, CanonicalRetentionHead, CanonicalRetentionManifest, + RetentionNamespaceAdmission, RetentionPublicationPhase, RetentionPublicationStorage, +}; + +/// Storage port that records every attempted publication phase. +#[derive(Default)] +pub struct RecordingStorage { + observed: Vec, +} + +impl RecordingStorage { + /// Creates an empty recorder. + pub const fn new() -> Self { + Self { + observed: Vec::new(), + } + } + + /// Returns every recorded phase in call order. + pub fn observed(&self) -> &[RetentionPublicationPhase] { + &self.observed + } + + fn record(&mut self, phase: RetentionPublicationPhase) { + self.observed.push(phase); + } +} + +impl RetentionPublicationStorage for RecordingStorage { + fn write_root_stage(&mut self, _root: &AdmittedRetentionRoot<'_>) -> io::Result<()> { + self.record(RetentionPublicationPhase::WriteRootStage); + Ok(()) + } + + fn synchronize_root_stage(&mut self) -> io::Result<()> { + self.record(RetentionPublicationPhase::SynchronizeRootStage); + Ok(()) + } + + fn admit_root_namespace( + &mut self, + _root: &AdmittedRetentionRoot<'_>, + ) -> io::Result { + self.record(RetentionPublicationPhase::AdmitRootNamespace); + Ok(RetentionNamespaceAdmission::Created) + } + + fn synchronize_roots_after_namespace(&mut self) -> io::Result<()> { + self.record(RetentionPublicationPhase::SynchronizeRootsAfterNamespace); + Ok(()) + } + + fn link_root(&mut self, _root: &AdmittedRetentionRoot<'_>) -> io::Result<()> { + self.record(RetentionPublicationPhase::LinkRoot); + Ok(()) + } + + fn synchronize_root_namespace(&mut self, _root: &AdmittedRetentionRoot<'_>) -> io::Result<()> { + self.record(RetentionPublicationPhase::SynchronizeRootNamespace); + Ok(()) + } + + fn write_manifest_stage(&mut self, _manifest: &CanonicalRetentionManifest) -> io::Result<()> { + self.record(RetentionPublicationPhase::WriteManifestStage); + Ok(()) + } + + fn synchronize_manifest_stage(&mut self) -> io::Result<()> { + self.record(RetentionPublicationPhase::SynchronizeManifestStage); + Ok(()) + } + + fn link_manifest(&mut self, _manifest: &CanonicalRetentionManifest) -> io::Result<()> { + self.record(RetentionPublicationPhase::LinkManifest); + Ok(()) + } + + fn synchronize_manifest_pool(&mut self) -> io::Result<()> { + self.record(RetentionPublicationPhase::SynchronizeManifestPool); + Ok(()) + } + + fn write_head_stage(&mut self, _head: &CanonicalRetentionHead) -> io::Result<()> { + self.record(RetentionPublicationPhase::WriteHeadStage); + Ok(()) + } + + fn synchronize_head_stage(&mut self) -> io::Result<()> { + self.record(RetentionPublicationPhase::SynchronizeHeadStage); + Ok(()) + } + + fn replace_head(&mut self) -> io::Result<()> { + self.record(RetentionPublicationPhase::ReplaceHead); + Ok(()) + } + + fn synchronize_retention_namespace(&mut self) -> io::Result<()> { + self.record(RetentionPublicationPhase::SynchronizeRetentionNamespace); + Ok(()) + } + + fn remove_root_stage(&mut self) -> io::Result<()> { + self.record(RetentionPublicationPhase::RemoveRootStage); + Ok(()) + } + + fn remove_manifest_stage(&mut self) -> io::Result<()> { + self.record(RetentionPublicationPhase::RemoveManifestStage); + Ok(()) + } + + fn synchronize_cleanup(&mut self) -> io::Result<()> { + self.record(RetentionPublicationPhase::SynchronizeCleanup); + Ok(()) + } +} From b8affa75aa99e9f512c3035908627186f63a9767 Mon Sep 17 00:00:00 2001 From: James Ross Date: Wed, 29 Jul 2026 23:57:00 -0700 Subject: [PATCH 022/111] Add: Prepare retention publication artifacts --- CHANGELOG.md | 3 +- README.md | 8 +- docs/formats/segment-store-v2/README.md | 7 +- docs/formats/segment-store-v2/requirements.md | 2 +- docs/formats/segment-store-v2/retention.md | 8 +- src/adapters/retention.rs | 8 + .../retention/manifest_entry_update.rs | 85 +++++++++ .../retention/prepared_publication.rs | 94 +++++++++ .../retention/publication_preparation.rs | 67 +++++++ .../publication_preparation_error.rs | 179 ++++++++++++++++++ src/adapters/retention/successor_manifest.rs | 146 ++++++++++++++ src/lib.rs | 21 +- src/retention/liveness_generation.rs | 3 + tests/retention_publication_preparation.rs | 11 ++ .../fixture.rs | 121 ++++++++++++ .../initial_laws.rs | 35 ++++ .../refusal_laws.rs | 78 ++++++++ .../successor_laws.rs | 117 ++++++++++++ 18 files changed, 973 insertions(+), 20 deletions(-) create mode 100644 src/adapters/retention/manifest_entry_update.rs create mode 100644 src/adapters/retention/prepared_publication.rs create mode 100644 src/adapters/retention/publication_preparation.rs create mode 100644 src/adapters/retention/publication_preparation_error.rs create mode 100644 src/adapters/retention/successor_manifest.rs create mode 100644 tests/retention_publication_preparation.rs create mode 100644 tests/retention_publication_preparation/fixture.rs create mode 100644 tests/retention_publication_preparation/initial_laws.rs create mode 100644 tests/retention_publication_preparation/refusal_laws.rs create mode 100644 tests/retention_publication_preparation/successor_laws.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index 5a25d50..766e479 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,7 +13,8 @@ after its public API and format compatibility policies are established. - Retention transition preflight now combines exact expected-generation planning with deterministic closure verification against one pinned catalog before any future publication storage call. A typed 17-phase vocabulary - and blocking storage port freeze the durability and crash-boundary contract. + and blocking storage port freeze the durability and crash-boundary contract; + preparation binds preflight to exact canonical manifest and head successors. - Repository crash-matrix execution now terminates isolated writer process groups at all 105 canonical before/during/after coordinates, retains open writer and stage authority until termination, executes production diff --git a/README.md b/README.md index c10200e..38623fb 100644 --- a/README.md +++ b/README.md @@ -120,9 +120,11 @@ manifest, and retention-head codecs; storage-independent expected-state transition planning; deterministic bounded closure verification against a pinned catalog; a combined transition preflight proof; and the exact 17-phase publication vocabulary with a blocking storage capability port are -implemented. Publication orchestration, filesystem execution, recovery, -compaction, and garbage collection remain planned. Presence in the reference -CAS does not claim retention, crash recovery, or durability. +implemented. Storage-independent preparation also binds preflight to exact +canonical manifest and head successors. Publication orchestration, filesystem +execution, recovery, compaction, and garbage collection remain planned. +Presence in the reference CAS does not claim durable retention or crash +recovery. Run the complete debug-profile matrix: diff --git a/docs/formats/segment-store-v2/README.md b/docs/formats/segment-store-v2/README.md index f1c627a..54c7c1d 100644 --- a/docs/formats/segment-store-v2/README.md +++ b/docs/formats/segment-store-v2/README.md @@ -77,10 +77,11 @@ reconstruction anchors, and semantic roots. Canonical root, manifest, and head codecs match their independent golden records. Storage-independent transition planning, deterministic bounded closure verification against one pinned catalog, their combined preflight proof, and the exact 17-phase publication -vocabulary with a blocking storage capability port are available. Publication +vocabulary with a blocking storage capability port are available. +Storage-independent preparation derives exact canonical manifest and head +successors from coherent preflight and current-manifest evidence. Publication orchestration and production filesystem retention publication, recovery, migration, and garbage collection do not exist yet. Requirements that remain planned or in progress in issue #19 or issue #21 are not complete implementation evidence. A store must refuse unsupported version-2 state until the relevant -corruption, model-based, crash-injection, recovery, and fuzz evidence is -implemented. +corruption, model-based, crash-injection, recovery, and fuzz evidence exists. diff --git a/docs/formats/segment-store-v2/requirements.md b/docs/formats/segment-store-v2/requirements.md index 298d6f9..58b56b1 100644 --- a/docs/formats/segment-store-v2/requirements.md +++ b/docs/formats/segment-store-v2/requirements.md @@ -12,7 +12,7 @@ case is not evidence. | `KEEP-RETENTION-001` | `RetentionNamespace`, `RootGeneration`, `LivenessGeneration`, profile coordinates, limits, anchors, and digests are validated typed values | `tests/retention_values.rs` and `tests/retention_root_encoding.rs` | Implemented | | `KEEP-RETENTION-002` | Root, manifest, and head codecs implement the exact canonical grammars and fixed bounds | independent golden corpus plus `tests/retention_root_encoding.rs`, `tests/retention_root_decoding.rs`, `tests/retention_manifest_codec.rs`, and `tests/retention_head_codec.rs` | Implemented | | `KEEP-RETENTION-003` | Every structural field, truncation boundary, ordering law, duplicate, overflow, flag, reserved byte, digest, checksum, and trailing byte has a precise refusal | corruption matrix and fuzz | Planned in #19 | -| `KEEP-RETENTION-004` | Retain and release compare expected and observed generations and publish exact successors only | storage-independent planning and generation-before-closure preflight in `tests/retention_transition.rs` and `tests/retention_preflight.rs`; publication evidence remains | In progress in #19 | +| `KEEP-RETENTION-004` | Retain and release compare expected and observed generations and publish exact successors only | storage-independent planning and generation-before-closure preflight in `tests/retention_transition.rs` and `tests/retention_preflight.rs`; exact global successor preparation in `tests/retention_publication_preparation.rs`; publication evidence remains | In progress in #19 | | `KEEP-RETENTION-005` | Closure derivation is deterministic, bounded, cycle-safe, fail-closed, and verifies complete blob reconstruction | exact accounting, reconstruction, adversarial-catalog, and exhaustive model laws in `tests/retention_closure.rs`; corrupt members refuse through the inherited segment-record admission laws and seeded `segment_format` fuzz target routed by `closure-corruption.md` | Implemented | | `KEEP-RETENTION-006` | Publication follows the exact ordered durability protocol, including new namespace-directory admission and retention of fixed-stage evidence until head commit, and returns only after cleanup synchronization | typed 17-phase vocabulary in `tests/retention_publication_phase.rs` and blocking capability port in `tests/retention_publication_storage.rs`; ordered execution and `KEEP-CRASH-036..=052` crash-injection tests remain | In progress in #19 | | `KEEP-RETENTION-007` | Restart resolves every fixed-stage crash prefix to one documented lawful state or typed ambiguity | debug and release crash matrix | Planned in #19 | diff --git a/docs/formats/segment-store-v2/retention.md b/docs/formats/segment-store-v2/retention.md index c55f051..cb9cc03 100644 --- a/docs/formats/segment-store-v2/retention.md +++ b/docs/formats/segment-store-v2/retention.md @@ -248,9 +248,11 @@ authenticated reconstruction, and canonical digest defined by transition. Keep never omits one failed member and continues with a smaller live set. -`preflight_retention_transition` now combines steps 3 and 4 below without I/O. -It returns a consequential publish or already-committed proof only after exact -generation planning and complete closure verification succeed in that order. +`preflight_retention_transition` combines steps 3 and 4 without I/O, returning +publish or already-committed only after generation and closure verification. +`prepare_retention_publication` binds that proof to the current manifest, +refuses incoherent root coordinates, and derives exact canonical successors; +exact retry produces no new global artifacts. Version-2 catalog publication holds the same writer authority and proves every current retained closure against its candidate catalog before replacing the diff --git a/src/adapters/retention.rs b/src/adapters/retention.rs index 73a63f5..addcd7c 100644 --- a/src/adapters/retention.rs +++ b/src/adapters/retention.rs @@ -23,12 +23,16 @@ mod manifest_decoder; mod manifest_encode_error; mod manifest_encoder; mod manifest_entry_decoder; +mod manifest_entry_update; mod manifest_field_decoder; mod manifest_header_decoder; mod manifest_integrity; mod manifest_semantic_header; mod namespace_admission; +mod prepared_publication; mod publication_phase; +mod publication_preparation; +mod publication_preparation_error; mod publication_storage; mod root_anchor_decoder; mod root_decode_error; @@ -40,6 +44,7 @@ mod root_field_decoder; mod root_header_decoder; mod root_integrity; mod root_semantic_header; +mod successor_manifest; mod transition_error; mod transition_planner; mod transition_preflight; @@ -59,7 +64,10 @@ pub use head_decode_error::RetentionHeadDecodeError; pub use manifest_decode_error::RetentionManifestDecodeError; pub use manifest_encode_error::RetentionManifestEncodeError; pub use namespace_admission::RetentionNamespaceAdmission; +pub use prepared_publication::{PreparedRetentionPublication, RetentionPublicationPreparation}; pub use publication_phase::RetentionPublicationPhase; +pub use publication_preparation::prepare_retention_publication; +pub use publication_preparation_error::RetentionPublicationPreparationError; pub use publication_storage::RetentionPublicationStorage; pub use root_decode_error::RetentionRootDecodeError; pub use root_encode_error::RetentionRootEncodeError; diff --git a/src/adapters/retention/manifest_entry_update.rs b/src/adapters/retention/manifest_entry_update.rs new file mode 100644 index 0000000..3047439 --- /dev/null +++ b/src/adapters/retention/manifest_entry_update.rs @@ -0,0 +1,85 @@ +//! This boundary module owns bounded retention manifest entry updates. + +use super::RetentionPublicationPreparationError; +use crate::{RetentionManifest, RetentionManifestEntry, RetentionManifestError}; + +#[derive(Clone, Copy)] +pub(super) enum ManifestEntryUpdate { + Insert { + index: usize, + entry: RetentionManifestEntry, + }, + Replace { + index: usize, + entry: RetentionManifestEntry, + }, +} + +pub(super) fn apply( + current: &[RetentionManifestEntry], + update: ManifestEntryUpdate, +) -> Result, RetentionPublicationPreparationError> { + let inserts = usize::from(matches!(update, ManifestEntryUpdate::Insert { .. })); + let observed = current.len().checked_add(inserts).ok_or( + RetentionPublicationPreparationError::Manifest { + source: RetentionManifestError::EntryCountExceeded { + maximum: RetentionManifest::MAXIMUM_ENTRY_COUNT, + observed: usize::MAX, + }, + }, + )?; + require_admitted_count(observed)?; + let mut entries = Vec::new(); + entries + .try_reserve_exact(observed) + .map_err(|source| RetentionPublicationPreparationError::EntryAllocation { source })?; + let (index, entry, skip) = match update { + ManifestEntryUpdate::Insert { index, entry } => (index, entry, 0), + ManifestEntryUpdate::Replace { index, entry } => (index, entry, 1), + }; + let (before, remaining) = split(current, index)?; + let after = + remaining + .get(skip..) + .ok_or(RetentionPublicationPreparationError::ManifestEntryIndex { + index, + entry_count: current.len(), + })?; + entries.extend_from_slice(before); + entries.push(entry); + entries.extend_from_slice(after); + Ok(entries) +} + +fn require_admitted_count(observed: usize) -> Result<(), RetentionPublicationPreparationError> { + let admitted = u32::try_from(observed).map_err(|_| entry_count_error(observed))?; + if admitted > RetentionManifest::MAXIMUM_ENTRY_COUNT { + Err(entry_count_error(observed)) + } else { + Ok(()) + } +} + +fn split( + current: &[RetentionManifestEntry], + index: usize, +) -> Result< + (&[RetentionManifestEntry], &[RetentionManifestEntry]), + RetentionPublicationPreparationError, +> { + current.split_at_checked(index).ok_or( + RetentionPublicationPreparationError::ManifestEntryIndex { + index, + entry_count: current.len(), + }, + ) +} + +const fn entry_count_error(observed: usize) -> RetentionPublicationPreparationError { + RetentionPublicationPreparationError::Manifest { + source: RetentionManifestError::EntryCountExceeded { + maximum: RetentionManifest::MAXIMUM_ENTRY_COUNT, + observed, + }, + } +} diff --git a/src/adapters/retention/prepared_publication.rs b/src/adapters/retention/prepared_publication.rs new file mode 100644 index 0000000..6eca896 --- /dev/null +++ b/src/adapters/retention/prepared_publication.rs @@ -0,0 +1,94 @@ +//! This boundary module owns storage-ready retention publication artifacts. + +use super::{ + AdmittedRetentionRoot, CanonicalRetentionHead, CanonicalRetentionManifest, + VerifiedRetentionClosure, +}; +use crate::LivenessGeneration; + +/// Canonical global artifacts ready for ordered storage execution. +#[must_use = "prepared retention publication must be executed or handled explicitly"] +#[derive(Debug)] +pub struct PreparedRetentionPublication { + manifest: CanonicalRetentionManifest, + head: CanonicalRetentionHead, + liveness_generation: LivenessGeneration, +} + +impl PreparedRetentionPublication { + /// Returns the complete canonical successor manifest. + pub const fn manifest(&self) -> &CanonicalRetentionManifest { + &self.manifest + } + + /// Returns the complete canonical successor head. + pub const fn head(&self) -> &CanonicalRetentionHead { + &self.head + } + + /// Returns the exact successor global liveness generation. + pub const fn liveness_generation(&self) -> LivenessGeneration { + self.liveness_generation + } + + pub(super) const fn new( + manifest: CanonicalRetentionManifest, + head: CanonicalRetentionHead, + liveness_generation: LivenessGeneration, + ) -> Self { + Self { + manifest, + head, + liveness_generation, + } + } +} + +/// Result of binding one preflight proof to the current global manifest. +#[must_use = "retention publication preparation must be handled explicitly"] +#[derive(Debug)] +pub struct RetentionPublicationPreparation<'encoded> { + candidate: AdmittedRetentionRoot<'encoded>, + closure: VerifiedRetentionClosure, + publication: Option, +} + +impl<'encoded> RetentionPublicationPreparation<'encoded> { + /// Borrows the admitted candidate root. + pub const fn candidate(&self) -> &AdmittedRetentionRoot<'encoded> { + &self.candidate + } + + /// Returns the revalidated candidate closure. + pub const fn closure(&self) -> VerifiedRetentionClosure { + self.closure + } + + /// Returns new global artifacts, or normal absence for an exact retry. + pub const fn publication(&self) -> Option<&PreparedRetentionPublication> { + self.publication.as_ref() + } + + pub(super) const fn publish( + candidate: AdmittedRetentionRoot<'encoded>, + closure: VerifiedRetentionClosure, + publication: PreparedRetentionPublication, + ) -> Self { + Self { + candidate, + closure, + publication: Some(publication), + } + } + + pub(super) const fn already_committed( + candidate: AdmittedRetentionRoot<'encoded>, + closure: VerifiedRetentionClosure, + ) -> Self { + Self { + candidate, + closure, + publication: None, + } + } +} diff --git a/src/adapters/retention/publication_preparation.rs b/src/adapters/retention/publication_preparation.rs new file mode 100644 index 0000000..da1b6ee --- /dev/null +++ b/src/adapters/retention/publication_preparation.rs @@ -0,0 +1,67 @@ +//! This boundary module owns storage-independent retention publication preparation. + +use super::{ + AdmittedRetentionManifest, CanonicalRetentionHead, CanonicalRetentionManifest, + PreparedRetentionPublication, RetentionPublicationPreparation, + RetentionPublicationPreparationError, RetentionTransitionPreflight, successor_manifest, +}; +use crate::{RetentionHead, RetentionManifestLength}; + +/// Binds preflight evidence to one current manifest and canonical successors. +/// +/// A publish result owns the complete manifest and head bytes required by the +/// storage protocol. An exact retry returns no new global artifacts. This +/// function performs no I/O. +/// +/// # Errors +/// +/// Returns [`RetentionPublicationPreparationError`] when the current manifest +/// disagrees with the preflight candidate, checked generation arithmetic or +/// bounded allocation fails, or canonical successor construction refuses. +pub fn prepare_retention_publication<'encoded>( + preflight: RetentionTransitionPreflight<'encoded>, + current_manifest: Option<&AdmittedRetentionManifest<'_>>, +) -> Result, RetentionPublicationPreparationError> { + match preflight { + RetentionTransitionPreflight::AlreadyCommitted { candidate, closure } => { + successor_manifest::require_current_selection(&candidate, current_manifest)?; + Ok(RetentionPublicationPreparation::already_committed( + candidate, closure, + )) + } + RetentionTransitionPreflight::Publish { candidate, closure } => { + let semantic_manifest = successor_manifest::build(&candidate, current_manifest)?; + let liveness_generation = semantic_manifest.generation(); + let predecessor = semantic_manifest.predecessor(); + let manifest = CanonicalRetentionManifest::from_manifest(&semantic_manifest).map_err( + |source| RetentionPublicationPreparationError::ManifestEncoding { source }, + )?; + let manifest_length = manifest_length(&manifest)?; + let semantic_head = RetentionHead::new( + liveness_generation, + manifest_length, + manifest.digest(), + predecessor, + ) + .map_err(|source| RetentionPublicationPreparationError::Head { source })?; + let head = CanonicalRetentionHead::from_head(&semantic_head); + let publication = + PreparedRetentionPublication::new(manifest, head, liveness_generation); + Ok(RetentionPublicationPreparation::publish( + candidate, + closure, + publication, + )) + } + } +} + +fn manifest_length( + manifest: &CanonicalRetentionManifest, +) -> Result { + let observed = manifest.encoded().len(); + let value = u64::try_from(observed) + .map_err(|_| RetentionPublicationPreparationError::ManifestLengthOverflow { observed })?; + RetentionManifestLength::new(value) + .map_err(|source| RetentionPublicationPreparationError::ManifestLength { source }) +} diff --git a/src/adapters/retention/publication_preparation_error.rs b/src/adapters/retention/publication_preparation_error.rs new file mode 100644 index 0000000..9a17d8c --- /dev/null +++ b/src/adapters/retention/publication_preparation_error.rs @@ -0,0 +1,179 @@ +//! This boundary module owns retention publication preparation failures. + +use std::collections::TryReserveError; +use std::error::Error; +use std::fmt; + +use super::RetentionManifestEncodeError; +use crate::{ + LivenessGenerationError, RetentionHeadError, RetentionManifestError, + RetentionManifestLengthError, RetentionNamespaceDigest, RetentionRootDigest, RootGeneration, + RootGenerationError, +}; + +/// Failure to bind preflight evidence into canonical global artifacts. +#[derive(Debug)] +pub enum RetentionPublicationPreparationError { + /// Exact retry had no current global manifest to select its root. + CurrentManifestRequired { + /// Candidate namespace that must be selected. + namespace: RetentionNamespaceDigest, + }, + /// Exact retry was absent from the current global manifest. + CurrentManifestEntryMissing { + /// Candidate namespace absent from the manifest. + namespace: RetentionNamespaceDigest, + /// Candidate root generation. + generation: RootGeneration, + /// Candidate root digest. + digest: RetentionRootDigest, + }, + /// Current manifest entry and candidate did not form a successor. + ManifestSuccessorMismatch { + /// Candidate namespace. + namespace: RetentionNamespaceDigest, + /// Generation selected by the current manifest. + current_generation: RootGeneration, + /// Root digest selected by the current manifest. + current_digest: RetentionRootDigest, + /// Candidate root generation. + candidate_generation: RootGeneration, + /// Candidate-declared predecessor. + candidate_predecessor: Option, + }, + /// Exact retry disagreed with the current manifest selection. + CurrentManifestEntryMismatch { + /// Candidate namespace. + namespace: RetentionNamespaceDigest, + /// Generation selected by the current manifest. + current_generation: RootGeneration, + /// Root digest selected by the current manifest. + current_digest: RetentionRootDigest, + /// Candidate root generation. + candidate_generation: RootGeneration, + /// Candidate root digest. + candidate_digest: RetentionRootDigest, + }, + /// A namespace absent from the manifest carried a noninitial candidate. + UnexpectedNamespaceSuccessor { + /// Candidate namespace. + namespace: RetentionNamespaceDigest, + /// Noninitial candidate generation. + generation: RootGeneration, + /// Candidate-declared predecessor. + predecessor: Option, + }, + /// Internal manifest update index escaped the admitted entry range. + ManifestEntryIndex { + /// Attempted insertion or replacement index. + index: usize, + /// Current manifest entry count. + entry_count: usize, + }, + /// Global liveness generation could not advance. + LivenessGeneration { + /// Preserved checked-generation refusal. + source: LivenessGenerationError, + }, + /// A manifest-selected root generation could not advance. + RootGeneration { + /// Preserved checked-generation refusal. + source: RootGenerationError, + }, + /// Bounded successor-entry allocation was refused. + EntryAllocation { + /// Preserved allocation refusal. + source: TryReserveError, + }, + /// Successor manifest semantics were refused. + Manifest { + /// Preserved semantic refusal. + source: RetentionManifestError, + }, + /// Canonical successor manifest encoding failed. + ManifestEncoding { + /// Preserved encoding refusal. + source: RetentionManifestEncodeError, + }, + /// Host length could not fit the protocol length domain. + ManifestLengthOverflow { + /// Observed canonical byte length. + observed: usize, + }, + /// Canonical successor manifest length was refused. + ManifestLength { + /// Preserved typed-length refusal. + source: RetentionManifestLengthError, + }, + /// Successor head semantics were refused. + Head { + /// Preserved semantic refusal. + source: RetentionHeadError, + }, +} + +impl fmt::Display for RetentionPublicationPreparationError { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::CurrentManifestRequired { .. } => { + formatter.write_str("retention retry requires a current global manifest") + } + Self::CurrentManifestEntryMissing { .. } => { + formatter.write_str("retention retry root is absent from the current manifest") + } + Self::ManifestSuccessorMismatch { .. } => { + formatter.write_str("retention manifest entry disagrees with the candidate root") + } + Self::CurrentManifestEntryMismatch { .. } => { + formatter.write_str("retention retry disagrees with the current manifest entry") + } + Self::UnexpectedNamespaceSuccessor { .. } => { + formatter.write_str("new retention namespace candidate is not generation one") + } + Self::ManifestEntryIndex { index, entry_count } => write!( + formatter, + "retention manifest update index {index} exceeds {entry_count} entries" + ), + Self::LivenessGeneration { source } => write!(formatter, "{source}"), + Self::RootGeneration { source } => write!(formatter, "{source}"), + Self::EntryAllocation { .. } => { + formatter.write_str("retention successor entry allocation failed") + } + Self::Manifest { .. } => { + formatter.write_str("retention successor manifest admission failed") + } + Self::ManifestEncoding { .. } => { + formatter.write_str("retention successor manifest encoding failed") + } + Self::ManifestLengthOverflow { observed } => write!( + formatter, + "retention manifest byte length {observed} exceeds the protocol integer domain" + ), + Self::ManifestLength { .. } => { + formatter.write_str("retention successor manifest length admission failed") + } + Self::Head { .. } => formatter.write_str("retention successor head admission failed"), + } + } +} + +impl Error for RetentionPublicationPreparationError { + fn source(&self) -> Option<&(dyn Error + 'static)> { + match self { + Self::LivenessGeneration { source } => Some(source), + Self::RootGeneration { source } => Some(source), + Self::EntryAllocation { source } => Some(source), + Self::Manifest { source } => Some(source), + Self::ManifestEncoding { source } => Some(source), + Self::ManifestLength { source } => Some(source), + Self::Head { source } => Some(source), + Self::CurrentManifestRequired { .. } + | Self::CurrentManifestEntryMissing { .. } + | Self::ManifestSuccessorMismatch { .. } + | Self::CurrentManifestEntryMismatch { .. } + | Self::UnexpectedNamespaceSuccessor { .. } + | Self::ManifestEntryIndex { .. } + | Self::ManifestLengthOverflow { .. } => None, + } + } +} diff --git a/src/adapters/retention/successor_manifest.rs b/src/adapters/retention/successor_manifest.rs new file mode 100644 index 0000000..d87b0d6 --- /dev/null +++ b/src/adapters/retention/successor_manifest.rs @@ -0,0 +1,146 @@ +//! This boundary module owns candidate-to-manifest successor binding. + +use super::{ + AdmittedRetentionManifest, AdmittedRetentionRoot, RetentionPublicationPreparationError, + manifest_entry_update::{self, ManifestEntryUpdate}, +}; +use crate::{LivenessGeneration, RetentionManifest, RetentionManifestEntry, RootGeneration}; + +pub(super) fn build( + candidate: &AdmittedRetentionRoot<'_>, + current: Option<&AdmittedRetentionManifest<'_>>, +) -> Result { + let entry = candidate_entry(candidate); + let Some(current) = current else { + require_initial(candidate)?; + let entries = + manifest_entry_update::apply(&[], ManifestEntryUpdate::Insert { index: 0, entry })?; + return RetentionManifest::new(LivenessGeneration::INITIAL, None, entries) + .map_err(|source| RetentionPublicationPreparationError::Manifest { source }); + }; + let generation = current + .manifest() + .generation() + .successor() + .map_err(|source| RetentionPublicationPreparationError::LivenessGeneration { source })?; + let namespace = entry.namespace(); + let entries = current.manifest().entries(); + let update = match entries.binary_search_by_key(&namespace, |item| item.namespace()) { + Ok(index) => { + let selected = entries.get(index).copied().ok_or( + RetentionPublicationPreparationError::ManifestEntryIndex { + index, + entry_count: entries.len(), + }, + )?; + require_successor(selected, candidate)?; + ManifestEntryUpdate::Replace { index, entry } + } + Err(index) => { + require_initial(candidate)?; + ManifestEntryUpdate::Insert { index, entry } + } + }; + let entries = manifest_entry_update::apply(entries, update)?; + RetentionManifest::new(generation, Some(current.digest()), entries) + .map_err(|source| RetentionPublicationPreparationError::Manifest { source }) +} + +pub(super) fn require_current_selection( + candidate: &AdmittedRetentionRoot<'_>, + current: Option<&AdmittedRetentionManifest<'_>>, +) -> Result<(), RetentionPublicationPreparationError> { + let namespace = candidate.root().namespace().digest(); + let current = current + .ok_or(RetentionPublicationPreparationError::CurrentManifestRequired { namespace })?; + let entry = current + .manifest() + .entries() + .binary_search_by_key(&namespace, |item| item.namespace()) + .ok() + .and_then(|index| current.manifest().entries().get(index)) + .copied() + .ok_or_else( + || RetentionPublicationPreparationError::CurrentManifestEntryMissing { + namespace, + generation: candidate.root().generation(), + digest: candidate.digest(), + }, + )?; + if entry.root_generation() == candidate.root().generation() + && entry.root_digest() == candidate.digest() + { + Ok(()) + } else { + Err(current_mismatch(entry, candidate)) + } +} + +fn require_initial( + candidate: &AdmittedRetentionRoot<'_>, +) -> Result<(), RetentionPublicationPreparationError> { + if candidate.root().generation() == RootGeneration::INITIAL + && candidate.root().predecessor().is_none() + { + Ok(()) + } else { + Err( + RetentionPublicationPreparationError::UnexpectedNamespaceSuccessor { + namespace: candidate.root().namespace().digest(), + generation: candidate.root().generation(), + predecessor: candidate.root().predecessor(), + }, + ) + } +} + +fn require_successor( + current: RetentionManifestEntry, + candidate: &AdmittedRetentionRoot<'_>, +) -> Result<(), RetentionPublicationPreparationError> { + let expected_generation = current + .root_generation() + .successor() + .map_err(|source| RetentionPublicationPreparationError::RootGeneration { source })?; + if candidate.root().generation() == expected_generation + && candidate.root().predecessor() == Some(current.root_digest()) + { + Ok(()) + } else { + Err(successor_mismatch(current, candidate)) + } +} + +fn candidate_entry(candidate: &AdmittedRetentionRoot<'_>) -> RetentionManifestEntry { + RetentionManifestEntry::new( + candidate.root().namespace().digest(), + candidate.root().generation(), + candidate.digest(), + ) +} + +const fn successor_mismatch( + current: RetentionManifestEntry, + candidate: &AdmittedRetentionRoot<'_>, +) -> RetentionPublicationPreparationError { + RetentionPublicationPreparationError::ManifestSuccessorMismatch { + namespace: current.namespace(), + current_generation: current.root_generation(), + current_digest: current.root_digest(), + candidate_generation: candidate.root().generation(), + candidate_predecessor: candidate.root().predecessor(), + } +} + +const fn current_mismatch( + current: RetentionManifestEntry, + candidate: &AdmittedRetentionRoot<'_>, +) -> RetentionPublicationPreparationError { + RetentionPublicationPreparationError::CurrentManifestEntryMismatch { + namespace: current.namespace(), + current_generation: current.root_generation(), + current_digest: current.root_digest(), + candidate_generation: candidate.root().generation(), + candidate_digest: candidate.digest(), + } +} diff --git a/src/lib.rs b/src/lib.rs index bda7161..624ef17 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -26,9 +26,10 @@ //! decoding, storage-independent expected-state transition planning, //! deterministic bounded closure verification against a pinned catalog, and a //! combined transition preflight proof and exact publication phase vocabulary -//! with a blocking storage capability port are available. Retention -//! publication orchestration, filesystem execution, recovery, and garbage -//! collection remain intentionally absent. +//! with a blocking storage capability port are available. Storage-independent +//! preparation binds preflight to exact canonical manifest and head successors. +//! Retention publication orchestration, filesystem execution, recovery, and +//! garbage collection remain intentionally absent. #[cfg(test)] extern crate self as keep; @@ -109,12 +110,14 @@ pub use adapters::{ pub use adapters::{ AdmittedRetentionManifest, AdmittedRetentionRoot, CanonicalRetentionHead, CanonicalRetentionManifest, CanonicalRetentionRoot, ChecksummedRetentionHead, - RetentionClosureVerificationError, RetentionHeadDecodeError, RetentionManifestDecodeError, - RetentionManifestEncodeError, RetentionNamespaceAdmission, RetentionPublicationPhase, - RetentionPublicationStorage, RetentionRootDecodeError, RetentionRootEncodeError, - RetentionTransitionError, RetentionTransitionPreflight, RetentionTransitionPreflightError, - RetentionTransitionReadiness, VerifiedRetentionClosure, plan_retention_transition, - preflight_retention_transition, verify_retention_closure, + PreparedRetentionPublication, RetentionClosureVerificationError, RetentionHeadDecodeError, + RetentionManifestDecodeError, RetentionManifestEncodeError, RetentionNamespaceAdmission, + RetentionPublicationPhase, RetentionPublicationPreparation, + RetentionPublicationPreparationError, RetentionPublicationStorage, RetentionRootDecodeError, + RetentionRootEncodeError, RetentionTransitionError, RetentionTransitionPreflight, + RetentionTransitionPreflightError, RetentionTransitionReadiness, VerifiedRetentionClosure, + plan_retention_transition, preflight_retention_transition, prepare_retention_publication, + verify_retention_closure, }; pub use blob::{ BlobHashError, BlobHasher, BlobId, BlobLength, BlobReadError, ByteLength, ByteOffset, diff --git a/src/retention/liveness_generation.rs b/src/retention/liveness_generation.rs index 3936c5d..0f67ec9 100644 --- a/src/retention/liveness_generation.rs +++ b/src/retention/liveness_generation.rs @@ -13,6 +13,9 @@ use super::LivenessGenerationError; pub struct LivenessGeneration(NonZeroU64); impl LivenessGeneration { + /// First global retention liveness generation. + pub const INITIAL: Self = Self(NonZeroU64::MIN); + /// Admits one positive liveness generation. /// /// # Errors diff --git a/tests/retention_publication_preparation.rs b/tests/retention_publication_preparation.rs new file mode 100644 index 0000000..d18b19b --- /dev/null +++ b/tests/retention_publication_preparation.rs @@ -0,0 +1,11 @@ +//! Retention publication preparation laws. + +#[path = "retention_publication_preparation/fixture.rs"] +pub mod fixture; +#[path = "retention_publication_preparation/initial_laws.rs"] +mod initial_laws; +#[path = "retention_publication_preparation/refusal_laws.rs"] +mod refusal_laws; +#[path = "retention_publication_preparation/successor_laws.rs"] +mod successor_laws; +mod support; diff --git a/tests/retention_publication_preparation/fixture.rs b/tests/retention_publication_preparation/fixture.rs new file mode 100644 index 0000000..f171d23 --- /dev/null +++ b/tests/retention_publication_preparation/fixture.rs @@ -0,0 +1,121 @@ +//! Shared admitted retention publication preparation fixtures. + +use std::error::Error; + +use keep::{ + AdmittedCatalog, AdmittedRetentionRoot, AdmittedSegment, CanonicalRetentionRoot, + CatalogSnapshot, ChecksummedCatalog, ChecksummedPublicationHead, LayoutEntryLimit, + RetentionNamespace, RetentionPolicy, RetentionRoot, SegmentReadPolicy, SegmentRecordLimit, +}; + +use crate::support::decode_hex; + +/// Frozen canonical generation-one root. +pub const ROOT_HEX: &str = include_str!("../../conformance/segment-store/v2/one-anchor-root.hex"); +/// Frozen canonical generation-one manifest. +pub const MANIFEST_HEX: &str = + include_str!("../../conformance/segment-store/v2/one-root-manifest.hex"); +/// Frozen canonical generation-one retention head. +pub const HEAD_HEX: &str = include_str!("../../conformance/segment-store/v2/one-root-head.hex"); +const SEGMENT_HEX: &str = + include_str!("../../conformance/segment-store/v1/one-zero-bundle-segment.hex"); +const CATALOG_HEX: &str = + include_str!("../../conformance/segment-store/v1/one-zero-bundle-catalog.hex"); +const CATALOG_HEAD_HEX: &str = + include_str!("../../conformance/segment-store/v1/one-zero-bundle-head.hex"); + +/// Decodes the frozen root transport. +/// +/// # Errors +/// +/// Returns the exact fixture transport refusal. +pub fn root_bytes() -> Result, Box> { + fixture(ROOT_HEX) +} + +/// Decodes the frozen manifest transport. +/// +/// # Errors +/// +/// Returns the exact fixture transport refusal. +pub fn manifest_bytes() -> Result, Box> { + fixture(MANIFEST_HEX) +} + +/// Builds the exact semantic successor of one admitted root. +/// +/// # Errors +/// +/// Returns the exact generation, semantic-root, or encoding refusal. +pub fn successor_root( + current: &AdmittedRetentionRoot<'_>, +) -> Result> { + let root = RetentionRoot::new( + current.root().namespace().clone(), + current.root().generation().successor()?, + RetentionPolicy::new(current.root().profile(), current.root().limits()), + Some(current.digest()), + current.root().anchors().to_vec(), + )?; + CanonicalRetentionRoot::from_root(&root).map_err(Into::into) +} + +/// Builds a generation-one root for another namespace. +/// +/// # Errors +/// +/// Returns the exact namespace, semantic-root, or encoding refusal. +pub fn initial_root( + namespace: &[u8], + template: &AdmittedRetentionRoot<'_>, +) -> Result> { + let root = RetentionRoot::new( + RetentionNamespace::try_from(namespace)?, + keep::RootGeneration::INITIAL, + RetentionPolicy::new(template.root().profile(), template.root().limits()), + None, + template.root().anchors().to_vec(), + )?; + CanonicalRetentionRoot::from_root(&root).map_err(Into::into) +} + +/// Runs one operation against the frozen one-zero catalog snapshot. +/// +/// # Errors +/// +/// Returns the exact fixture, segment, catalog, head, or snapshot refusal. +pub fn with_snapshot( + operation: impl FnOnce(&CatalogSnapshot<'_, '_, '_>) -> T, +) -> Result> { + let segment_bytes = fixture(SEGMENT_HEX)?; + let catalog_bytes = fixture(CATALOG_HEX)?; + let head_bytes = fixture(CATALOG_HEAD_HEX)?; + let segment = AdmittedSegment::decode(&segment_bytes, maximum_policy())?; + let segments = [segment]; + let catalog = admitted_catalog(&catalog_bytes, &segments)?; + let head = ChecksummedPublicationHead::decode(&head_bytes)?; + let snapshot = head.admit(catalog)?; + Ok(operation(&snapshot)) +} + +/// Decodes one LF-terminated lowercase hexadecimal fixture. +/// +/// # Errors +/// +/// Returns a framing or hexadecimal transport refusal. +pub fn fixture(hex: &str) -> Result, Box> { + decode_hex(hex.strip_suffix('\n').ok_or("fixture must end in one LF")?).map_err(Into::into) +} + +fn admitted_catalog<'catalog, 'records>( + catalog_bytes: &'catalog [u8], + segments: &'records [AdmittedSegment<'records>], +) -> Result, Box> { + ChecksummedCatalog::decode(catalog_bytes)? + .admit(segments) + .map_err(Into::into) +} + +const fn maximum_policy() -> SegmentReadPolicy { + SegmentReadPolicy::new(SegmentRecordLimit::MAXIMUM, LayoutEntryLimit::MAXIMUM) +} diff --git a/tests/retention_publication_preparation/initial_laws.rs b/tests/retention_publication_preparation/initial_laws.rs new file mode 100644 index 0000000..93fbea4 --- /dev/null +++ b/tests/retention_publication_preparation/initial_laws.rs @@ -0,0 +1,35 @@ +//! Initial retention publication preparation laws. + +use std::error::Error; + +use keep::{ + AdmittedRetentionRoot, RetentionGenerationExpectation, preflight_retention_transition, + prepare_retention_publication, +}; + +use super::fixture::{HEAD_HEX, MANIFEST_HEX, fixture, root_bytes, with_snapshot}; + +#[test] +fn initial_preparation_reproduces_frozen_manifest_and_head() -> Result<(), Box> { + let root_bytes = root_bytes()?; + let candidate = AdmittedRetentionRoot::decode(&root_bytes)?; + let preflight = with_snapshot(|snapshot| { + preflight_retention_transition( + RetentionGenerationExpectation::Absent, + None, + candidate, + snapshot, + ) + })??; + + let preparation = prepare_retention_publication(preflight, None)?; + let publication = preparation + .publication() + .ok_or("initial transition did not prepare publication")?; + + assert_eq!(preparation.candidate().encoded(), root_bytes); + assert_eq!(publication.manifest().encoded(), fixture(MANIFEST_HEX)?); + assert_eq!(publication.head().encoded().as_slice(), fixture(HEAD_HEX)?); + assert_eq!(preparation.closure().usage().node_count(), 2); + Ok(()) +} diff --git a/tests/retention_publication_preparation/refusal_laws.rs b/tests/retention_publication_preparation/refusal_laws.rs new file mode 100644 index 0000000..d1b4018 --- /dev/null +++ b/tests/retention_publication_preparation/refusal_laws.rs @@ -0,0 +1,78 @@ +//! Retention publication preparation refusal laws. + +use std::error::Error; + +use keep::{ + AdmittedRetentionManifest, AdmittedRetentionRoot, LivenessGeneration, LivenessGenerationError, + RetentionGenerationExpectation, RetentionManifest, RetentionPublicationPreparationError, + preflight_retention_transition, prepare_retention_publication, +}; + +use super::fixture::{initial_root, manifest_bytes, root_bytes, with_snapshot}; +use crate::support::require_error; + +#[test] +fn manifest_disagreement_refuses_before_global_artifact_construction() -> Result<(), Box> +{ + let root_bytes = root_bytes()?; + let candidate = AdmittedRetentionRoot::decode(&root_bytes)?; + let manifest_bytes = manifest_bytes()?; + let current_manifest = AdmittedRetentionManifest::decode(&manifest_bytes)?; + let preflight = with_snapshot(|snapshot| { + preflight_retention_transition( + RetentionGenerationExpectation::Absent, + None, + candidate, + snapshot, + ) + })??; + + let error = require_error( + prepare_retention_publication(preflight, Some(¤t_manifest)), + "manifest disagreement prepared a publication", + )?; + + assert!(matches!( + error, + RetentionPublicationPreparationError::ManifestSuccessorMismatch { .. } + )); + Ok(()) +} + +#[test] +fn exhausted_liveness_generation_refuses_before_entry_replacement() -> Result<(), Box> { + let root_bytes = root_bytes()?; + let template = AdmittedRetentionRoot::decode(&root_bytes)?; + let manifest_bytes = manifest_bytes()?; + let current_manifest = AdmittedRetentionManifest::decode(&manifest_bytes)?; + let maximum_manifest = RetentionManifest::new( + LivenessGeneration::new(u64::MAX)?, + Some(current_manifest.digest()), + current_manifest.manifest().entries().to_vec(), + )?; + let maximum_bytes = keep::CanonicalRetentionManifest::from_manifest(&maximum_manifest)?; + let maximum = AdmittedRetentionManifest::decode(maximum_bytes.encoded())?; + let candidate_bytes = initial_root(b"exhaustion-candidate", &template)?; + let candidate = AdmittedRetentionRoot::decode(candidate_bytes.encoded())?; + let preflight = with_snapshot(|snapshot| { + preflight_retention_transition( + RetentionGenerationExpectation::Absent, + None, + candidate, + snapshot, + ) + })??; + + let error = require_error( + prepare_retention_publication(preflight, Some(&maximum)), + "exhausted liveness generation prepared a publication", + )?; + + assert!(matches!( + error, + RetentionPublicationPreparationError::LivenessGeneration { + source: LivenessGenerationError::Exhausted { current: u64::MAX } + } + )); + Ok(()) +} diff --git a/tests/retention_publication_preparation/successor_laws.rs b/tests/retention_publication_preparation/successor_laws.rs new file mode 100644 index 0000000..35172c4 --- /dev/null +++ b/tests/retention_publication_preparation/successor_laws.rs @@ -0,0 +1,117 @@ +//! Successor retention publication preparation laws. + +use std::error::Error; + +use keep::{ + AdmittedRetentionManifest, AdmittedRetentionRoot, ChecksummedRetentionHead, + RetentionGenerationExpectation, RootGeneration, preflight_retention_transition, + prepare_retention_publication, +}; + +use super::fixture::{initial_root, manifest_bytes, root_bytes, successor_root, with_snapshot}; + +#[test] +fn successor_replaces_only_the_selected_manifest_entry() -> Result<(), Box> { + let root_bytes = root_bytes()?; + let current = AdmittedRetentionRoot::decode(&root_bytes)?; + let candidate_bytes = successor_root(¤t)?; + let candidate = AdmittedRetentionRoot::decode(candidate_bytes.encoded())?; + let manifest_bytes = manifest_bytes()?; + let current_manifest = AdmittedRetentionManifest::decode(&manifest_bytes)?; + let candidate_digest = candidate.digest(); + let preflight = with_snapshot(|snapshot| { + preflight_retention_transition( + RetentionGenerationExpectation::Current(RootGeneration::INITIAL), + Some(¤t), + candidate, + snapshot, + ) + })??; + + let preparation = prepare_retention_publication(preflight, Some(¤t_manifest))?; + let publication = preparation + .publication() + .ok_or("successor transition did not prepare publication")?; + let manifest = AdmittedRetentionManifest::decode(publication.manifest().encoded())?; + let head = ChecksummedRetentionHead::decode(publication.head().encoded())?; + let entry = manifest + .manifest() + .entries() + .first() + .ok_or("successor manifest omitted the namespace")?; + + assert_eq!(manifest.manifest().generation().get(), 2); + assert_eq!( + manifest.manifest().predecessor(), + Some(current_manifest.digest()) + ); + assert_eq!(entry.root_generation().get(), 2); + assert_eq!(entry.root_digest(), candidate_digest); + assert_eq!(head.head().manifest_digest(), manifest.digest()); + Ok(()) +} + +#[test] +fn new_namespace_is_inserted_without_changing_existing_entry() -> Result<(), Box> { + let root_bytes = root_bytes()?; + let template = AdmittedRetentionRoot::decode(&root_bytes)?; + let candidate_bytes = initial_root(b"second-namespace", &template)?; + let candidate = AdmittedRetentionRoot::decode(candidate_bytes.encoded())?; + let candidate_namespace = candidate.root().namespace().digest(); + let manifest_bytes = manifest_bytes()?; + let current_manifest = AdmittedRetentionManifest::decode(&manifest_bytes)?; + let existing = *current_manifest + .manifest() + .entries() + .first() + .ok_or("fixture manifest omitted its root")?; + let preflight = with_snapshot(|snapshot| { + preflight_retention_transition( + RetentionGenerationExpectation::Absent, + None, + candidate, + snapshot, + ) + })??; + + let preparation = prepare_retention_publication(preflight, Some(¤t_manifest))?; + let publication = preparation + .publication() + .ok_or("new namespace did not prepare publication")?; + let manifest = AdmittedRetentionManifest::decode(publication.manifest().encoded())?; + + assert_eq!(manifest.manifest().entry_count(), 2); + assert!(manifest.manifest().entries().contains(&existing)); + assert!( + manifest + .manifest() + .entries() + .iter() + .any(|entry| entry.namespace() == candidate_namespace) + ); + Ok(()) +} + +#[test] +fn exact_retry_prepares_no_new_global_artifacts() -> Result<(), Box> { + let root_bytes = root_bytes()?; + let current = AdmittedRetentionRoot::decode(&root_bytes)?; + let candidate = AdmittedRetentionRoot::decode(&root_bytes)?; + let manifest_bytes = manifest_bytes()?; + let current_manifest = AdmittedRetentionManifest::decode(&manifest_bytes)?; + let preflight = with_snapshot(|snapshot| { + preflight_retention_transition( + RetentionGenerationExpectation::Absent, + Some(¤t), + candidate, + snapshot, + ) + })??; + + let preparation = prepare_retention_publication(preflight, Some(¤t_manifest))?; + + assert!(preparation.publication().is_none()); + assert_eq!(preparation.candidate().digest(), current.digest()); + assert_eq!(preparation.closure().usage().node_count(), 2); + Ok(()) +} From ef6668478ea181d9a99e21916a162068926bf57b Mon Sep 17 00:00:00 2001 From: James Ross Date: Thu, 30 Jul 2026 00:09:58 -0700 Subject: [PATCH 023/111] Fix: Preserve retention transition coordinates --- CHANGELOG.md | 3 +- README.md | 7 ++-- docs/formats/segment-store-v2/requirements.md | 2 +- docs/formats/segment-store-v2/retention.md | 4 +- .../retention/prepared_publication.rs | 22 ++++++++++- .../retention/publication_preparation.rs | 18 +++++++-- .../retention/transition_preflight.rs | 39 +++++++++++++++++-- tests/retention_preflight.rs | 5 +++ .../successor_laws.rs | 15 +++++++ 9 files changed, 101 insertions(+), 14 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 766e479..91cc9aa 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,7 +14,8 @@ after its public API and format compatibility policies are established. planning with deterministic closure verification against one pinned catalog before any future publication storage call. A typed 17-phase vocabulary and blocking storage port freeze the durability and crash-boundary contract; - preparation binds preflight to exact canonical manifest and head successors. + preparation preserves expected and observed namespace generations while + binding preflight to exact canonical manifest and head successors. - Repository crash-matrix execution now terminates isolated writer process groups at all 105 canonical before/during/after coordinates, retains open writer and stage authority until termination, executes production diff --git a/README.md b/README.md index 38623fb..82d0d2c 100644 --- a/README.md +++ b/README.md @@ -120,9 +120,10 @@ manifest, and retention-head codecs; storage-independent expected-state transition planning; deterministic bounded closure verification against a pinned catalog; a combined transition preflight proof; and the exact 17-phase publication vocabulary with a blocking storage capability port are -implemented. Storage-independent preparation also binds preflight to exact -canonical manifest and head successors. Publication orchestration, filesystem -execution, recovery, compaction, and garbage collection remain planned. +implemented. Storage-independent preparation preserves expected and observed +namespace generations while binding preflight to canonical manifest and head +successors. Publication orchestration, filesystem execution, recovery, +compaction, and garbage collection remain planned. Presence in the reference CAS does not claim durable retention or crash recovery. diff --git a/docs/formats/segment-store-v2/requirements.md b/docs/formats/segment-store-v2/requirements.md index 58b56b1..a747567 100644 --- a/docs/formats/segment-store-v2/requirements.md +++ b/docs/formats/segment-store-v2/requirements.md @@ -12,7 +12,7 @@ case is not evidence. | `KEEP-RETENTION-001` | `RetentionNamespace`, `RootGeneration`, `LivenessGeneration`, profile coordinates, limits, anchors, and digests are validated typed values | `tests/retention_values.rs` and `tests/retention_root_encoding.rs` | Implemented | | `KEEP-RETENTION-002` | Root, manifest, and head codecs implement the exact canonical grammars and fixed bounds | independent golden corpus plus `tests/retention_root_encoding.rs`, `tests/retention_root_decoding.rs`, `tests/retention_manifest_codec.rs`, and `tests/retention_head_codec.rs` | Implemented | | `KEEP-RETENTION-003` | Every structural field, truncation boundary, ordering law, duplicate, overflow, flag, reserved byte, digest, checksum, and trailing byte has a precise refusal | corruption matrix and fuzz | Planned in #19 | -| `KEEP-RETENTION-004` | Retain and release compare expected and observed generations and publish exact successors only | storage-independent planning and generation-before-closure preflight in `tests/retention_transition.rs` and `tests/retention_preflight.rs`; exact global successor preparation in `tests/retention_publication_preparation.rs`; publication evidence remains | In progress in #19 | +| `KEEP-RETENTION-004` | Retain and release compare expected and observed generations and publish exact successors only | storage-independent planning, generation-before-closure preflight, and preserved coordinates in `tests/retention_transition.rs` and `tests/retention_preflight.rs`; exact global successor preparation in `tests/retention_publication_preparation.rs`; publication evidence remains | In progress in #19 | | `KEEP-RETENTION-005` | Closure derivation is deterministic, bounded, cycle-safe, fail-closed, and verifies complete blob reconstruction | exact accounting, reconstruction, adversarial-catalog, and exhaustive model laws in `tests/retention_closure.rs`; corrupt members refuse through the inherited segment-record admission laws and seeded `segment_format` fuzz target routed by `closure-corruption.md` | Implemented | | `KEEP-RETENTION-006` | Publication follows the exact ordered durability protocol, including new namespace-directory admission and retention of fixed-stage evidence until head commit, and returns only after cleanup synchronization | typed 17-phase vocabulary in `tests/retention_publication_phase.rs` and blocking capability port in `tests/retention_publication_storage.rs`; ordered execution and `KEEP-CRASH-036..=052` crash-injection tests remain | In progress in #19 | | `KEEP-RETENTION-007` | Restart resolves every fixed-stage crash prefix to one documented lawful state or typed ambiguity | debug and release crash matrix | Planned in #19 | diff --git a/docs/formats/segment-store-v2/retention.md b/docs/formats/segment-store-v2/retention.md index cb9cc03..1f37cec 100644 --- a/docs/formats/segment-store-v2/retention.md +++ b/docs/formats/segment-store-v2/retention.md @@ -251,8 +251,8 @@ live set. `preflight_retention_transition` combines steps 3 and 4 without I/O, returning publish or already-committed only after generation and closure verification. `prepare_retention_publication` binds that proof to the current manifest, -refuses incoherent root coordinates, and derives exact canonical successors; -exact retry produces no new global artifacts. +preserves expected and observed generations, refuses incoherent coordinates, +and derives exact canonical successors; exact retry creates no global artifacts. Version-2 catalog publication holds the same writer authority and proves every current retained closure against its candidate catalog before replacing the diff --git a/src/adapters/retention/prepared_publication.rs b/src/adapters/retention/prepared_publication.rs index 6eca896..a2a7abb 100644 --- a/src/adapters/retention/prepared_publication.rs +++ b/src/adapters/retention/prepared_publication.rs @@ -4,7 +4,7 @@ use super::{ AdmittedRetentionRoot, CanonicalRetentionHead, CanonicalRetentionManifest, VerifiedRetentionClosure, }; -use crate::LivenessGeneration; +use crate::{LivenessGeneration, RetentionGenerationExpectation, RootGeneration}; /// Canonical global artifacts ready for ordered storage execution. #[must_use = "prepared retention publication must be executed or handled explicitly"] @@ -48,12 +48,24 @@ impl PreparedRetentionPublication { #[must_use = "retention publication preparation must be handled explicitly"] #[derive(Debug)] pub struct RetentionPublicationPreparation<'encoded> { + expected: RetentionGenerationExpectation, + observed: Option, candidate: AdmittedRetentionRoot<'encoded>, closure: VerifiedRetentionClosure, publication: Option, } impl<'encoded> RetentionPublicationPreparation<'encoded> { + /// Returns the caller-supplied expected namespace generation. + pub const fn expected(&self) -> RetentionGenerationExpectation { + self.expected + } + + /// Returns the namespace generation observed during transition planning. + pub const fn observed(&self) -> Option { + self.observed + } + /// Borrows the admitted candidate root. pub const fn candidate(&self) -> &AdmittedRetentionRoot<'encoded> { &self.candidate @@ -70,11 +82,15 @@ impl<'encoded> RetentionPublicationPreparation<'encoded> { } pub(super) const fn publish( + expected: RetentionGenerationExpectation, + observed: Option, candidate: AdmittedRetentionRoot<'encoded>, closure: VerifiedRetentionClosure, publication: PreparedRetentionPublication, ) -> Self { Self { + expected, + observed, candidate, closure, publication: Some(publication), @@ -82,10 +98,14 @@ impl<'encoded> RetentionPublicationPreparation<'encoded> { } pub(super) const fn already_committed( + expected: RetentionGenerationExpectation, + observed: Option, candidate: AdmittedRetentionRoot<'encoded>, closure: VerifiedRetentionClosure, ) -> Self { Self { + expected, + observed, candidate, closure, publication: None, diff --git a/src/adapters/retention/publication_preparation.rs b/src/adapters/retention/publication_preparation.rs index da1b6ee..aeae9d6 100644 --- a/src/adapters/retention/publication_preparation.rs +++ b/src/adapters/retention/publication_preparation.rs @@ -23,13 +23,23 @@ pub fn prepare_retention_publication<'encoded>( current_manifest: Option<&AdmittedRetentionManifest<'_>>, ) -> Result, RetentionPublicationPreparationError> { match preflight { - RetentionTransitionPreflight::AlreadyCommitted { candidate, closure } => { + RetentionTransitionPreflight::AlreadyCommitted { + expected, + observed, + candidate, + closure, + } => { successor_manifest::require_current_selection(&candidate, current_manifest)?; Ok(RetentionPublicationPreparation::already_committed( - candidate, closure, + expected, observed, candidate, closure, )) } - RetentionTransitionPreflight::Publish { candidate, closure } => { + RetentionTransitionPreflight::Publish { + expected, + observed, + candidate, + closure, + } => { let semantic_manifest = successor_manifest::build(&candidate, current_manifest)?; let liveness_generation = semantic_manifest.generation(); let predecessor = semantic_manifest.predecessor(); @@ -48,6 +58,8 @@ pub fn prepare_retention_publication<'encoded>( let publication = PreparedRetentionPublication::new(manifest, head, liveness_generation); Ok(RetentionPublicationPreparation::publish( + expected, + observed, candidate, closure, publication, diff --git a/src/adapters/retention/transition_preflight.rs b/src/adapters/retention/transition_preflight.rs index 06992b6..ae9c0e7 100644 --- a/src/adapters/retention/transition_preflight.rs +++ b/src/adapters/retention/transition_preflight.rs @@ -4,8 +4,8 @@ use super::{ AdmittedRetentionRoot, RetentionTransitionPreflightError, RetentionTransitionReadiness, VerifiedRetentionClosure, plan_retention_transition, verify_retention_closure, }; -use crate::CatalogSnapshot; use crate::retention::RetentionGenerationExpectation; +use crate::{CatalogSnapshot, RootGeneration}; /// Complete storage-independent proof required before retention publication. #[must_use = "retention preflight must be consumed by publication or handled explicitly"] @@ -13,6 +13,10 @@ use crate::retention::RetentionGenerationExpectation; pub enum RetentionTransitionPreflight<'encoded> { /// The candidate is an exact successor whose verified closure must publish. Publish { + /// Caller-supplied expected namespace generation. + expected: RetentionGenerationExpectation, + /// Namespace generation observed during transition planning. + observed: Option, /// Fully admitted canonical candidate root. candidate: AdmittedRetentionRoot<'encoded>, /// Closure proof against the exact pinned catalog. @@ -20,6 +24,10 @@ pub enum RetentionTransitionPreflight<'encoded> { }, /// The exact candidate is current and its closure still verifies. AlreadyCommitted { + /// Caller-supplied expected namespace generation. + expected: RetentionGenerationExpectation, + /// Namespace generation observed during transition planning. + observed: Option, /// Fully admitted byte-identical current root. candidate: AdmittedRetentionRoot<'encoded>, /// Current closure proof against the exact pinned catalog. @@ -28,6 +36,20 @@ pub enum RetentionTransitionPreflight<'encoded> { } impl<'encoded> RetentionTransitionPreflight<'encoded> { + /// Returns the caller-supplied expected namespace generation. + pub const fn expected(&self) -> RetentionGenerationExpectation { + match self { + Self::Publish { expected, .. } | Self::AlreadyCommitted { expected, .. } => *expected, + } + } + + /// Returns the namespace generation observed during transition planning. + pub const fn observed(&self) -> Option { + match self { + Self::Publish { observed, .. } | Self::AlreadyCommitted { observed, .. } => *observed, + } + } + /// Borrows the fully admitted candidate root. pub const fn candidate(&self) -> &AdmittedRetentionRoot<'encoded> { match self { @@ -61,6 +83,7 @@ pub fn preflight_retention_transition<'encoded>( candidate: AdmittedRetentionRoot<'encoded>, catalog: &CatalogSnapshot<'_, '_, '_>, ) -> Result, RetentionTransitionPreflightError> { + let observed = current.map(|root| root.root().generation()); let readiness = plan_retention_transition(expected, current, candidate) .map_err(|source| RetentionTransitionPreflightError::Transition { source })?; let closure = @@ -71,10 +94,20 @@ pub fn preflight_retention_transition<'encoded>( })?; Ok(match readiness { RetentionTransitionReadiness::Publish { candidate } => { - RetentionTransitionPreflight::Publish { candidate, closure } + RetentionTransitionPreflight::Publish { + expected, + observed, + candidate, + closure, + } } RetentionTransitionReadiness::AlreadyCommitted { candidate } => { - RetentionTransitionPreflight::AlreadyCommitted { candidate, closure } + RetentionTransitionPreflight::AlreadyCommitted { + expected, + observed, + candidate, + closure, + } } }) } diff --git a/tests/retention_preflight.rs b/tests/retention_preflight.rs index 7eee6b5..774e098 100644 --- a/tests/retention_preflight.rs +++ b/tests/retention_preflight.rs @@ -44,11 +44,14 @@ fn publish_preflight_binds_generation_and_closure_proofs() -> Result<(), Box Result<(), Box Result<(), Box Result<(), Box })??; let preparation = prepare_retention_publication(preflight, Some(¤t_manifest))?; + assert_eq!( + preparation.expected(), + RetentionGenerationExpectation::Absent + ); + assert_eq!(preparation.observed(), None); let publication = preparation .publication() .ok_or("new namespace did not prepare publication")?; @@ -110,6 +120,11 @@ fn exact_retry_prepares_no_new_global_artifacts() -> Result<(), Box> let preparation = prepare_retention_publication(preflight, Some(¤t_manifest))?; + assert_eq!( + preparation.expected(), + RetentionGenerationExpectation::Absent + ); + assert_eq!(preparation.observed(), Some(RootGeneration::INITIAL)); assert!(preparation.publication().is_none()); assert_eq!(preparation.candidate().digest(), current.digest()); assert_eq!(preparation.closure().usage().node_count(), 2); From 78b0c54a047de562ef4fa914a0bc89db860b37f7 Mon Sep 17 00:00:00 2001 From: James Ross Date: Thu, 30 Jul 2026 00:24:29 -0700 Subject: [PATCH 024/111] Fix: Seal retention transition proofs --- CHANGELOG.md | 4 +- README.md | 8 +- docs/formats/segment-store-v2/requirements.md | 2 +- docs/formats/segment-store-v2/retention.md | 2 +- src/adapters/retention.rs | 2 + .../retention/prepared_publication.rs | 10 +- .../retention/publication_preparation.rs | 20 ++-- .../retention/transition_disposition.rs | 11 ++ src/adapters/retention/transition_planner.rs | 9 +- .../retention/transition_preflight.rs | 100 ++++++++---------- .../retention/transition_readiness.rs | 84 +++++++++++---- src/lib.rs | 8 +- tests/retention_preflight.rs | 30 +++--- .../successor_laws.rs | 16 ++- tests/retention_transition.rs | 44 +++++--- 15 files changed, 213 insertions(+), 137 deletions(-) create mode 100644 src/adapters/retention/transition_disposition.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index 91cc9aa..c0aedd9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,8 +14,8 @@ after its public API and format compatibility policies are established. planning with deterministic closure verification against one pinned catalog before any future publication storage call. A typed 17-phase vocabulary and blocking storage port freeze the durability and crash-boundary contract; - preparation preserves expected and observed namespace generations while - binding preflight to exact canonical manifest and head successors. + unforgeable proof values preserve typed disposition plus expected and + observed generations through exact manifest and head preparation. - Repository crash-matrix execution now terminates isolated writer process groups at all 105 canonical before/during/after coordinates, retains open writer and stage authority until termination, executes production diff --git a/README.md b/README.md index 82d0d2c..66bfd14 100644 --- a/README.md +++ b/README.md @@ -120,10 +120,10 @@ manifest, and retention-head codecs; storage-independent expected-state transition planning; deterministic bounded closure verification against a pinned catalog; a combined transition preflight proof; and the exact 17-phase publication vocabulary with a blocking storage capability port are -implemented. Storage-independent preparation preserves expected and observed -namespace generations while binding preflight to canonical manifest and head -successors. Publication orchestration, filesystem execution, recovery, -compaction, and garbage collection remain planned. +implemented. Private-field proofs preserve typed disposition plus expected and +observed namespace generations through canonical manifest and head preparation. +Publication orchestration, filesystem execution, recovery, compaction, and +garbage collection remain planned. Presence in the reference CAS does not claim durable retention or crash recovery. diff --git a/docs/formats/segment-store-v2/requirements.md b/docs/formats/segment-store-v2/requirements.md index a747567..7fe3852 100644 --- a/docs/formats/segment-store-v2/requirements.md +++ b/docs/formats/segment-store-v2/requirements.md @@ -12,7 +12,7 @@ case is not evidence. | `KEEP-RETENTION-001` | `RetentionNamespace`, `RootGeneration`, `LivenessGeneration`, profile coordinates, limits, anchors, and digests are validated typed values | `tests/retention_values.rs` and `tests/retention_root_encoding.rs` | Implemented | | `KEEP-RETENTION-002` | Root, manifest, and head codecs implement the exact canonical grammars and fixed bounds | independent golden corpus plus `tests/retention_root_encoding.rs`, `tests/retention_root_decoding.rs`, `tests/retention_manifest_codec.rs`, and `tests/retention_head_codec.rs` | Implemented | | `KEEP-RETENTION-003` | Every structural field, truncation boundary, ordering law, duplicate, overflow, flag, reserved byte, digest, checksum, and trailing byte has a precise refusal | corruption matrix and fuzz | Planned in #19 | -| `KEEP-RETENTION-004` | Retain and release compare expected and observed generations and publish exact successors only | storage-independent planning, generation-before-closure preflight, and preserved coordinates in `tests/retention_transition.rs` and `tests/retention_preflight.rs`; exact global successor preparation in `tests/retention_publication_preparation.rs`; publication evidence remains | In progress in #19 | +| `KEEP-RETENTION-004` | Retain and release compare expected and observed generations and publish exact successors only | unforgeable storage-independent readiness and preflight proofs with preserved disposition and coordinates in `tests/retention_transition.rs` and `tests/retention_preflight.rs`; exact global successor preparation in `tests/retention_publication_preparation.rs`; publication evidence remains | In progress in #19 | | `KEEP-RETENTION-005` | Closure derivation is deterministic, bounded, cycle-safe, fail-closed, and verifies complete blob reconstruction | exact accounting, reconstruction, adversarial-catalog, and exhaustive model laws in `tests/retention_closure.rs`; corrupt members refuse through the inherited segment-record admission laws and seeded `segment_format` fuzz target routed by `closure-corruption.md` | Implemented | | `KEEP-RETENTION-006` | Publication follows the exact ordered durability protocol, including new namespace-directory admission and retention of fixed-stage evidence until head commit, and returns only after cleanup synchronization | typed 17-phase vocabulary in `tests/retention_publication_phase.rs` and blocking capability port in `tests/retention_publication_storage.rs`; ordered execution and `KEEP-CRASH-036..=052` crash-injection tests remain | In progress in #19 | | `KEEP-RETENTION-007` | Restart resolves every fixed-stage crash prefix to one documented lawful state or typed ambiguity | debug and release crash matrix | Planned in #19 | diff --git a/docs/formats/segment-store-v2/retention.md b/docs/formats/segment-store-v2/retention.md index 1f37cec..b04b150 100644 --- a/docs/formats/segment-store-v2/retention.md +++ b/docs/formats/segment-store-v2/retention.md @@ -249,7 +249,7 @@ transition. Keep never omits one failed member and continues with a smaller live set. `preflight_retention_transition` combines steps 3 and 4 without I/O, returning -publish or already-committed only after generation and closure verification. +an unforgeable typed disposition only after generation and closure verification. `prepare_retention_publication` binds that proof to the current manifest, preserves expected and observed generations, refuses incoherent coordinates, and derives exact canonical successors; exact retry creates no global artifacts. diff --git a/src/adapters/retention.rs b/src/adapters/retention.rs index addcd7c..4978032 100644 --- a/src/adapters/retention.rs +++ b/src/adapters/retention.rs @@ -45,6 +45,7 @@ mod root_header_decoder; mod root_integrity; mod root_semantic_header; mod successor_manifest; +mod transition_disposition; mod transition_error; mod transition_planner; mod transition_preflight; @@ -71,6 +72,7 @@ pub use publication_preparation_error::RetentionPublicationPreparationError; pub use publication_storage::RetentionPublicationStorage; pub use root_decode_error::RetentionRootDecodeError; pub use root_encode_error::RetentionRootEncodeError; +pub use transition_disposition::RetentionTransitionDisposition; pub use transition_error::RetentionTransitionError; pub use transition_planner::plan_retention_transition; pub use transition_preflight::{RetentionTransitionPreflight, preflight_retention_transition}; diff --git a/src/adapters/retention/prepared_publication.rs b/src/adapters/retention/prepared_publication.rs index a2a7abb..d534fa6 100644 --- a/src/adapters/retention/prepared_publication.rs +++ b/src/adapters/retention/prepared_publication.rs @@ -2,7 +2,7 @@ use super::{ AdmittedRetentionRoot, CanonicalRetentionHead, CanonicalRetentionManifest, - VerifiedRetentionClosure, + RetentionTransitionDisposition, VerifiedRetentionClosure, }; use crate::{LivenessGeneration, RetentionGenerationExpectation, RootGeneration}; @@ -48,6 +48,7 @@ impl PreparedRetentionPublication { #[must_use = "retention publication preparation must be handled explicitly"] #[derive(Debug)] pub struct RetentionPublicationPreparation<'encoded> { + disposition: RetentionTransitionDisposition, expected: RetentionGenerationExpectation, observed: Option, candidate: AdmittedRetentionRoot<'encoded>, @@ -56,6 +57,11 @@ pub struct RetentionPublicationPreparation<'encoded> { } impl<'encoded> RetentionPublicationPreparation<'encoded> { + /// Returns whether the candidate requires publication or is current. + pub const fn disposition(&self) -> RetentionTransitionDisposition { + self.disposition + } + /// Returns the caller-supplied expected namespace generation. pub const fn expected(&self) -> RetentionGenerationExpectation { self.expected @@ -89,6 +95,7 @@ impl<'encoded> RetentionPublicationPreparation<'encoded> { publication: PreparedRetentionPublication, ) -> Self { Self { + disposition: RetentionTransitionDisposition::Publish, expected, observed, candidate, @@ -104,6 +111,7 @@ impl<'encoded> RetentionPublicationPreparation<'encoded> { closure: VerifiedRetentionClosure, ) -> Self { Self { + disposition: RetentionTransitionDisposition::AlreadyCommitted, expected, observed, candidate, diff --git a/src/adapters/retention/publication_preparation.rs b/src/adapters/retention/publication_preparation.rs index aeae9d6..01680f1 100644 --- a/src/adapters/retention/publication_preparation.rs +++ b/src/adapters/retention/publication_preparation.rs @@ -3,7 +3,8 @@ use super::{ AdmittedRetentionManifest, CanonicalRetentionHead, CanonicalRetentionManifest, PreparedRetentionPublication, RetentionPublicationPreparation, - RetentionPublicationPreparationError, RetentionTransitionPreflight, successor_manifest, + RetentionPublicationPreparationError, RetentionTransitionDisposition, + RetentionTransitionPreflight, successor_manifest, }; use crate::{RetentionHead, RetentionManifestLength}; @@ -22,24 +23,15 @@ pub fn prepare_retention_publication<'encoded>( preflight: RetentionTransitionPreflight<'encoded>, current_manifest: Option<&AdmittedRetentionManifest<'_>>, ) -> Result, RetentionPublicationPreparationError> { - match preflight { - RetentionTransitionPreflight::AlreadyCommitted { - expected, - observed, - candidate, - closure, - } => { + let (disposition, expected, observed, candidate, closure) = preflight.into_parts(); + match disposition { + RetentionTransitionDisposition::AlreadyCommitted => { successor_manifest::require_current_selection(&candidate, current_manifest)?; Ok(RetentionPublicationPreparation::already_committed( expected, observed, candidate, closure, )) } - RetentionTransitionPreflight::Publish { - expected, - observed, - candidate, - closure, - } => { + RetentionTransitionDisposition::Publish => { let semantic_manifest = successor_manifest::build(&candidate, current_manifest)?; let liveness_generation = semantic_manifest.generation(); let predecessor = semantic_manifest.predecessor(); diff --git a/src/adapters/retention/transition_disposition.rs b/src/adapters/retention/transition_disposition.rs new file mode 100644 index 0000000..257a36b --- /dev/null +++ b/src/adapters/retention/transition_disposition.rs @@ -0,0 +1,11 @@ +//! This boundary module owns retention transition disposition vocabulary. + +/// Storage-independent result of one admitted retention transition comparison. +#[must_use] +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum RetentionTransitionDisposition { + /// The candidate is an exact successor that still requires publication. + Publish, + /// The byte-identical candidate is already the selected current root. + AlreadyCommitted, +} diff --git a/src/adapters/retention/transition_planner.rs b/src/adapters/retention/transition_planner.rs index 894e0a4..d25e83e 100644 --- a/src/adapters/retention/transition_planner.rs +++ b/src/adapters/retention/transition_planner.rs @@ -20,15 +20,20 @@ pub fn plan_retention_transition<'encoded>( current: Option<&AdmittedRetentionRoot<'_>>, candidate: AdmittedRetentionRoot<'encoded>, ) -> Result, RetentionTransitionError> { + let observed = current.map(|root| root.root().generation()); if is_exact_replay(expected, current, &candidate)? { - return Ok(RetentionTransitionReadiness::AlreadyCommitted { candidate }); + return Ok(RetentionTransitionReadiness::already_committed( + expected, observed, candidate, + )); } require_expected_state(expected, current)?; match current { Some(current) => validate_successor(current, &candidate)?, None => validate_initial(&candidate)?, } - Ok(RetentionTransitionReadiness::Publish { candidate }) + Ok(RetentionTransitionReadiness::publish( + expected, observed, candidate, + )) } fn is_exact_replay( diff --git a/src/adapters/retention/transition_preflight.rs b/src/adapters/retention/transition_preflight.rs index ae9c0e7..de36960 100644 --- a/src/adapters/retention/transition_preflight.rs +++ b/src/adapters/retention/transition_preflight.rs @@ -1,67 +1,64 @@ //! This boundary module owns complete retention transition preflight. use super::{ - AdmittedRetentionRoot, RetentionTransitionPreflightError, RetentionTransitionReadiness, + AdmittedRetentionRoot, RetentionTransitionDisposition, RetentionTransitionPreflightError, VerifiedRetentionClosure, plan_retention_transition, verify_retention_closure, }; -use crate::retention::RetentionGenerationExpectation; -use crate::{CatalogSnapshot, RootGeneration}; +use crate::{CatalogSnapshot, RetentionGenerationExpectation, RootGeneration}; -/// Complete storage-independent proof required before retention publication. +/// Unforgeable storage-independent proof required before publication. #[must_use = "retention preflight must be consumed by publication or handled explicitly"] #[derive(Debug)] -pub enum RetentionTransitionPreflight<'encoded> { - /// The candidate is an exact successor whose verified closure must publish. - Publish { - /// Caller-supplied expected namespace generation. - expected: RetentionGenerationExpectation, - /// Namespace generation observed during transition planning. - observed: Option, - /// Fully admitted canonical candidate root. - candidate: AdmittedRetentionRoot<'encoded>, - /// Closure proof against the exact pinned catalog. - closure: VerifiedRetentionClosure, - }, - /// The exact candidate is current and its closure still verifies. - AlreadyCommitted { - /// Caller-supplied expected namespace generation. - expected: RetentionGenerationExpectation, - /// Namespace generation observed during transition planning. - observed: Option, - /// Fully admitted byte-identical current root. - candidate: AdmittedRetentionRoot<'encoded>, - /// Current closure proof against the exact pinned catalog. - closure: VerifiedRetentionClosure, - }, +pub struct RetentionTransitionPreflight<'encoded> { + disposition: RetentionTransitionDisposition, + expected: RetentionGenerationExpectation, + observed: Option, + candidate: AdmittedRetentionRoot<'encoded>, + closure: VerifiedRetentionClosure, } impl<'encoded> RetentionTransitionPreflight<'encoded> { + /// Returns whether the candidate requires publication or is current. + pub const fn disposition(&self) -> RetentionTransitionDisposition { + self.disposition + } + /// Returns the caller-supplied expected namespace generation. pub const fn expected(&self) -> RetentionGenerationExpectation { - match self { - Self::Publish { expected, .. } | Self::AlreadyCommitted { expected, .. } => *expected, - } + self.expected } /// Returns the namespace generation observed during transition planning. pub const fn observed(&self) -> Option { - match self { - Self::Publish { observed, .. } | Self::AlreadyCommitted { observed, .. } => *observed, - } + self.observed } /// Borrows the fully admitted candidate root. pub const fn candidate(&self) -> &AdmittedRetentionRoot<'encoded> { - match self { - Self::Publish { candidate, .. } | Self::AlreadyCommitted { candidate, .. } => candidate, - } + &self.candidate } /// Returns the complete verified closure evidence. pub const fn closure(&self) -> VerifiedRetentionClosure { - match self { - Self::Publish { closure, .. } | Self::AlreadyCommitted { closure, .. } => *closure, - } + self.closure + } + + pub(super) fn into_parts( + self, + ) -> ( + RetentionTransitionDisposition, + RetentionGenerationExpectation, + Option, + AdmittedRetentionRoot<'encoded>, + VerifiedRetentionClosure, + ) { + ( + self.disposition, + self.expected, + self.observed, + self.candidate, + self.closure, + ) } } @@ -83,7 +80,6 @@ pub fn preflight_retention_transition<'encoded>( candidate: AdmittedRetentionRoot<'encoded>, catalog: &CatalogSnapshot<'_, '_, '_>, ) -> Result, RetentionTransitionPreflightError> { - let observed = current.map(|root| root.root().generation()); let readiness = plan_retention_transition(expected, current, candidate) .map_err(|source| RetentionTransitionPreflightError::Transition { source })?; let closure = @@ -92,22 +88,12 @@ pub fn preflight_retention_transition<'encoded>( source: Box::new(source), } })?; - Ok(match readiness { - RetentionTransitionReadiness::Publish { candidate } => { - RetentionTransitionPreflight::Publish { - expected, - observed, - candidate, - closure, - } - } - RetentionTransitionReadiness::AlreadyCommitted { candidate } => { - RetentionTransitionPreflight::AlreadyCommitted { - expected, - observed, - candidate, - closure, - } - } + let (disposition, expected, observed, candidate) = readiness.into_parts(); + Ok(RetentionTransitionPreflight { + disposition, + expected, + observed, + candidate, + closure, }) } diff --git a/src/adapters/retention/transition_readiness.rs b/src/adapters/retention/transition_readiness.rs index 2231f15..0fa9461 100644 --- a/src/adapters/retention/transition_readiness.rs +++ b/src/adapters/retention/transition_readiness.rs @@ -1,35 +1,83 @@ //! This boundary module owns admitted retention transition readiness. -use super::AdmittedRetentionRoot; +use super::{AdmittedRetentionRoot, RetentionTransitionDisposition}; +use crate::{RetentionGenerationExpectation, RootGeneration}; -/// Result of comparing one expected, observed, and candidate root. +/// Unforgeable result of comparing expected, observed, and candidate state. #[must_use] #[derive(Debug, Eq, PartialEq)] -pub enum RetentionTransitionReadiness<'encoded> { - /// The candidate is the exact next root and still requires publication. - Publish { - /// Fully admitted candidate root. - candidate: AdmittedRetentionRoot<'encoded>, - }, - /// The exact candidate bytes are already the current published root. - AlreadyCommitted { - /// Fully admitted byte-identical replay candidate. - candidate: AdmittedRetentionRoot<'encoded>, - }, +pub struct RetentionTransitionReadiness<'encoded> { + disposition: RetentionTransitionDisposition, + expected: RetentionGenerationExpectation, + observed: Option, + candidate: AdmittedRetentionRoot<'encoded>, } impl<'encoded> RetentionTransitionReadiness<'encoded> { + /// Returns whether the candidate requires publication or is current. + pub const fn disposition(&self) -> RetentionTransitionDisposition { + self.disposition + } + + /// Returns the caller-supplied expected namespace generation. + pub const fn expected(&self) -> RetentionGenerationExpectation { + self.expected + } + + /// Returns the namespace generation observed during transition planning. + pub const fn observed(&self) -> Option { + self.observed + } + /// Borrows the fully admitted candidate root. pub const fn candidate(&self) -> &AdmittedRetentionRoot<'encoded> { - match self { - Self::Publish { candidate } | Self::AlreadyCommitted { candidate } => candidate, - } + &self.candidate } /// Consumes the readiness proof and returns the admitted candidate root. pub fn into_candidate(self) -> AdmittedRetentionRoot<'encoded> { - match self { - Self::Publish { candidate } | Self::AlreadyCommitted { candidate } => candidate, + self.candidate + } + + pub(super) const fn publish( + expected: RetentionGenerationExpectation, + observed: Option, + candidate: AdmittedRetentionRoot<'encoded>, + ) -> Self { + Self { + disposition: RetentionTransitionDisposition::Publish, + expected, + observed, + candidate, } } + + pub(super) const fn already_committed( + expected: RetentionGenerationExpectation, + observed: Option, + candidate: AdmittedRetentionRoot<'encoded>, + ) -> Self { + Self { + disposition: RetentionTransitionDisposition::AlreadyCommitted, + expected, + observed, + candidate, + } + } + + pub(super) fn into_parts( + self, + ) -> ( + RetentionTransitionDisposition, + RetentionGenerationExpectation, + Option, + AdmittedRetentionRoot<'encoded>, + ) { + ( + self.disposition, + self.expected, + self.observed, + self.candidate, + ) + } } diff --git a/src/lib.rs b/src/lib.rs index 624ef17..adbbffd 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -114,10 +114,10 @@ pub use adapters::{ RetentionManifestDecodeError, RetentionManifestEncodeError, RetentionNamespaceAdmission, RetentionPublicationPhase, RetentionPublicationPreparation, RetentionPublicationPreparationError, RetentionPublicationStorage, RetentionRootDecodeError, - RetentionRootEncodeError, RetentionTransitionError, RetentionTransitionPreflight, - RetentionTransitionPreflightError, RetentionTransitionReadiness, VerifiedRetentionClosure, - plan_retention_transition, preflight_retention_transition, prepare_retention_publication, - verify_retention_closure, + RetentionRootEncodeError, RetentionTransitionDisposition, RetentionTransitionError, + RetentionTransitionPreflight, RetentionTransitionPreflightError, RetentionTransitionReadiness, + VerifiedRetentionClosure, plan_retention_transition, preflight_retention_transition, + prepare_retention_publication, verify_retention_closure, }; pub use blob::{ BlobHashError, BlobHasher, BlobId, BlobLength, BlobReadError, ByteLength, ByteOffset, diff --git a/tests/retention_preflight.rs b/tests/retention_preflight.rs index 774e098..510a5b8 100644 --- a/tests/retention_preflight.rs +++ b/tests/retention_preflight.rs @@ -7,7 +7,7 @@ use std::error::Error; use keep::{ AdmittedCatalog, AdmittedRetentionRoot, AdmittedSegment, CatalogSnapshot, ChecksummedCatalog, ChecksummedPublicationHead, LayoutEntryLimit, RetentionClosureVerificationError, - RetentionGenerationExpectation, RetentionTransitionError, RetentionTransitionPreflight, + RetentionGenerationExpectation, RetentionTransitionDisposition, RetentionTransitionError, RetentionTransitionPreflightError, RootGeneration, SegmentReadPolicy, SegmentRecordIdentity, SegmentRecordLimit, preflight_retention_transition, }; @@ -46,15 +46,15 @@ fn publish_preflight_binds_generation_and_closure_proofs() -> Result<(), Box Result<(), Box Result<(), Box Result<(), Box RetentionGenerationExpectation::Absent ); assert_eq!(preparation.observed(), None); + assert_eq!( + preparation.disposition(), + RetentionTransitionDisposition::Publish + ); let publication = preparation .publication() .ok_or("new namespace did not prepare publication")?; @@ -125,6 +133,10 @@ fn exact_retry_prepares_no_new_global_artifacts() -> Result<(), Box> RetentionGenerationExpectation::Absent ); assert_eq!(preparation.observed(), Some(RootGeneration::INITIAL)); + assert_eq!( + preparation.disposition(), + RetentionTransitionDisposition::AlreadyCommitted + ); assert!(preparation.publication().is_none()); assert_eq!(preparation.candidate().digest(), current.digest()); assert_eq!(preparation.closure().usage().node_count(), 2); diff --git a/tests/retention_transition.rs b/tests/retention_transition.rs index a673e45..fc8064e 100644 --- a/tests/retention_transition.rs +++ b/tests/retention_transition.rs @@ -8,7 +8,7 @@ use std::io; use keep::{ AdmittedRetentionRoot, CanonicalRetentionRoot, RetentionGenerationExpectation, - RetentionNamespace, RetentionRoot, RetentionRootDigest, RetentionTransitionReadiness, + RetentionNamespace, RetentionRoot, RetentionRootDigest, RetentionTransitionDisposition, RootGeneration, plan_retention_transition, }; @@ -20,11 +20,13 @@ fn absent_namespace_admits_only_the_initial_candidate() -> Result<(), Box Date: Thu, 30 Jul 2026 00:34:55 -0700 Subject: [PATCH 025/111] Add: Expose verified retention anchor digest --- CHANGELOG.md | 4 ++-- README.md | 8 ++++---- docs/formats/segment-store-v2/requirements.md | 2 +- docs/formats/segment-store-v2/retention.md | 6 +++--- src/adapters/retention/admitted_root.rs | 10 +++++++++- src/adapters/retention/root_decoder.rs | 7 ++++++- src/adapters/retention/root_integrity.rs | 5 +++-- src/lib.rs | 8 ++++---- src/retention/anchor_set_digest.rs | 18 ++++++++++++++++++ src/retention/mod.rs | 2 ++ tests/retention_root_decoding.rs | 7 +++++++ 11 files changed, 59 insertions(+), 18 deletions(-) create mode 100644 src/retention/anchor_set_digest.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index c0aedd9..e45a7c4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,8 +14,8 @@ after its public API and format compatibility policies are established. planning with deterministic closure verification against one pinned catalog before any future publication storage call. A typed 17-phase vocabulary and blocking storage port freeze the durability and crash-boundary contract; - unforgeable proof values preserve typed disposition plus expected and - observed generations through exact manifest and head preparation. + unforgeable proofs preserve disposition, expected and observed generations, + and the verified anchor-set digest through manifest and head preparation. - Repository crash-matrix execution now terminates isolated writer process groups at all 105 canonical before/during/after coordinates, retains open writer and stage authority until termination, executes production diff --git a/README.md b/README.md index 66bfd14..8cf827f 100644 --- a/README.md +++ b/README.md @@ -120,10 +120,10 @@ manifest, and retention-head codecs; storage-independent expected-state transition planning; deterministic bounded closure verification against a pinned catalog; a combined transition preflight proof; and the exact 17-phase publication vocabulary with a blocking storage capability port are -implemented. Private-field proofs preserve typed disposition plus expected and -observed namespace generations through canonical manifest and head preparation. -Publication orchestration, filesystem execution, recovery, compaction, and -garbage collection remain planned. +implemented. Private-field proofs preserve typed disposition, expected and +observed namespace generations, and the verified anchor-set digest through +canonical manifest and head preparation. Publication orchestration, filesystem +execution, recovery, compaction, and garbage collection remain planned. Presence in the reference CAS does not claim durable retention or crash recovery. diff --git a/docs/formats/segment-store-v2/requirements.md b/docs/formats/segment-store-v2/requirements.md index 7fe3852..41ba7b1 100644 --- a/docs/formats/segment-store-v2/requirements.md +++ b/docs/formats/segment-store-v2/requirements.md @@ -9,7 +9,7 @@ case is not evidence. | ID | Requirement | Evidence | Status | | --- | --- | --- | --- | -| `KEEP-RETENTION-001` | `RetentionNamespace`, `RootGeneration`, `LivenessGeneration`, profile coordinates, limits, anchors, and digests are validated typed values | `tests/retention_values.rs` and `tests/retention_root_encoding.rs` | Implemented | +| `KEEP-RETENTION-001` | `RetentionNamespace`, `RootGeneration`, `LivenessGeneration`, profile coordinates, limits, anchors, and digests are validated typed values | `tests/retention_values.rs`, `tests/retention_root_encoding.rs`, and typed verified anchor-set evidence in `tests/retention_root_decoding.rs` | Implemented | | `KEEP-RETENTION-002` | Root, manifest, and head codecs implement the exact canonical grammars and fixed bounds | independent golden corpus plus `tests/retention_root_encoding.rs`, `tests/retention_root_decoding.rs`, `tests/retention_manifest_codec.rs`, and `tests/retention_head_codec.rs` | Implemented | | `KEEP-RETENTION-003` | Every structural field, truncation boundary, ordering law, duplicate, overflow, flag, reserved byte, digest, checksum, and trailing byte has a precise refusal | corruption matrix and fuzz | Planned in #19 | | `KEEP-RETENTION-004` | Retain and release compare expected and observed generations and publish exact successors only | unforgeable storage-independent readiness and preflight proofs with preserved disposition and coordinates in `tests/retention_transition.rs` and `tests/retention_preflight.rs`; exact global successor preparation in `tests/retention_publication_preparation.rs`; publication evidence remains | In progress in #19 | diff --git a/docs/formats/segment-store-v2/retention.md b/docs/formats/segment-store-v2/retention.md index b04b150..dddb002 100644 --- a/docs/formats/segment-store-v2/retention.md +++ b/docs/formats/segment-store-v2/retention.md @@ -160,9 +160,9 @@ retention/roots// ``` Names with alternate width, case, suffix, generation, or digest refuse. Keep -implements validated in-memory root, manifest, and head codecs, -storage-independent expected-state transition planning, deterministic closure -verification, and a blocking publication storage capability port. Publication +implements root, manifest, and head codecs with a typed verified anchor-set +digest, expected-state transition planning, deterministic closure verification, +and a blocking publication storage capability port. Publication orchestration, filesystem execution, recovery, and garbage collection remain absent. diff --git a/src/adapters/retention/admitted_root.rs b/src/adapters/retention/admitted_root.rs index ac0f33f..952b690 100644 --- a/src/adapters/retention/admitted_root.rs +++ b/src/adapters/retention/admitted_root.rs @@ -1,7 +1,7 @@ //! This boundary module owns one decoded and admitted retention root. use super::{RetentionRootDecodeError, root_decoder}; -use crate::{RetentionRoot, RetentionRootDigest}; +use crate::{RetentionAnchorSetDigest, RetentionRoot, RetentionRootDigest}; /// Borrowed canonical bytes paired with their admitted semantic root. /// @@ -14,6 +14,7 @@ use crate::{RetentionRoot, RetentionRootDigest}; pub struct AdmittedRetentionRoot<'encoded> { encoded: &'encoded [u8], root: RetentionRoot, + anchor_set_digest: RetentionAnchorSetDigest, digest: RetentionRootDigest, } @@ -39,6 +40,11 @@ impl<'encoded> AdmittedRetentionRoot<'encoded> { &self.root } + /// Returns the verified canonical anchor-set digest. + pub const fn anchor_set_digest(&self) -> RetentionAnchorSetDigest { + self.anchor_set_digest + } + /// Returns the verified canonical root digest. pub const fn digest(&self) -> RetentionRootDigest { self.digest @@ -47,11 +53,13 @@ impl<'encoded> AdmittedRetentionRoot<'encoded> { pub(super) const fn admitted( encoded: &'encoded [u8], root: RetentionRoot, + anchor_set_digest: RetentionAnchorSetDigest, digest: RetentionRootDigest, ) -> Self { Self { encoded, root, + anchor_set_digest, digest, } } diff --git a/src/adapters/retention/root_decoder.rs b/src/adapters/retention/root_decoder.rs index cf4a7b1..454f51b 100644 --- a/src/adapters/retention/root_decoder.rs +++ b/src/adapters/retention/root_decoder.rs @@ -28,7 +28,11 @@ pub(super) fn decode( observed: encoded.len(), }, )?; - root_integrity::verify_anchor_set(header.anchor_count, anchor_bytes, header.anchor_set_digest)?; + let anchor_set_digest = root_integrity::verify_anchor_set( + header.anchor_count, + anchor_bytes, + header.anchor_set_digest, + )?; let admitted_header = root_semantic_header::admit(&header)?; let namespace = RetentionNamespace::try_from(namespace_bytes) .map_err(|source| RetentionRootDecodeError::Namespace { source })?; @@ -45,6 +49,7 @@ pub(super) fn decode( Ok(AdmittedRetentionRoot::admitted( encoded, root, + anchor_set_digest, RetentionRootDigest::from_hash(digest), )) } diff --git a/src/adapters/retention/root_integrity.rs b/src/adapters/retention/root_integrity.rs index 71f842d..d753751 100644 --- a/src/adapters/retention/root_integrity.rs +++ b/src/adapters/retention/root_integrity.rs @@ -1,6 +1,7 @@ //! This boundary module owns retention root digest and checksum verification. use super::RetentionRootDecodeError; +use crate::RetentionAnchorSetDigest; pub(super) fn verify( encoded: &[u8], @@ -45,14 +46,14 @@ pub(super) fn verify_anchor_set( anchor_count: u32, anchors: &[u8], observed: [u8; 32], -) -> Result<(), RetentionRootDecodeError> { +) -> Result { let mut hasher = blake3::Hasher::new(); hasher.update(b"keep.retention-anchor-set/v2\0"); hasher.update(&anchor_count.to_be_bytes()); hasher.update(anchors); let expected = *hasher.finalize().as_bytes(); if observed == expected { - Ok(()) + Ok(RetentionAnchorSetDigest::from_verified(expected)) } else { Err(RetentionRootDecodeError::AnchorSetDigestMismatch { expected, observed }) } diff --git a/src/lib.rs b/src/lib.rs index adbbffd..13ad241 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -141,10 +141,10 @@ pub use reference::{ }; pub use retention::{ LivenessGeneration, LivenessGenerationError, RegisteredRetentionProfile, RetentionAnchor, - RetentionClosureCounter, RetentionClosureDigest, RetentionClosureLimit, - RetentionClosureLimitError, RetentionClosureLimits, RetentionClosureUsage, - RetentionGenerationExpectation, RetentionHead, RetentionHeadError, RetentionManifest, - RetentionManifestDigest, RetentionManifestEntry, RetentionManifestError, + RetentionAnchorSetDigest, RetentionClosureCounter, RetentionClosureDigest, + RetentionClosureLimit, RetentionClosureLimitError, RetentionClosureLimits, + RetentionClosureUsage, RetentionGenerationExpectation, RetentionHead, RetentionHeadError, + RetentionManifest, RetentionManifestDigest, RetentionManifestEntry, RetentionManifestError, RetentionManifestLength, RetentionManifestLengthError, RetentionNamespace, RetentionNamespaceDigest, RetentionNamespaceError, RetentionPolicy, RetentionProfileAdmissionError, RetentionRoot, RetentionRootDigest, RetentionRootError, diff --git a/src/retention/anchor_set_digest.rs b/src/retention/anchor_set_digest.rs new file mode 100644 index 0000000..4f31742 --- /dev/null +++ b/src/retention/anchor_set_digest.rs @@ -0,0 +1,18 @@ +//! This module owns one verified version-2 retention anchor-set digest. + +/// BLAKE3-256 digest of one canonical retention anchor set. +#[must_use] +#[derive(Clone, Copy, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)] +pub struct RetentionAnchorSetDigest([u8; 32]); + +impl RetentionAnchorSetDigest { + /// Returns the exact digest bytes. + #[must_use] + pub const fn as_bytes(&self) -> &[u8; 32] { + &self.0 + } + + pub(crate) const fn from_verified(bytes: [u8; 32]) -> Self { + Self(bytes) + } +} diff --git a/src/retention/mod.rs b/src/retention/mod.rs index a9da928..481b794 100644 --- a/src/retention/mod.rs +++ b/src/retention/mod.rs @@ -6,6 +6,7 @@ //! collection. mod anchor; +mod anchor_set_digest; mod closure_counter; mod closure_digest; mod closure_limit; @@ -36,6 +37,7 @@ mod root_generation; mod root_generation_error; pub use anchor::RetentionAnchor; +pub use anchor_set_digest::RetentionAnchorSetDigest; pub use closure_counter::RetentionClosureCounter; pub use closure_digest::RetentionClosureDigest; pub use closure_limit::RetentionClosureLimit; diff --git a/tests/retention_root_decoding.rs b/tests/retention_root_decoding.rs index 9405541..be1a4d3 100644 --- a/tests/retention_root_decoding.rs +++ b/tests/retention_root_decoding.rs @@ -8,6 +8,7 @@ use keep::{AdmittedRetentionRoot, RetentionRootDecodeError}; const ONE_ANCHOR_ROOT: &str = include_str!("../conformance/segment-store/v2/one-anchor-root.hex"); const ANCHOR_SET_DIGEST_OFFSET: usize = 148; +const ANCHOR_SET_DIGEST_END: usize = 180; const ANCHOR_BODY_OFFSET: usize = 195; const ROOT_DIGEST_OFFSET: usize = 314; const CHECKSUM_OFFSET: usize = 346; @@ -21,6 +22,12 @@ fn frozen_root_decodes_to_one_complete_semantic_generation() assert_eq!(admitted.root().namespace().as_bytes(), &[0x00, 0x2f, 0xff]); assert_eq!(admitted.root().generation().get(), 1); assert_eq!(admitted.root().anchor_count(), 1); + assert_eq!( + admitted.anchor_set_digest().as_bytes(), + bytes + .get(ANCHOR_SET_DIGEST_OFFSET..ANCHOR_SET_DIGEST_END) + .ok_or_else(|| io::Error::other("frozen root lacks its anchor-set digest"))? + ); assert_eq!( admitted.digest().as_bytes(), bytes.get(314..346).ok_or_else(|| { From 1635bc8eac4c429f1dd450d32c3881a0cc41b6eb Mon Sep 17 00:00:00 2001 From: James Ross Date: Thu, 30 Jul 2026 00:58:35 -0700 Subject: [PATCH 026/111] Add: Execute retention publication --- CHANGELOG.md | 4 +- README.md | 8 +- docs/formats/segment-store-v2/README.md | 11 +- docs/formats/segment-store-v2/requirements.md | 6 +- docs/formats/segment-store-v2/retention.md | 10 +- src/adapters/retention.rs | 8 + .../retention/prepared_publication.rs | 24 ++- src/adapters/retention/publication_error.rs | 61 +++++++ .../retention/publication_execution.rs | 153 ++++++++++++++++++ src/adapters/retention/publication_outcome.rs | 11 ++ .../retention/publication_preparation.rs | 5 +- src/adapters/retention/publication_receipt.rs | 127 +++++++++++++++ src/adapters/retention/publication_storage.rs | 25 ++- src/adapters/retention/successor_manifest.rs | 8 +- src/lib.rs | 14 +- tests/retention_publication_execution.rs | 144 +++++++++++++++++ .../refusal_laws.rs | 94 +++++++++++ .../recording_storage.rs | 146 ++++++++++++----- 18 files changed, 784 insertions(+), 75 deletions(-) create mode 100644 src/adapters/retention/publication_error.rs create mode 100644 src/adapters/retention/publication_execution.rs create mode 100644 src/adapters/retention/publication_outcome.rs create mode 100644 src/adapters/retention/publication_receipt.rs create mode 100644 tests/retention_publication_execution.rs create mode 100644 tests/retention_publication_execution/refusal_laws.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index e45a7c4..7122bc3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,8 +14,8 @@ after its public API and format compatibility policies are established. planning with deterministic closure verification against one pinned catalog before any future publication storage call. A typed 17-phase vocabulary and blocking storage port freeze the durability and crash-boundary contract; - unforgeable proofs preserve disposition, expected and observed generations, - and the verified anchor-set digest through manifest and head preparation. + authority-revalidated orchestration executes every phase and returns an + unforgeable complete-coordinate receipt after durable cleanup. - Repository crash-matrix execution now terminates isolated writer process groups at all 105 canonical before/during/after coordinates, retains open writer and stage authority until termination, executes production diff --git a/README.md b/README.md index 8cf827f..331ab28 100644 --- a/README.md +++ b/README.md @@ -120,10 +120,10 @@ manifest, and retention-head codecs; storage-independent expected-state transition planning; deterministic bounded closure verification against a pinned catalog; a combined transition preflight proof; and the exact 17-phase publication vocabulary with a blocking storage capability port are -implemented. Private-field proofs preserve typed disposition, expected and -observed namespace generations, and the verified anchor-set digest through -canonical manifest and head preparation. Publication orchestration, filesystem -execution, recovery, compaction, and garbage collection remain planned. +implemented. Private-field proofs retain every receipt coordinate. Ordered +storage-port orchestration revalidates current authority, executes all 17 +durability phases, and returns a consequential complete-coordinate receipt. +Filesystem execution, recovery, compaction, and garbage collection remain planned. Presence in the reference CAS does not claim durable retention or crash recovery. diff --git a/docs/formats/segment-store-v2/README.md b/docs/formats/segment-store-v2/README.md index 54c7c1d..150022b 100644 --- a/docs/formats/segment-store-v2/README.md +++ b/docs/formats/segment-store-v2/README.md @@ -79,9 +79,10 @@ planning, deterministic bounded closure verification against one pinned catalog, their combined preflight proof, and the exact 17-phase publication vocabulary with a blocking storage capability port are available. Storage-independent preparation derives exact canonical manifest and head -successors from coherent preflight and current-manifest evidence. Publication -orchestration and production filesystem retention publication, recovery, -migration, and garbage collection do not exist yet. Requirements that remain -planned or in progress in issue #19 or issue #21 are not complete implementation -evidence. A store must refuse unsupported version-2 state until the relevant +successors from coherent preflight and current-manifest evidence. Ordered +storage-port orchestration revalidates authority and returns a complete receipt. +Production filesystem retention publication, recovery, migration, and garbage +collection do not exist yet. Requirements still in progress in issue #19 or +issue #21 are not complete evidence. A store must refuse version-2 state until +the relevant corruption, model-based, crash-injection, recovery, and fuzz evidence exists. diff --git a/docs/formats/segment-store-v2/requirements.md b/docs/formats/segment-store-v2/requirements.md index 41ba7b1..53b12d3 100644 --- a/docs/formats/segment-store-v2/requirements.md +++ b/docs/formats/segment-store-v2/requirements.md @@ -12,12 +12,12 @@ case is not evidence. | `KEEP-RETENTION-001` | `RetentionNamespace`, `RootGeneration`, `LivenessGeneration`, profile coordinates, limits, anchors, and digests are validated typed values | `tests/retention_values.rs`, `tests/retention_root_encoding.rs`, and typed verified anchor-set evidence in `tests/retention_root_decoding.rs` | Implemented | | `KEEP-RETENTION-002` | Root, manifest, and head codecs implement the exact canonical grammars and fixed bounds | independent golden corpus plus `tests/retention_root_encoding.rs`, `tests/retention_root_decoding.rs`, `tests/retention_manifest_codec.rs`, and `tests/retention_head_codec.rs` | Implemented | | `KEEP-RETENTION-003` | Every structural field, truncation boundary, ordering law, duplicate, overflow, flag, reserved byte, digest, checksum, and trailing byte has a precise refusal | corruption matrix and fuzz | Planned in #19 | -| `KEEP-RETENTION-004` | Retain and release compare expected and observed generations and publish exact successors only | unforgeable storage-independent readiness and preflight proofs with preserved disposition and coordinates in `tests/retention_transition.rs` and `tests/retention_preflight.rs`; exact global successor preparation in `tests/retention_publication_preparation.rs`; publication evidence remains | In progress in #19 | +| `KEEP-RETENTION-004` | Retain and release compare expected and observed generations and publish exact successors only | unforgeable readiness and preflight proofs in `tests/retention_transition.rs` and `tests/retention_preflight.rs`; exact successor preparation and complete receipt evidence in `tests/retention_publication_preparation.rs` and `tests/retention_publication_execution.rs`; filesystem evidence remains | In progress in #19 | | `KEEP-RETENTION-005` | Closure derivation is deterministic, bounded, cycle-safe, fail-closed, and verifies complete blob reconstruction | exact accounting, reconstruction, adversarial-catalog, and exhaustive model laws in `tests/retention_closure.rs`; corrupt members refuse through the inherited segment-record admission laws and seeded `segment_format` fuzz target routed by `closure-corruption.md` | Implemented | -| `KEEP-RETENTION-006` | Publication follows the exact ordered durability protocol, including new namespace-directory admission and retention of fixed-stage evidence until head commit, and returns only after cleanup synchronization | typed 17-phase vocabulary in `tests/retention_publication_phase.rs` and blocking capability port in `tests/retention_publication_storage.rs`; ordered execution and `KEEP-CRASH-036..=052` crash-injection tests remain | In progress in #19 | +| `KEEP-RETENTION-006` | Publication follows the exact ordered durability protocol, including new namespace-directory admission and retention of fixed-stage evidence until head commit, and returns only after cleanup synchronization | typed vocabulary and blocking port in `tests/retention_publication_phase.rs` and `tests/retention_publication_storage.rs`; ordered execution, conditional namespace sync, and all 17 exact storage-fault boundaries in `tests/retention_publication_execution.rs`; crash injection remains | In progress in #19 | | `KEEP-RETENTION-007` | Restart resolves every fixed-stage crash prefix to one documented lawful state or typed ambiguity | debug and release crash matrix | Planned in #19 | | `KEEP-RETENTION-008` | Readers double-collect catalog and retention heads and bind one complete catalog, manifest, and root-generation view under a `ReaderFence` | immutable snapshot and concurrency tests | Planned in #19 | -| `KEEP-RETENTION-009` | Exact already-committed retry is idempotent only while its successor remains current | byte-identical readiness and stale-state planning in `tests/retention_transition.rs`; publication retry remains | In progress in #19 | +| `KEEP-RETENTION-009` | Exact already-committed retry is idempotent only while its successor remains current | byte-identical planning in `tests/retention_transition.rs`; authority-revalidated zero-mutation retry receipt in `tests/retention_publication_execution.rs`; filesystem retry remains | In progress in #19 | | `KEEP-RETENTION-010` | Model operation sequences agree with a deterministic namespace-to-anchor-set map and never admit caller identity, paths, clocks, or application policy | model-based and source-architecture tests | Planned in #19 | diff --git a/docs/formats/segment-store-v2/retention.md b/docs/formats/segment-store-v2/retention.md index dddb002..c82960d 100644 --- a/docs/formats/segment-store-v2/retention.md +++ b/docs/formats/segment-store-v2/retention.md @@ -248,11 +248,11 @@ authenticated reconstruction, and canonical digest defined by transition. Keep never omits one failed member and continues with a smaller live set. -`preflight_retention_transition` combines steps 3 and 4 without I/O, returning -an unforgeable typed disposition only after generation and closure verification. -`prepare_retention_publication` binds that proof to the current manifest, -preserves expected and observed generations, refuses incoherent coordinates, -and derives exact canonical successors; exact retry creates no global artifacts. +Preflight verifies steps 3 and 4 without I/O; preparation binds that proof to +the current manifest and derives exact canonical successors. +`execute_retention_publication` revalidates current authority, executes all 17 +ordered durability phases, and returns the complete receipt only after cleanup; +exact already-committed retry revalidates authority and performs no mutation. Version-2 catalog publication holds the same writer authority and proves every current retained closure against its candidate catalog before replacing the diff --git a/src/adapters/retention.rs b/src/adapters/retention.rs index 4978032..10943c0 100644 --- a/src/adapters/retention.rs +++ b/src/adapters/retention.rs @@ -30,9 +30,13 @@ mod manifest_integrity; mod manifest_semantic_header; mod namespace_admission; mod prepared_publication; +mod publication_error; +mod publication_execution; +mod publication_outcome; mod publication_phase; mod publication_preparation; mod publication_preparation_error; +mod publication_receipt; mod publication_storage; mod root_anchor_decoder; mod root_decode_error; @@ -66,9 +70,13 @@ pub use manifest_decode_error::RetentionManifestDecodeError; pub use manifest_encode_error::RetentionManifestEncodeError; pub use namespace_admission::RetentionNamespaceAdmission; pub use prepared_publication::{PreparedRetentionPublication, RetentionPublicationPreparation}; +pub use publication_error::RetentionPublicationError; +pub use publication_execution::execute_retention_publication; +pub use publication_outcome::RetentionPublicationOutcome; pub use publication_phase::RetentionPublicationPhase; pub use publication_preparation::prepare_retention_publication; pub use publication_preparation_error::RetentionPublicationPreparationError; +pub use publication_receipt::RetentionPublicationReceipt; pub use publication_storage::RetentionPublicationStorage; pub use root_decode_error::RetentionRootDecodeError; pub use root_encode_error::RetentionRootEncodeError; diff --git a/src/adapters/retention/prepared_publication.rs b/src/adapters/retention/prepared_publication.rs index d534fa6..fdcf95b 100644 --- a/src/adapters/retention/prepared_publication.rs +++ b/src/adapters/retention/prepared_publication.rs @@ -1,9 +1,10 @@ //! This boundary module owns storage-ready retention publication artifacts. use super::{ - AdmittedRetentionRoot, CanonicalRetentionHead, CanonicalRetentionManifest, - RetentionTransitionDisposition, VerifiedRetentionClosure, + AdmittedRetentionManifest, AdmittedRetentionRoot, CanonicalRetentionHead, + CanonicalRetentionManifest, RetentionTransitionDisposition, VerifiedRetentionClosure, }; +use crate::RetentionManifestDigest; use crate::{LivenessGeneration, RetentionGenerationExpectation, RootGeneration}; /// Canonical global artifacts ready for ordered storage execution. @@ -53,6 +54,8 @@ pub struct RetentionPublicationPreparation<'encoded> { observed: Option, candidate: AdmittedRetentionRoot<'encoded>, closure: VerifiedRetentionClosure, + liveness_generation: LivenessGeneration, + manifest_digest: RetentionManifestDigest, publication: Option, } @@ -82,6 +85,16 @@ impl<'encoded> RetentionPublicationPreparation<'encoded> { self.closure } + /// Returns the exact selected global liveness generation. + pub const fn liveness_generation(&self) -> LivenessGeneration { + self.liveness_generation + } + + /// Returns the exact selected global manifest digest. + pub const fn manifest_digest(&self) -> RetentionManifestDigest { + self.manifest_digest + } + /// Returns new global artifacts, or normal absence for an exact retry. pub const fn publication(&self) -> Option<&PreparedRetentionPublication> { self.publication.as_ref() @@ -94,12 +107,16 @@ impl<'encoded> RetentionPublicationPreparation<'encoded> { closure: VerifiedRetentionClosure, publication: PreparedRetentionPublication, ) -> Self { + let liveness_generation = publication.liveness_generation(); + let manifest_digest = publication.manifest().digest(); Self { disposition: RetentionTransitionDisposition::Publish, expected, observed, candidate, closure, + liveness_generation, + manifest_digest, publication: Some(publication), } } @@ -109,6 +126,7 @@ impl<'encoded> RetentionPublicationPreparation<'encoded> { observed: Option, candidate: AdmittedRetentionRoot<'encoded>, closure: VerifiedRetentionClosure, + current_manifest: &AdmittedRetentionManifest<'_>, ) -> Self { Self { disposition: RetentionTransitionDisposition::AlreadyCommitted, @@ -116,6 +134,8 @@ impl<'encoded> RetentionPublicationPreparation<'encoded> { observed, candidate, closure, + liveness_generation: current_manifest.manifest().generation(), + manifest_digest: current_manifest.digest(), publication: None, } } diff --git a/src/adapters/retention/publication_error.rs b/src/adapters/retention/publication_error.rs new file mode 100644 index 0000000..f002c72 --- /dev/null +++ b/src/adapters/retention/publication_error.rs @@ -0,0 +1,61 @@ +//! This boundary module owns retention publication execution failures. + +use std::error::Error; +use std::fmt; +use std::io; + +use super::{RetentionPublicationPhase, RetentionTransitionDisposition}; + +/// Failure before or during ordered retention publication. +#[derive(Debug)] +pub enum RetentionPublicationError { + /// Current authority could not be revalidated before mutation. + CurrentVerification { + /// Preserved storage refusal. + source: io::Error, + }, + /// Storage requested publication from an already-committed preparation. + DispositionMismatch { + /// Disposition proven during storage-independent preparation. + prepared: RetentionTransitionDisposition, + /// Disposition observed under current writer authority. + observed: RetentionTransitionDisposition, + }, + /// A publish disposition lacked its private canonical artifacts. + MissingPublicationArtifacts, + /// One exact durability phase failed. + Storage { + /// Phase attempted when storage refused. + phase: RetentionPublicationPhase, + /// Preserved storage refusal. + source: io::Error, + }, +} + +impl fmt::Display for RetentionPublicationError { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::CurrentVerification { .. } => { + formatter.write_str("retention publication authority verification failed") + } + Self::DispositionMismatch { .. } => { + formatter.write_str("retention publication disposition changed inconsistently") + } + Self::MissingPublicationArtifacts => { + formatter.write_str("retention publication artifacts are missing") + } + Self::Storage { phase, .. } => { + write!(formatter, "retention publication phase {phase} failed") + } + } + } +} + +impl Error for RetentionPublicationError { + fn source(&self) -> Option<&(dyn Error + 'static)> { + match self { + Self::CurrentVerification { source } | Self::Storage { source, .. } => Some(source), + Self::DispositionMismatch { .. } | Self::MissingPublicationArtifacts => None, + } + } +} diff --git a/src/adapters/retention/publication_execution.rs b/src/adapters/retention/publication_execution.rs new file mode 100644 index 0000000..e7a7d30 --- /dev/null +++ b/src/adapters/retention/publication_execution.rs @@ -0,0 +1,153 @@ +//! This boundary module owns ordered retention publication execution. + +use std::io; + +use super::{ + PreparedRetentionPublication, RetentionNamespaceAdmission, RetentionPublicationError, + RetentionPublicationOutcome, RetentionPublicationPhase, RetentionPublicationPreparation, + RetentionPublicationReceipt, RetentionPublicationStorage, RetentionTransitionDisposition, +}; + +/// Executes one prepared retention transition under revalidated authority. +/// +/// Exact already-committed state performs no publication mutation. A new +/// publication returns only after head visibility and cleanup are synchronized. +/// +/// # Errors +/// +/// Returns [`RetentionPublicationError`] for current-state revalidation, +/// disposition disagreement, missing private artifacts, or the exact failed +/// durability phase. Failure returns no receipt. +pub fn execute_retention_publication( + storage: &mut impl RetentionPublicationStorage, + preparation: &RetentionPublicationPreparation<'_>, +) -> Result { + let observed = storage + .verify_current(preparation) + .map_err(|source| RetentionPublicationError::CurrentVerification { source })?; + if observed == RetentionTransitionDisposition::AlreadyCommitted { + return Ok(RetentionPublicationReceipt::new( + RetentionPublicationOutcome::AlreadyCommitted, + None, + preparation, + )); + } + if preparation.disposition() != RetentionTransitionDisposition::Publish { + return Err(RetentionPublicationError::DispositionMismatch { + prepared: preparation.disposition(), + observed, + }); + } + let publication = preparation + .publication() + .ok_or(RetentionPublicationError::MissingPublicationArtifacts)?; + let namespace_admission = execute_root(storage, preparation)?; + execute_manifest(storage, publication)?; + execute_head(storage, publication)?; + execute_cleanup(storage)?; + Ok(RetentionPublicationReceipt::new( + RetentionPublicationOutcome::Published, + Some(namespace_admission), + preparation, + )) +} + +fn execute_root( + storage: &mut impl RetentionPublicationStorage, + preparation: &RetentionPublicationPreparation<'_>, +) -> Result { + let root = preparation.candidate(); + require( + storage.write_root_stage(root), + RetentionPublicationPhase::WriteRootStage, + )?; + require( + storage.synchronize_root_stage(), + RetentionPublicationPhase::SynchronizeRootStage, + )?; + let admission = require( + storage.admit_root_namespace(root), + RetentionPublicationPhase::AdmitRootNamespace, + )?; + if admission == RetentionNamespaceAdmission::Created { + require( + storage.synchronize_roots_after_namespace(), + RetentionPublicationPhase::SynchronizeRootsAfterNamespace, + )?; + } + require(storage.link_root(root), RetentionPublicationPhase::LinkRoot)?; + require( + storage.synchronize_root_namespace(root), + RetentionPublicationPhase::SynchronizeRootNamespace, + )?; + Ok(admission) +} + +fn execute_manifest( + storage: &mut impl RetentionPublicationStorage, + publication: &PreparedRetentionPublication, +) -> Result<(), RetentionPublicationError> { + let manifest = publication.manifest(); + require( + storage.write_manifest_stage(manifest), + RetentionPublicationPhase::WriteManifestStage, + )?; + require( + storage.synchronize_manifest_stage(), + RetentionPublicationPhase::SynchronizeManifestStage, + )?; + require( + storage.link_manifest(manifest), + RetentionPublicationPhase::LinkManifest, + )?; + require( + storage.synchronize_manifest_pool(), + RetentionPublicationPhase::SynchronizeManifestPool, + ) +} + +fn execute_head( + storage: &mut impl RetentionPublicationStorage, + publication: &PreparedRetentionPublication, +) -> Result<(), RetentionPublicationError> { + require( + storage.write_head_stage(publication.head()), + RetentionPublicationPhase::WriteHeadStage, + )?; + require( + storage.synchronize_head_stage(), + RetentionPublicationPhase::SynchronizeHeadStage, + )?; + require( + storage.replace_head(), + RetentionPublicationPhase::ReplaceHead, + )?; + require( + storage.synchronize_retention_namespace(), + RetentionPublicationPhase::SynchronizeRetentionNamespace, + ) +} + +fn execute_cleanup( + storage: &mut impl RetentionPublicationStorage, +) -> Result<(), RetentionPublicationError> { + require( + storage.remove_root_stage(), + RetentionPublicationPhase::RemoveRootStage, + )?; + require( + storage.remove_manifest_stage(), + RetentionPublicationPhase::RemoveManifestStage, + )?; + require( + storage.synchronize_cleanup(), + RetentionPublicationPhase::SynchronizeCleanup, + ) +} + +fn require( + result: io::Result, + phase: RetentionPublicationPhase, +) -> Result { + result.map_err(|source| RetentionPublicationError::Storage { phase, source }) +} diff --git a/src/adapters/retention/publication_outcome.rs b/src/adapters/retention/publication_outcome.rs new file mode 100644 index 0000000..6f88a3f --- /dev/null +++ b/src/adapters/retention/publication_outcome.rs @@ -0,0 +1,11 @@ +//! This boundary module owns retention publication outcome vocabulary. + +/// Durable outcome of one authority-revalidated retention publication. +#[must_use] +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum RetentionPublicationOutcome { + /// The complete successor became durable and visible. + Published, + /// The exact candidate and global manifest were already current. + AlreadyCommitted, +} diff --git a/src/adapters/retention/publication_preparation.rs b/src/adapters/retention/publication_preparation.rs index 01680f1..600306f 100644 --- a/src/adapters/retention/publication_preparation.rs +++ b/src/adapters/retention/publication_preparation.rs @@ -26,9 +26,10 @@ pub fn prepare_retention_publication<'encoded>( let (disposition, expected, observed, candidate, closure) = preflight.into_parts(); match disposition { RetentionTransitionDisposition::AlreadyCommitted => { - successor_manifest::require_current_selection(&candidate, current_manifest)?; + let current = + successor_manifest::require_current_selection(&candidate, current_manifest)?; Ok(RetentionPublicationPreparation::already_committed( - expected, observed, candidate, closure, + expected, observed, candidate, closure, current, )) } RetentionTransitionDisposition::Publish => { diff --git a/src/adapters/retention/publication_receipt.rs b/src/adapters/retention/publication_receipt.rs new file mode 100644 index 0000000..cd8238e --- /dev/null +++ b/src/adapters/retention/publication_receipt.rs @@ -0,0 +1,127 @@ +//! This boundary module owns consequential retention publication receipts. + +use super::{ + RetentionNamespaceAdmission, RetentionPublicationOutcome, RetentionPublicationPreparation, +}; +use crate::{ + CatalogDigest, CatalogGeneration, LivenessGeneration, RegisteredRetentionProfile, + RetentionAnchorSetDigest, RetentionClosureDigest, RetentionGenerationExpectation, + RetentionManifestDigest, RetentionNamespaceDigest, RetentionRootDigest, RootGeneration, +}; + +/// Complete durable coordinates returned after retention publication. +#[must_use = "retention publication receipts bind the durable outcome"] +#[derive(Debug, Eq, PartialEq)] +pub struct RetentionPublicationReceipt { + outcome: RetentionPublicationOutcome, + namespace_admission: Option, + namespace: RetentionNamespaceDigest, + expected: RetentionGenerationExpectation, + observed: Option, + root_generation: RootGeneration, + root_digest: RetentionRootDigest, + liveness_generation: LivenessGeneration, + manifest_digest: RetentionManifestDigest, + profile: RegisteredRetentionProfile, + anchor_set_digest: RetentionAnchorSetDigest, + closure_digest: RetentionClosureDigest, + catalog_generation: CatalogGeneration, + catalog_digest: CatalogDigest, +} + +impl RetentionPublicationReceipt { + /// Returns the durable publication outcome. + pub const fn outcome(&self) -> RetentionPublicationOutcome { + self.outcome + } + + /// Returns namespace creation or admission for a new publication. + pub const fn namespace_admission(&self) -> Option { + self.namespace_admission + } + + /// Returns the selected retention namespace digest. + pub const fn namespace(&self) -> RetentionNamespaceDigest { + self.namespace + } + + /// Returns the caller-supplied expected namespace generation. + pub const fn expected(&self) -> RetentionGenerationExpectation { + self.expected + } + + /// Returns the namespace generation observed before publication. + pub const fn observed(&self) -> Option { + self.observed + } + + /// Returns the committed namespace root generation. + pub const fn root_generation(&self) -> RootGeneration { + self.root_generation + } + + /// Returns the committed canonical root digest. + pub const fn root_digest(&self) -> RetentionRootDigest { + self.root_digest + } + + /// Returns the selected global liveness generation. + pub const fn liveness_generation(&self) -> LivenessGeneration { + self.liveness_generation + } + + /// Returns the selected global manifest digest. + pub const fn manifest_digest(&self) -> RetentionManifestDigest { + self.manifest_digest + } + + /// Returns the registered realization profile. + pub const fn profile(&self) -> RegisteredRetentionProfile { + self.profile + } + + /// Returns the verified anchor-set digest. + pub const fn anchor_set_digest(&self) -> RetentionAnchorSetDigest { + self.anchor_set_digest + } + + /// Returns the verified closure transcript digest. + pub const fn closure_digest(&self) -> RetentionClosureDigest { + self.closure_digest + } + + /// Returns the pinned catalog generation. + pub const fn catalog_generation(&self) -> CatalogGeneration { + self.catalog_generation + } + + /// Returns the pinned catalog digest. + pub const fn catalog_digest(&self) -> CatalogDigest { + self.catalog_digest + } + + pub(super) fn new( + outcome: RetentionPublicationOutcome, + namespace_admission: Option, + preparation: &RetentionPublicationPreparation<'_>, + ) -> Self { + let candidate = preparation.candidate(); + let closure = preparation.closure(); + Self { + outcome, + namespace_admission, + namespace: candidate.root().namespace().digest(), + expected: preparation.expected(), + observed: preparation.observed(), + root_generation: candidate.root().generation(), + root_digest: candidate.digest(), + liveness_generation: preparation.liveness_generation(), + manifest_digest: preparation.manifest_digest(), + profile: candidate.root().profile(), + anchor_set_digest: candidate.anchor_set_digest(), + closure_digest: closure.digest(), + catalog_generation: closure.catalog_generation(), + catalog_digest: closure.catalog_digest(), + } + } +} diff --git a/src/adapters/retention/publication_storage.rs b/src/adapters/retention/publication_storage.rs index cbef5d8..7a28232 100644 --- a/src/adapters/retention/publication_storage.rs +++ b/src/adapters/retention/publication_storage.rs @@ -4,17 +4,32 @@ use std::io; use super::{ AdmittedRetentionRoot, CanonicalRetentionHead, CanonicalRetentionManifest, - RetentionNamespaceAdmission, + RetentionNamespaceAdmission, RetentionPublicationPreparation, RetentionTransitionDisposition, }; /// Blocking storage capabilities for one writer-locked retention publication. /// /// An implementation must retain exclusive writer authority and one pinned -/// store root for the complete operation. Each method corresponds to one -/// [`RetentionPublicationPhase`](super::RetentionPublicationPhase) and must not -/// report success before the named durability and verification obligations are -/// complete. +/// store root for the complete operation. After `verify_current`, each method +/// corresponds to one [`RetentionPublicationPhase`](super::RetentionPublicationPhase) +/// and must not report success before the named durability and verification +/// obligations are complete. pub trait RetentionPublicationStorage { + /// Reopens and verifies current authority against the complete preparation. + /// + /// `Publish` requires the expected predecessor and global manifest + /// coordinates to remain current. `AlreadyCommitted` requires the exact + /// candidate root and selected global manifest coordinates to be current. + /// Fixed-stage recovery state must refuse before either disposition. + /// + /// # Errors + /// + /// Returns the exact current-state or recovery-required refusal. + fn verify_current( + &mut self, + preparation: &RetentionPublicationPreparation<'_>, + ) -> io::Result; + /// Exclusively creates and completely writes the canonical root stage. /// /// # Errors diff --git a/src/adapters/retention/successor_manifest.rs b/src/adapters/retention/successor_manifest.rs index d87b0d6..3859c16 100644 --- a/src/adapters/retention/successor_manifest.rs +++ b/src/adapters/retention/successor_manifest.rs @@ -46,10 +46,10 @@ pub(super) fn build( .map_err(|source| RetentionPublicationPreparationError::Manifest { source }) } -pub(super) fn require_current_selection( +pub(super) fn require_current_selection<'borrow, 'encoded>( candidate: &AdmittedRetentionRoot<'_>, - current: Option<&AdmittedRetentionManifest<'_>>, -) -> Result<(), RetentionPublicationPreparationError> { + current: Option<&'borrow AdmittedRetentionManifest<'encoded>>, +) -> Result<&'borrow AdmittedRetentionManifest<'encoded>, RetentionPublicationPreparationError> { let namespace = candidate.root().namespace().digest(); let current = current .ok_or(RetentionPublicationPreparationError::CurrentManifestRequired { namespace })?; @@ -70,7 +70,7 @@ pub(super) fn require_current_selection( if entry.root_generation() == candidate.root().generation() && entry.root_digest() == candidate.digest() { - Ok(()) + Ok(current) } else { Err(current_mismatch(entry, candidate)) } diff --git a/src/lib.rs b/src/lib.rs index 13ad241..91cd10a 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -28,8 +28,9 @@ //! combined transition preflight proof and exact publication phase vocabulary //! with a blocking storage capability port are available. Storage-independent //! preparation binds preflight to exact canonical manifest and head successors. -//! Retention publication orchestration, filesystem execution, recovery, and -//! garbage collection remain intentionally absent. +//! Ordered publication revalidates authority, executes all durability phases, +//! and returns a complete receipt. Filesystem execution, recovery, and garbage +//! collection remain intentionally absent. #[cfg(test)] extern crate self as keep; @@ -112,12 +113,13 @@ pub use adapters::{ CanonicalRetentionManifest, CanonicalRetentionRoot, ChecksummedRetentionHead, PreparedRetentionPublication, RetentionClosureVerificationError, RetentionHeadDecodeError, RetentionManifestDecodeError, RetentionManifestEncodeError, RetentionNamespaceAdmission, - RetentionPublicationPhase, RetentionPublicationPreparation, - RetentionPublicationPreparationError, RetentionPublicationStorage, RetentionRootDecodeError, + RetentionPublicationError, RetentionPublicationOutcome, RetentionPublicationPhase, + RetentionPublicationPreparation, RetentionPublicationPreparationError, + RetentionPublicationReceipt, RetentionPublicationStorage, RetentionRootDecodeError, RetentionRootEncodeError, RetentionTransitionDisposition, RetentionTransitionError, RetentionTransitionPreflight, RetentionTransitionPreflightError, RetentionTransitionReadiness, - VerifiedRetentionClosure, plan_retention_transition, preflight_retention_transition, - prepare_retention_publication, verify_retention_closure, + VerifiedRetentionClosure, execute_retention_publication, plan_retention_transition, + preflight_retention_transition, prepare_retention_publication, verify_retention_closure, }; pub use blob::{ BlobHashError, BlobHasher, BlobId, BlobLength, BlobReadError, ByteLength, ByteOffset, diff --git a/tests/retention_publication_execution.rs b/tests/retention_publication_execution.rs new file mode 100644 index 0000000..59813ef --- /dev/null +++ b/tests/retention_publication_execution.rs @@ -0,0 +1,144 @@ +//! Ordered retention publication and consequential receipt laws. + +#[path = "retention_publication_preparation/fixture.rs"] +pub mod fixture; +#[path = "retention_publication_storage/recording_storage.rs"] +pub mod recording_storage; +#[path = "retention_publication_execution/refusal_laws.rs"] +mod refusal_laws; +mod support; + +use std::error::Error; + +use keep::{ + AdmittedRetentionManifest, AdmittedRetentionRoot, RetentionGenerationExpectation, + RetentionNamespaceAdmission, RetentionPublicationOutcome, RetentionPublicationPhase, + RetentionPublicationPreparation, RootGeneration, execute_retention_publication, + preflight_retention_transition, prepare_retention_publication, +}; + +use fixture::{manifest_bytes, root_bytes, with_snapshot}; +use recording_storage::RecordingStorage; + +#[test] +fn publication_executes_every_phase_and_returns_complete_coordinates() -> Result<(), Box> +{ + let root_bytes = root_bytes()?; + let candidate = AdmittedRetentionRoot::decode(&root_bytes)?; + let namespace = candidate.root().namespace().digest(); + let root_generation = candidate.root().generation(); + let root_digest = candidate.digest(); + let profile = candidate.root().profile(); + let anchor_set_digest = candidate.anchor_set_digest(); + let preflight = with_snapshot(|snapshot| { + preflight_retention_transition( + RetentionGenerationExpectation::Absent, + None, + candidate, + snapshot, + ) + })??; + let closure = preflight.closure(); + let preparation = prepare_retention_publication(preflight, None)?; + let publication = preparation + .publication() + .ok_or("initial transition omitted publication artifacts")?; + let liveness_generation = publication.liveness_generation(); + let manifest_digest = publication.manifest().digest(); + let mut storage = RecordingStorage::new(); + + let receipt = execute_retention_publication(&mut storage, &preparation)?; + + assert_eq!(storage.verification_count(), 1); + assert_eq!(storage.observed(), RetentionPublicationPhase::ALL); + assert_eq!(receipt.outcome(), RetentionPublicationOutcome::Published); + assert_eq!( + receipt.namespace_admission(), + Some(RetentionNamespaceAdmission::Created) + ); + assert_eq!(receipt.namespace(), namespace); + assert_eq!(receipt.expected(), RetentionGenerationExpectation::Absent); + assert_eq!(receipt.observed(), None); + assert_eq!(receipt.root_generation(), root_generation); + assert_eq!(receipt.root_digest(), root_digest); + assert_eq!(receipt.liveness_generation(), liveness_generation); + assert_eq!(receipt.manifest_digest(), manifest_digest); + assert_eq!(receipt.profile(), profile); + assert_eq!(receipt.anchor_set_digest(), anchor_set_digest); + assert_eq!(receipt.closure_digest(), closure.digest()); + assert_eq!(receipt.catalog_generation(), closure.catalog_generation()); + assert_eq!(receipt.catalog_digest(), closure.catalog_digest()); + Ok(()) +} + +#[test] +fn exact_retry_revalidates_authority_without_publication_mutation() -> Result<(), Box> { + let root_bytes = root_bytes()?; + let current = AdmittedRetentionRoot::decode(&root_bytes)?; + let candidate = AdmittedRetentionRoot::decode(&root_bytes)?; + let manifest_bytes = manifest_bytes()?; + let current_manifest = AdmittedRetentionManifest::decode(&manifest_bytes)?; + let preflight = with_snapshot(|snapshot| { + preflight_retention_transition( + RetentionGenerationExpectation::Absent, + Some(¤t), + candidate, + snapshot, + ) + })??; + let preparation = prepare_retention_publication(preflight, Some(¤t_manifest))?; + let mut storage = RecordingStorage::already_committed(); + + let receipt = execute_retention_publication(&mut storage, &preparation)?; + + assert_eq!(storage.verification_count(), 1); + assert!(storage.observed().is_empty()); + assert_eq!( + receipt.outcome(), + RetentionPublicationOutcome::AlreadyCommitted + ); + assert_eq!(receipt.namespace_admission(), None); + assert_eq!(receipt.observed(), Some(RootGeneration::INITIAL)); + assert_eq!( + receipt.liveness_generation(), + current_manifest.manifest().generation() + ); + assert_eq!(receipt.manifest_digest(), current_manifest.digest()); + Ok(()) +} + +#[test] +fn existing_namespace_skips_only_the_parent_directory_synchronization() -> Result<(), Box> +{ + let root_bytes = root_bytes()?; + let preparation = initial_preparation(&root_bytes)?; + let mut storage = RecordingStorage::existing_namespace(); + let expected = RetentionPublicationPhase::ALL + .into_iter() + .filter(|phase| *phase != RetentionPublicationPhase::SynchronizeRootsAfterNamespace) + .collect::>(); + + let receipt = execute_retention_publication(&mut storage, &preparation)?; + + assert_eq!(storage.observed(), expected); + assert_eq!( + receipt.namespace_admission(), + Some(RetentionNamespaceAdmission::Existing) + ); + Ok(()) +} + +pub(crate) fn initial_preparation( + root_bytes: &[u8], +) -> Result, Box> { + let candidate = AdmittedRetentionRoot::decode(root_bytes)?; + let preflight = with_snapshot(|snapshot| { + preflight_retention_transition( + RetentionGenerationExpectation::Absent, + None, + candidate, + snapshot, + ) + })??; + prepare_retention_publication(preflight, None).map_err(Into::into) +} diff --git a/tests/retention_publication_execution/refusal_laws.rs b/tests/retention_publication_execution/refusal_laws.rs new file mode 100644 index 0000000..f76a081 --- /dev/null +++ b/tests/retention_publication_execution/refusal_laws.rs @@ -0,0 +1,94 @@ +//! Retention publication execution refusal laws. + +use std::error::Error; +use std::io; + +use keep::{ + AdmittedRetentionManifest, AdmittedRetentionRoot, RetentionGenerationExpectation, + RetentionPublicationError, RetentionPublicationPhase, RetentionTransitionDisposition, + execute_retention_publication, preflight_retention_transition, prepare_retention_publication, +}; + +use crate::fixture::{manifest_bytes, root_bytes, with_snapshot}; +use crate::recording_storage::RecordingStorage; +use crate::support::require_error; + +#[test] +fn current_authority_refusal_precedes_every_publication_phase() -> Result<(), Box> { + let root_bytes = root_bytes()?; + let preparation = crate::initial_preparation(&root_bytes)?; + let mut storage = RecordingStorage::verification_failure(); + + let error = require_error( + execute_retention_publication(&mut storage, &preparation), + "authority refusal returned a receipt", + )?; + + assert!(matches!( + error, + RetentionPublicationError::CurrentVerification { source } + if source.kind() == io::ErrorKind::PermissionDenied + )); + assert_eq!(storage.verification_count(), 1); + assert!(storage.observed().is_empty()); + Ok(()) +} + +#[test] +fn every_phase_failure_stops_before_all_later_mutation() -> Result<(), Box> { + let mut expected = Vec::new(); + for failing_phase in RetentionPublicationPhase::ALL { + expected.push(failing_phase); + let root_bytes = root_bytes()?; + let preparation = crate::initial_preparation(&root_bytes)?; + let mut storage = RecordingStorage::failing_at(failing_phase); + + let error = require_error( + execute_retention_publication(&mut storage, &preparation), + "phase failure returned a receipt", + )?; + + assert!(matches!( + error, + RetentionPublicationError::Storage { phase, source } + if phase == failing_phase && source.kind() == io::ErrorKind::Other + )); + assert_eq!(storage.verification_count(), 1); + assert_eq!(storage.observed(), expected); + } + Ok(()) +} + +#[test] +fn changed_disposition_refuses_before_publication_mutation() -> Result<(), Box> { + let root_bytes = root_bytes()?; + let current = AdmittedRetentionRoot::decode(&root_bytes)?; + let candidate = AdmittedRetentionRoot::decode(&root_bytes)?; + let manifest_bytes = manifest_bytes()?; + let current_manifest = AdmittedRetentionManifest::decode(&manifest_bytes)?; + let preflight = with_snapshot(|snapshot| { + preflight_retention_transition( + RetentionGenerationExpectation::Absent, + Some(¤t), + candidate, + snapshot, + ) + })??; + let preparation = prepare_retention_publication(preflight, Some(¤t_manifest))?; + let mut storage = RecordingStorage::new(); + + let error = require_error( + execute_retention_publication(&mut storage, &preparation), + "changed disposition returned a receipt", + )?; + + assert!(matches!( + error, + RetentionPublicationError::DispositionMismatch { + prepared: RetentionTransitionDisposition::AlreadyCommitted, + observed: RetentionTransitionDisposition::Publish, + } + )); + assert!(storage.observed().is_empty()); + Ok(()) +} diff --git a/tests/retention_publication_storage/recording_storage.rs b/tests/retention_publication_storage/recording_storage.rs index 51047dc..fb7a07d 100644 --- a/tests/retention_publication_storage/recording_storage.rs +++ b/tests/retention_publication_storage/recording_storage.rs @@ -4,13 +4,24 @@ use std::io; use keep::{ AdmittedRetentionRoot, CanonicalRetentionHead, CanonicalRetentionManifest, - RetentionNamespaceAdmission, RetentionPublicationPhase, RetentionPublicationStorage, + RetentionNamespaceAdmission, RetentionPublicationPhase, RetentionPublicationPreparation, + RetentionPublicationStorage, RetentionTransitionDisposition, }; /// Storage port that records every attempted publication phase. -#[derive(Default)] pub struct RecordingStorage { observed: Vec, + verification_count: usize, + disposition: RetentionTransitionDisposition, + namespace_admission: RetentionNamespaceAdmission, + fail_at: Option, + verification_failure: Option, +} + +impl Default for RecordingStorage { + fn default() -> Self { + Self::new() + } } impl RecordingStorage { @@ -18,6 +29,59 @@ impl RecordingStorage { pub const fn new() -> Self { Self { observed: Vec::new(), + verification_count: 0, + disposition: RetentionTransitionDisposition::Publish, + namespace_admission: RetentionNamespaceAdmission::Created, + fail_at: None, + verification_failure: None, + } + } + + /// Creates a recorder that observes the candidate as already committed. + pub const fn already_committed() -> Self { + Self { + observed: Vec::new(), + verification_count: 0, + disposition: RetentionTransitionDisposition::AlreadyCommitted, + namespace_admission: RetentionNamespaceAdmission::Created, + fail_at: None, + verification_failure: None, + } + } + + /// Creates a publisher that admits an existing root namespace. + pub const fn existing_namespace() -> Self { + Self { + observed: Vec::new(), + verification_count: 0, + disposition: RetentionTransitionDisposition::Publish, + namespace_admission: RetentionNamespaceAdmission::Existing, + fail_at: None, + verification_failure: None, + } + } + + /// Creates a publisher that fails at one exact durability phase. + pub const fn failing_at(phase: RetentionPublicationPhase) -> Self { + Self { + observed: Vec::new(), + verification_count: 0, + disposition: RetentionTransitionDisposition::Publish, + namespace_admission: RetentionNamespaceAdmission::Created, + fail_at: Some(phase), + verification_failure: None, + } + } + + /// Creates a publisher that refuses current-authority verification. + pub const fn verification_failure() -> Self { + Self { + observed: Vec::new(), + verification_count: 0, + disposition: RetentionTransitionDisposition::Publish, + namespace_admission: RetentionNamespaceAdmission::Created, + fail_at: None, + verification_failure: Some(io::ErrorKind::PermissionDenied), } } @@ -26,97 +90,105 @@ impl RecordingStorage { &self.observed } - fn record(&mut self, phase: RetentionPublicationPhase) { + /// Returns the number of authority-verification calls. + pub const fn verification_count(&self) -> usize { + self.verification_count + } + + fn record(&mut self, phase: RetentionPublicationPhase) -> io::Result<()> { self.observed.push(phase); + if self.fail_at == Some(phase) { + Err(io::Error::other("injected retention publication failure")) + } else { + Ok(()) + } } } impl RetentionPublicationStorage for RecordingStorage { + fn verify_current( + &mut self, + _preparation: &RetentionPublicationPreparation<'_>, + ) -> io::Result { + self.verification_count = self + .verification_count + .checked_add(1) + .ok_or_else(|| io::Error::other("verification count overflow"))?; + match self.verification_failure { + Some(kind) => Err(io::Error::new(kind, "injected authority failure")), + None => Ok(self.disposition), + } + } + fn write_root_stage(&mut self, _root: &AdmittedRetentionRoot<'_>) -> io::Result<()> { - self.record(RetentionPublicationPhase::WriteRootStage); - Ok(()) + self.record(RetentionPublicationPhase::WriteRootStage) } fn synchronize_root_stage(&mut self) -> io::Result<()> { - self.record(RetentionPublicationPhase::SynchronizeRootStage); - Ok(()) + self.record(RetentionPublicationPhase::SynchronizeRootStage) } fn admit_root_namespace( &mut self, _root: &AdmittedRetentionRoot<'_>, ) -> io::Result { - self.record(RetentionPublicationPhase::AdmitRootNamespace); - Ok(RetentionNamespaceAdmission::Created) + self.record(RetentionPublicationPhase::AdmitRootNamespace)?; + Ok(self.namespace_admission) } fn synchronize_roots_after_namespace(&mut self) -> io::Result<()> { - self.record(RetentionPublicationPhase::SynchronizeRootsAfterNamespace); - Ok(()) + self.record(RetentionPublicationPhase::SynchronizeRootsAfterNamespace) } fn link_root(&mut self, _root: &AdmittedRetentionRoot<'_>) -> io::Result<()> { - self.record(RetentionPublicationPhase::LinkRoot); - Ok(()) + self.record(RetentionPublicationPhase::LinkRoot) } fn synchronize_root_namespace(&mut self, _root: &AdmittedRetentionRoot<'_>) -> io::Result<()> { - self.record(RetentionPublicationPhase::SynchronizeRootNamespace); - Ok(()) + self.record(RetentionPublicationPhase::SynchronizeRootNamespace) } fn write_manifest_stage(&mut self, _manifest: &CanonicalRetentionManifest) -> io::Result<()> { - self.record(RetentionPublicationPhase::WriteManifestStage); - Ok(()) + self.record(RetentionPublicationPhase::WriteManifestStage) } fn synchronize_manifest_stage(&mut self) -> io::Result<()> { - self.record(RetentionPublicationPhase::SynchronizeManifestStage); - Ok(()) + self.record(RetentionPublicationPhase::SynchronizeManifestStage) } fn link_manifest(&mut self, _manifest: &CanonicalRetentionManifest) -> io::Result<()> { - self.record(RetentionPublicationPhase::LinkManifest); - Ok(()) + self.record(RetentionPublicationPhase::LinkManifest) } fn synchronize_manifest_pool(&mut self) -> io::Result<()> { - self.record(RetentionPublicationPhase::SynchronizeManifestPool); - Ok(()) + self.record(RetentionPublicationPhase::SynchronizeManifestPool) } fn write_head_stage(&mut self, _head: &CanonicalRetentionHead) -> io::Result<()> { - self.record(RetentionPublicationPhase::WriteHeadStage); - Ok(()) + self.record(RetentionPublicationPhase::WriteHeadStage) } fn synchronize_head_stage(&mut self) -> io::Result<()> { - self.record(RetentionPublicationPhase::SynchronizeHeadStage); - Ok(()) + self.record(RetentionPublicationPhase::SynchronizeHeadStage) } fn replace_head(&mut self) -> io::Result<()> { - self.record(RetentionPublicationPhase::ReplaceHead); - Ok(()) + self.record(RetentionPublicationPhase::ReplaceHead) } fn synchronize_retention_namespace(&mut self) -> io::Result<()> { - self.record(RetentionPublicationPhase::SynchronizeRetentionNamespace); - Ok(()) + self.record(RetentionPublicationPhase::SynchronizeRetentionNamespace) } fn remove_root_stage(&mut self) -> io::Result<()> { - self.record(RetentionPublicationPhase::RemoveRootStage); - Ok(()) + self.record(RetentionPublicationPhase::RemoveRootStage) } fn remove_manifest_stage(&mut self) -> io::Result<()> { - self.record(RetentionPublicationPhase::RemoveManifestStage); - Ok(()) + self.record(RetentionPublicationPhase::RemoveManifestStage) } fn synchronize_cleanup(&mut self) -> io::Result<()> { - self.record(RetentionPublicationPhase::SynchronizeCleanup); - Ok(()) + self.record(RetentionPublicationPhase::SynchronizeCleanup) } } From c0fb6496698fe83dc2f29649bf5c221e6c65ea13 Mon Sep 17 00:00:00 2001 From: James Ross Date: Thu, 30 Jul 2026 01:03:54 -0700 Subject: [PATCH 027/111] Fix: Seal retention receipt outcomes --- .../retention/publication_execution.rs | 15 +++---- src/adapters/retention/publication_receipt.rs | 43 +++++++++++++++---- 2 files changed, 39 insertions(+), 19 deletions(-) diff --git a/src/adapters/retention/publication_execution.rs b/src/adapters/retention/publication_execution.rs index e7a7d30..cb4aa09 100644 --- a/src/adapters/retention/publication_execution.rs +++ b/src/adapters/retention/publication_execution.rs @@ -4,8 +4,8 @@ use std::io; use super::{ PreparedRetentionPublication, RetentionNamespaceAdmission, RetentionPublicationError, - RetentionPublicationOutcome, RetentionPublicationPhase, RetentionPublicationPreparation, - RetentionPublicationReceipt, RetentionPublicationStorage, RetentionTransitionDisposition, + RetentionPublicationPhase, RetentionPublicationPreparation, RetentionPublicationReceipt, + RetentionPublicationStorage, RetentionTransitionDisposition, }; /// Executes one prepared retention transition under revalidated authority. @@ -26,11 +26,7 @@ pub fn execute_retention_publication( .verify_current(preparation) .map_err(|source| RetentionPublicationError::CurrentVerification { source })?; if observed == RetentionTransitionDisposition::AlreadyCommitted { - return Ok(RetentionPublicationReceipt::new( - RetentionPublicationOutcome::AlreadyCommitted, - None, - preparation, - )); + return Ok(RetentionPublicationReceipt::already_committed(preparation)); } if preparation.disposition() != RetentionTransitionDisposition::Publish { return Err(RetentionPublicationError::DispositionMismatch { @@ -45,9 +41,8 @@ pub fn execute_retention_publication( execute_manifest(storage, publication)?; execute_head(storage, publication)?; execute_cleanup(storage)?; - Ok(RetentionPublicationReceipt::new( - RetentionPublicationOutcome::Published, - Some(namespace_admission), + Ok(RetentionPublicationReceipt::published( + namespace_admission, preparation, )) } diff --git a/src/adapters/retention/publication_receipt.rs b/src/adapters/retention/publication_receipt.rs index cd8238e..f3405af 100644 --- a/src/adapters/retention/publication_receipt.rs +++ b/src/adapters/retention/publication_receipt.rs @@ -13,8 +13,7 @@ use crate::{ #[must_use = "retention publication receipts bind the durable outcome"] #[derive(Debug, Eq, PartialEq)] pub struct RetentionPublicationReceipt { - outcome: RetentionPublicationOutcome, - namespace_admission: Option, + effect: RetentionPublicationEffect, namespace: RetentionNamespaceDigest, expected: RetentionGenerationExpectation, observed: Option, @@ -32,12 +31,20 @@ pub struct RetentionPublicationReceipt { impl RetentionPublicationReceipt { /// Returns the durable publication outcome. pub const fn outcome(&self) -> RetentionPublicationOutcome { - self.outcome + match self.effect { + RetentionPublicationEffect::Published(_) => RetentionPublicationOutcome::Published, + RetentionPublicationEffect::AlreadyCommitted => { + RetentionPublicationOutcome::AlreadyCommitted + } + } } /// Returns namespace creation or admission for a new publication. pub const fn namespace_admission(&self) -> Option { - self.namespace_admission + match self.effect { + RetentionPublicationEffect::Published(admission) => Some(admission), + RetentionPublicationEffect::AlreadyCommitted => None, + } } /// Returns the selected retention namespace digest. @@ -100,16 +107,28 @@ impl RetentionPublicationReceipt { self.catalog_digest } - pub(super) fn new( - outcome: RetentionPublicationOutcome, - namespace_admission: Option, + pub(super) fn published( + namespace_admission: RetentionNamespaceAdmission, + preparation: &RetentionPublicationPreparation<'_>, + ) -> Self { + Self::new( + RetentionPublicationEffect::Published(namespace_admission), + preparation, + ) + } + + pub(super) fn already_committed(preparation: &RetentionPublicationPreparation<'_>) -> Self { + Self::new(RetentionPublicationEffect::AlreadyCommitted, preparation) + } + + fn new( + effect: RetentionPublicationEffect, preparation: &RetentionPublicationPreparation<'_>, ) -> Self { let candidate = preparation.candidate(); let closure = preparation.closure(); Self { - outcome, - namespace_admission, + effect, namespace: candidate.root().namespace().digest(), expected: preparation.expected(), observed: preparation.observed(), @@ -125,3 +144,9 @@ impl RetentionPublicationReceipt { } } } + +#[derive(Debug, Eq, PartialEq)] +enum RetentionPublicationEffect { + Published(RetentionNamespaceAdmission), + AlreadyCommitted, +} From 13988860f50f6de70e3795fb66f71ef9bf7a0728 Mon Sep 17 00:00:00 2001 From: James Ross Date: Thu, 30 Jul 2026 01:05:58 -0700 Subject: [PATCH 028/111] Docs: Correct retention execution boundary --- docs/formats/segment-store-v2/retention.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/formats/segment-store-v2/retention.md b/docs/formats/segment-store-v2/retention.md index c82960d..e1d192c 100644 --- a/docs/formats/segment-store-v2/retention.md +++ b/docs/formats/segment-store-v2/retention.md @@ -162,9 +162,9 @@ retention/roots// Names with alternate width, case, suffix, generation, or digest refuse. Keep implements root, manifest, and head codecs with a typed verified anchor-set digest, expected-state transition planning, deterministic closure verification, -and a blocking publication storage capability port. Publication -orchestration, filesystem execution, recovery, and garbage collection remain -absent. +a blocking publication storage capability port, and ordered storage-port +orchestration. Production filesystem execution, recovery, and garbage +collection remain absent. ## Global retention manifest From eee636b7b90a127fba8364e5a283251b696f221a Mon Sep 17 00:00:00 2001 From: James Ross Date: Thu, 30 Jul 2026 01:18:48 -0700 Subject: [PATCH 029/111] Test: Fuzz retention record parsers --- docs/formats/segment-store-v2/requirements.md | 2 +- fuzz/Cargo.toml | 7 +++ fuzz/README.md | 5 ++ fuzz/fuzz_targets/retention_format.rs | 35 ++++++++++++++ xtask/src/fuzz_campaign/target/tests.rs | 1 + xtask/src/fuzz_seed_corpus.rs | 2 + xtask/src/fuzz_seed_corpus/retention_seeds.rs | 46 +++++++++++++++++++ .../fuzz_seed_corpus/tests/materialization.rs | 30 +++++++++++- .../retention_store_v2_protocol_contract.rs | 2 + .../parser_fuzz_laws.rs | 31 +++++++++++++ 10 files changed, 158 insertions(+), 3 deletions(-) create mode 100644 fuzz/fuzz_targets/retention_format.rs create mode 100644 xtask/src/fuzz_seed_corpus/retention_seeds.rs create mode 100644 xtask/tests/retention_store_v2_protocol_contract/parser_fuzz_laws.rs diff --git a/docs/formats/segment-store-v2/requirements.md b/docs/formats/segment-store-v2/requirements.md index 53b12d3..456f139 100644 --- a/docs/formats/segment-store-v2/requirements.md +++ b/docs/formats/segment-store-v2/requirements.md @@ -11,7 +11,7 @@ case is not evidence. | --- | --- | --- | --- | | `KEEP-RETENTION-001` | `RetentionNamespace`, `RootGeneration`, `LivenessGeneration`, profile coordinates, limits, anchors, and digests are validated typed values | `tests/retention_values.rs`, `tests/retention_root_encoding.rs`, and typed verified anchor-set evidence in `tests/retention_root_decoding.rs` | Implemented | | `KEEP-RETENTION-002` | Root, manifest, and head codecs implement the exact canonical grammars and fixed bounds | independent golden corpus plus `tests/retention_root_encoding.rs`, `tests/retention_root_decoding.rs`, `tests/retention_manifest_codec.rs`, and `tests/retention_head_codec.rs` | Implemented | -| `KEEP-RETENTION-003` | Every structural field, truncation boundary, ordering law, duplicate, overflow, flag, reserved byte, digest, checksum, and trailing byte has a precise refusal | corruption matrix and fuzz | Planned in #19 | +| `KEEP-RETENTION-003` | Every structural field, truncation boundary, ordering law, duplicate, overflow, flag, reserved byte, digest, checksum, and trailing byte has a precise refusal | seeded `retention_format` fuzz target plus the root, manifest, and head corruption matrix; mutation coverage remains | In progress in #19 | | `KEEP-RETENTION-004` | Retain and release compare expected and observed generations and publish exact successors only | unforgeable readiness and preflight proofs in `tests/retention_transition.rs` and `tests/retention_preflight.rs`; exact successor preparation and complete receipt evidence in `tests/retention_publication_preparation.rs` and `tests/retention_publication_execution.rs`; filesystem evidence remains | In progress in #19 | | `KEEP-RETENTION-005` | Closure derivation is deterministic, bounded, cycle-safe, fail-closed, and verifies complete blob reconstruction | exact accounting, reconstruction, adversarial-catalog, and exhaustive model laws in `tests/retention_closure.rs`; corrupt members refuse through the inherited segment-record admission laws and seeded `segment_format` fuzz target routed by `closure-corruption.md` | Implemented | | `KEEP-RETENTION-006` | Publication follows the exact ordered durability protocol, including new namespace-directory admission and retention of fixed-stage evidence until head commit, and returns only after cleanup synchronization | typed vocabulary and blocking port in `tests/retention_publication_phase.rs` and `tests/retention_publication_storage.rs`; ordered execution, conditional namespace sync, and all 17 exact storage-fault boundaries in `tests/retention_publication_execution.rs`; crash injection remains | In progress in #19 | diff --git a/fuzz/Cargo.toml b/fuzz/Cargo.toml index 5ab41b7..b041049 100644 --- a/fuzz/Cargo.toml +++ b/fuzz/Cargo.toml @@ -124,6 +124,13 @@ test = false doc = false bench = false +[[bin]] +name = "retention_format" +path = "fuzz_targets/retention_format.rs" +test = false +doc = false +bench = false + [[bin]] name = "segment_format" path = "fuzz_targets/segment_format.rs" diff --git a/fuzz/README.md b/fuzz/README.md index 0bcff69..8b6af19 100644 --- a/fuzz/README.md +++ b/fuzz/README.md @@ -65,6 +65,11 @@ decoders. Canonical generation-1, generation-2, and two-record bundle artifacts keep mutations inside framing, ordering, coordinate, checksum, and digest validation; every admitted value must retain its exact input bytes. +The `retention_format` seeds select the public retention-root, +retention-manifest, and retention-head decoders. The canonical one-root +generation keeps mutations inside framing, semantic, ordering, checksum, and +digest validation; every admitted value must retain its exact input bytes. + The `segment_format` seeds select the public segment-header, record-header, complete-record, seal, and complete-segment boundaries. Canonical empty, one-record, and bundled segments keep mutations inside the nested parsers; diff --git a/fuzz/fuzz_targets/retention_format.rs b/fuzz/fuzz_targets/retention_format.rs new file mode 100644 index 0000000..b35808c --- /dev/null +++ b/fuzz/fuzz_targets/retention_format.rs @@ -0,0 +1,35 @@ +#![no_main] + +//! This target owns canonical retention-record parser fuzzing. + +use keep::{AdmittedRetentionManifest, AdmittedRetentionRoot, ChecksummedRetentionHead}; +use libfuzzer_sys::fuzz_target; + +fuzz_target!(|bytes: &[u8]| { + let Some((&selector, input)) = bytes.split_first() else { + return; + }; + match selector { + 0 => root(input), + 1 => manifest(input), + _ => head(input), + } +}); + +fn root(input: &[u8]) { + if let Ok(root) = AdmittedRetentionRoot::decode(input) { + assert_eq!(root.encoded(), input); + } +} + +fn manifest(input: &[u8]) { + if let Ok(manifest) = AdmittedRetentionManifest::decode(input) { + assert_eq!(manifest.encoded(), input); + } +} + +fn head(input: &[u8]) { + if let Ok(head) = ChecksummedRetentionHead::decode(input) { + assert_eq!(head.encoded(), input); + } +} diff --git a/xtask/src/fuzz_campaign/target/tests.rs b/xtask/src/fuzz_campaign/target/tests.rs index 82e88b3..7b72f99 100644 --- a/xtask/src/fuzz_campaign/target/tests.rs +++ b/xtask/src/fuzz_campaign/target/tests.rs @@ -31,6 +31,7 @@ fn checked_in_harness_set_is_exact_and_sorted() -> Result<(), Box> { "golden_protocol", "layout_record", "repository_json", + "retention_format", "segment_format", ] ); diff --git a/xtask/src/fuzz_seed_corpus.rs b/xtask/src/fuzz_seed_corpus.rs index 61c76c0..1455e04 100644 --- a/xtask/src/fuzz_seed_corpus.rs +++ b/xtask/src/fuzz_seed_corpus.rs @@ -5,6 +5,7 @@ mod cdc_seeds; mod filesystem; mod identity_seeds; mod layout_seeds; +mod retention_seeds; mod segment_seeds; use std::error::Error; @@ -69,6 +70,7 @@ pub(super) fn prepare(repository_root: &Path) -> Result<(), FuzzSeedError> { seeds.extend(cdc_seeds::seeds()?); seeds.extend(golden_protocol_seeds_from(&files)?); seeds.extend(layout_seeds::seeds(&files)?); + seeds.extend(retention_seeds::seeds(&files)?); seeds.extend(segment_seeds::seeds(&files)?); files.write_seeds(&seeds) } diff --git a/xtask/src/fuzz_seed_corpus/retention_seeds.rs b/xtask/src/fuzz_seed_corpus/retention_seeds.rs new file mode 100644 index 0000000..152dc16 --- /dev/null +++ b/xtask/src/fuzz_seed_corpus/retention_seeds.rs @@ -0,0 +1,46 @@ +//! This module owns canonical retention-record fuzz seeds. + +use std::path::Path; + +use super::filesystem::RepositoryFiles; +use super::{FuzzSeedError, MAX_SEED_BYTES, Seed, prefixed}; +use xtask::protocol_admission::{EmptyHex, decode_lower_hex, framed_lines}; + +const SEGMENT_STORE_ROOT: &str = "conformance/segment-store/v2"; + +pub(super) const FIXTURES: [(u8, &str); 3] = [ + (0, "one-anchor-root.hex"), + (1, "one-root-manifest.hex"), + (2, "one-root-head.hex"), +]; + +pub(super) fn seeds(files: &RepositoryFiles) -> Result, FuzzSeedError> { + let mut seeds = Vec::new(); + for (selector, fixture) in FIXTURES { + let name = fixture + .strip_suffix(".hex") + .ok_or_else(|| FuzzSeedError::violation("retention fixture lacks .hex suffix"))?; + let encoded = fixture_bytes(files, fixture)?; + seeds.push(Seed::new( + "retention_format", + name, + prefixed(selector, &encoded)?, + )?); + } + Ok(seeds) +} + +fn fixture_bytes(files: &RepositoryFiles, fixture: &'static str) -> Result, FuzzSeedError> { + let relative = Path::new(SEGMENT_STORE_ROOT).join(fixture); + let transport = files.read_bounded(&relative, MAX_SEED_BYTES)?; + let lines = framed_lines(&transport, MAX_SEED_BYTES) + .map_err(|source| FuzzSeedError::violation(format!("{fixture} framing moved: {source}")))?; + let [encoded] = lines.as_slice() else { + return Err(FuzzSeedError::violation(format!( + "{fixture} must contain exactly one hexadecimal line" + ))); + }; + decode_lower_hex(encoded, MAX_SEED_BYTES, EmptyHex::Refuse).map_err(|source| { + FuzzSeedError::violation(format!("{fixture} is not canonical hexadecimal: {source}")) + }) +} diff --git a/xtask/src/fuzz_seed_corpus/tests/materialization.rs b/xtask/src/fuzz_seed_corpus/tests/materialization.rs index 856ccf5..5ed813f 100644 --- a/xtask/src/fuzz_seed_corpus/tests/materialization.rs +++ b/xtask/src/fuzz_seed_corpus/tests/materialization.rs @@ -3,7 +3,9 @@ use std::collections::BTreeMap; use std::path::Path; -use super::super::{FuzzSeedError, catalog_seeds, layout_seeds, prepare, segment_seeds}; +use super::super::{ + FuzzSeedError, catalog_seeds, layout_seeds, prepare, retention_seeds, segment_seeds, +}; use crate::test_directory::TestDirectory; const TABLES: [&str; 5] = [ @@ -39,14 +41,16 @@ fn seed_preparation_materializes_the_complete_deterministic_set() copy_layout_fixtures(source_root, root)?; copy_segment_fixtures(source_root, root)?; copy_catalog_fixtures(source_root, root)?; + copy_retention_fixtures(source_root, root)?; prepare(root)?; let corpus = root.join("fuzz/corpus"); let first = seed_contents(&corpus)?; - assert_eq!(first.len(), 40); + assert_eq!(first.len(), 43); assert_eq!(target_seed_count(&first, "catalog_format/"), 6); assert_eq!(target_seed_count(&first, "golden_protocol/"), 9); assert_eq!(target_seed_count(&first, "layout_record/"), 4); + assert_eq!(target_seed_count(&first, "retention_format/"), 3); assert_eq!(target_seed_count(&first, "segment_format/"), 8); prepare(root)?; assert_eq!(seed_contents(&corpus)?, first); @@ -112,6 +116,28 @@ fn copy_catalog_fixtures(source_root: &Path, root: &Path) -> Result<(), FuzzSeed Ok(()) } +fn copy_retention_fixtures(source_root: &Path, root: &Path) -> Result<(), FuzzSeedError> { + use std::fs; + + let retention_directory = root.join("conformance/segment-store/v2"); + fs::create_dir_all(&retention_directory).map_err(|source| { + FuzzSeedError::io( + "create test retention conformance root", + &retention_directory, + source, + ) + })?; + for (_selector, fixture) in retention_seeds::FIXTURES { + let source_path = source_root + .join("conformance/segment-store/v2") + .join(fixture); + let destination = retention_directory.join(fixture); + fs::copy(&source_path, &destination) + .map_err(|source| FuzzSeedError::io("copy test retention", &destination, source))?; + } + Ok(()) +} + fn target_seed_count(contents: &BTreeMap>, prefix: &str) -> usize { contents .keys() diff --git a/xtask/tests/retention_store_v2_protocol_contract.rs b/xtask/tests/retention_store_v2_protocol_contract.rs index d2dc2ae..09f2316 100644 --- a/xtask/tests/retention_store_v2_protocol_contract.rs +++ b/xtask/tests/retention_store_v2_protocol_contract.rs @@ -6,6 +6,8 @@ mod closure_contract_laws; #[path = "retention_store_v2_protocol_contract/migration_contract_laws.rs"] mod migration_contract_laws; +#[path = "retention_store_v2_protocol_contract/parser_fuzz_laws.rs"] +mod parser_fuzz_laws; use std::fs; use std::io; diff --git a/xtask/tests/retention_store_v2_protocol_contract/parser_fuzz_laws.rs b/xtask/tests/retention_store_v2_protocol_contract/parser_fuzz_laws.rs new file mode 100644 index 0000000..bc2919f --- /dev/null +++ b/xtask/tests/retention_store_v2_protocol_contract/parser_fuzz_laws.rs @@ -0,0 +1,31 @@ +//! Fuzz-evidence laws for durable retention parser boundaries. + +use std::error::Error; +use std::path::Path; + +const FUZZ_MANIFEST: &str = include_str!("../../../fuzz/Cargo.toml"); +const FUZZ_GUIDE: &str = include_str!("../../../fuzz/README.md"); +const REQUIREMENTS: &str = include_str!("../../../docs/formats/segment-store-v2/requirements.md"); + +#[test] +fn retention_decoders_have_registered_seeded_fuzz_evidence() -> Result<(), Box> { + let repository_root = Path::new(env!("CARGO_MANIFEST_DIR")) + .parent() + .ok_or("xtask manifest must have a repository parent")?; + + assert!( + repository_root + .join("fuzz/fuzz_targets/retention_format.rs") + .is_file() + ); + assert!( + repository_root + .join("xtask/src/fuzz_seed_corpus/retention_seeds.rs") + .is_file() + ); + assert!(FUZZ_MANIFEST.contains("name = \"retention_format\"")); + assert!(FUZZ_MANIFEST.contains("path = \"fuzz_targets/retention_format.rs\"")); + assert!(FUZZ_GUIDE.contains("The `retention_format` seeds")); + assert!(REQUIREMENTS.contains("`retention_format`")); + Ok(()) +} From aabfb895a24569fc30f236e1f24aa09c99eeb304 Mon Sep 17 00:00:00 2001 From: James Ross Date: Thu, 30 Jul 2026 01:40:17 -0700 Subject: [PATCH 030/111] Add: Admit version two format markers --- CHANGELOG.md | 12 +- README.md | 6 +- docs/formats/segment-store-v2/recovery.md | 5 + docs/formats/segment-store-v2/requirements.md | 2 +- src/adapters/mod.rs | 2 + src/adapters/store_migration.rs | 16 ++ .../store_migration/admitted_format_marker.rs | 58 ++++++ .../canonical_format_marker.rs | 36 ++++ .../format_definition_digest.rs | 25 +++ .../format_marker_decode_error.rs | 63 +++++++ .../format_marker_decode_error_display.rs | 50 ++++++ .../store_migration/format_marker_decoder.rs | 137 ++++++++++++++ .../store_migration/format_marker_digest.rs | 18 ++ .../store_migration/format_marker_encoder.rs | 27 +++ src/lib.rs | 77 ++++---- tests/store_format_marker.rs | 168 ++++++++++++++++++ 16 files changed, 656 insertions(+), 46 deletions(-) create mode 100644 src/adapters/store_migration.rs create mode 100644 src/adapters/store_migration/admitted_format_marker.rs create mode 100644 src/adapters/store_migration/canonical_format_marker.rs create mode 100644 src/adapters/store_migration/format_definition_digest.rs create mode 100644 src/adapters/store_migration/format_marker_decode_error.rs create mode 100644 src/adapters/store_migration/format_marker_decode_error_display.rs create mode 100644 src/adapters/store_migration/format_marker_decoder.rs create mode 100644 src/adapters/store_migration/format_marker_digest.rs create mode 100644 src/adapters/store_migration/format_marker_encoder.rs create mode 100644 tests/store_format_marker.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index 7122bc3..2afa12d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,12 +10,12 @@ after its public API and format compatibility policies are established. ### Changed -- Retention transition preflight now combines exact expected-generation - planning with deterministic closure verification against one pinned catalog - before any future publication storage call. A typed 17-phase vocabulary - and blocking storage port freeze the durability and crash-boundary contract; - authority-revalidated orchestration executes every phase and returns an - unforgeable complete-coordinate receipt after durable cleanup. +- The version-2 store-format marker now has exact canonical encoding, + registered-definition admission, checksum verification, and domain-separated + identity. Retention transition preflight combines expected-generation + planning with deterministic closure verification against one pinned catalog; + authority-revalidated 17-phase orchestration returns an unforgeable + complete-coordinate receipt after durable cleanup. - Repository crash-matrix execution now terminates isolated writer process groups at all 105 canonical before/during/after coordinates, retains open writer and stage authority until termination, executes production diff --git a/README.md b/README.md index 331ab28..00f27e8 100644 --- a/README.md +++ b/README.md @@ -115,7 +115,8 @@ bytes, checks hard-link identity and writer-lock release, runs the production recovery classifiers and immutable-artifact admission, and reconstructs the exact published generation and visible one-zero chunk when `HEAD` exists. This matrix proves application process-death behavior; it does not simulate host -power loss. Version-2 retention values; canonical in-memory root, global +power loss. Canonical version-2 store-format marker encoding and admission are +implemented. Version-2 retention values; canonical in-memory root, global manifest, and retention-head codecs; storage-independent expected-state transition planning; deterministic bounded closure verification against a pinned catalog; a combined transition preflight proof; and the exact 17-phase @@ -123,7 +124,8 @@ publication vocabulary with a blocking storage capability port are implemented. Private-field proofs retain every receipt coordinate. Ordered storage-port orchestration revalidates current authority, executes all 17 durability phases, and returns a consequential complete-coordinate receipt. -Filesystem execution, recovery, compaction, and garbage collection remain planned. +Filesystem migration, retention execution, recovery, compaction, and garbage +collection remain planned. Presence in the reference CAS does not claim durable retention or crash recovery. diff --git a/docs/formats/segment-store-v2/recovery.md b/docs/formats/segment-store-v2/recovery.md index 604d0e3..ee9bcb3 100644 --- a/docs/formats/segment-store-v2/recovery.md +++ b/docs/formats/segment-store-v2/recovery.md @@ -60,6 +60,11 @@ The format-definition digest is BLAKE3-256 of its domain followed by the exact corpus `definition.tsv` bytes. The format-marker digest is BLAKE3-256 of `keep.store-format-marker/v2\0` followed by all 96 marker bytes. +`CanonicalStoreFormatMarker` produces the one registered marker, and +`AdmittedStoreFormatMarker` admits exact canonical bytes only after framing, +checksum, definition, and namespace-bound validation. Store detection and +filesystem migration remain absent. + ## Reader fence `reader.lock` is a persistent regular zero-length file. Its contents and diff --git a/docs/formats/segment-store-v2/requirements.md b/docs/formats/segment-store-v2/requirements.md index 456f139..c84c9cf 100644 --- a/docs/formats/segment-store-v2/requirements.md +++ b/docs/formats/segment-store-v2/requirements.md @@ -29,7 +29,7 @@ case is not evidence. | ID | Requirement | Evidence | Status | | --- | --- | --- | --- | | `KEEP-MIGRATION-001` | Exact version-1 stores remain admitted until a durable migration artifact exists | compatibility fixtures | Planned in #19 | -| `KEEP-MIGRATION-002` | Intent and receipt have complete fixed byte tables, named domains, bounds, checksums, deterministic store identity, and exact initial-state digests | golden-format fixtures | Planned in #19 | +| `KEEP-MIGRATION-002` | Format marker, intent, and receipt have complete fixed byte tables, named domains, bounds, checksums, deterministic store identity, and exact initial-state digests | exact format-marker codec and `tests/store_format_marker.rs`; intent and receipt admission remain | In progress in #19 | | `KEEP-MIGRATION-003` | Migration revalidates version-1 head, catalog, pools, root identity, and writer authority before mutation | capability-relative integration tests | Planned in #19 | | `KEEP-MIGRATION-004` | Every partial migration prefix continues idempotently under writer authority | state-machine and recovery tests | Planned in #19 | | `KEEP-MIGRATION-005` | Unknown, out-of-order, substituted, corrupt, conflicting, or changed evidence is unrecoverable ambiguity | corruption and mutation matrix | Planned in #19 | diff --git a/src/adapters/mod.rs b/src/adapters/mod.rs index d55d271..20952e2 100644 --- a/src/adapters/mod.rs +++ b/src/adapters/mod.rs @@ -299,6 +299,7 @@ mod store_initialization_error; mod store_initialization_phase; mod store_initialization_receipt; mod store_initialization_storage; +mod store_migration; mod sync_capable_directory; #[cfg(test)] #[path = "../../tests/support/mod.rs"] @@ -481,6 +482,7 @@ pub use store_initialization_error::StoreInitializationError; pub use store_initialization_phase::StoreInitializationPhase; pub use store_initialization_receipt::StoreInitializationReceipt; pub use store_initialization_storage::StoreInitializationStorage; +pub use store_migration::*; pub use writer_lock_acquire_error::WriterLockAcquireError; pub use writer_lock_acquire_phase::WriterLockAcquirePhase; diff --git a/src/adapters/store_migration.rs b/src/adapters/store_migration.rs new file mode 100644 index 0000000..87d54bc --- /dev/null +++ b/src/adapters/store_migration.rs @@ -0,0 +1,16 @@ +//! Canonical version-2 store migration record adapters. + +mod admitted_format_marker; +mod canonical_format_marker; +mod format_definition_digest; +mod format_marker_decode_error; +mod format_marker_decode_error_display; +mod format_marker_decoder; +mod format_marker_digest; +mod format_marker_encoder; + +pub use admitted_format_marker::AdmittedStoreFormatMarker; +pub use canonical_format_marker::CanonicalStoreFormatMarker; +pub use format_definition_digest::StoreFormatDefinitionDigest; +pub use format_marker_decode_error::StoreFormatMarkerDecodeError; +pub use format_marker_digest::StoreFormatMarkerDigest; diff --git a/src/adapters/store_migration/admitted_format_marker.rs b/src/adapters/store_migration/admitted_format_marker.rs new file mode 100644 index 0000000..5a66024 --- /dev/null +++ b/src/adapters/store_migration/admitted_format_marker.rs @@ -0,0 +1,58 @@ +//! This boundary module owns admitted version-2 store-format markers. + +use super::{ + StoreFormatDefinitionDigest, StoreFormatMarkerDecodeError, StoreFormatMarkerDigest, + format_marker_decoder, +}; + +/// Borrowed canonical marker bytes with verified version-2 format identity. +#[must_use] +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub struct AdmittedStoreFormatMarker<'encoded> { + encoded: &'encoded [u8], + definition_digest: StoreFormatDefinitionDigest, + digest: StoreFormatMarkerDigest, +} + +impl<'encoded> AdmittedStoreFormatMarker<'encoded> { + /// Decodes and verifies one exact version-2 store-format marker. + /// + /// This operation performs no allocation or I/O. + /// + /// # Errors + /// + /// Returns [`StoreFormatMarkerDecodeError`] for wrong framing, + /// unsupported fields, checksum disagreement, or an unregistered + /// definition or namespace bound. + pub fn decode(encoded: &'encoded [u8]) -> Result { + format_marker_decoder::decode(encoded) + } + + /// Returns the exact borrowed canonical bytes. + #[must_use] + pub const fn encoded(&self) -> &'encoded [u8] { + self.encoded + } + + /// Returns the registered format-definition digest. + pub const fn definition_digest(&self) -> StoreFormatDefinitionDigest { + self.definition_digest + } + + /// Returns the identity of all marker bytes. + pub const fn digest(&self) -> StoreFormatMarkerDigest { + self.digest + } + + pub(super) const fn admitted( + encoded: &'encoded [u8], + definition_digest: StoreFormatDefinitionDigest, + digest: StoreFormatMarkerDigest, + ) -> Self { + Self { + encoded, + definition_digest, + digest, + } + } +} diff --git a/src/adapters/store_migration/canonical_format_marker.rs b/src/adapters/store_migration/canonical_format_marker.rs new file mode 100644 index 0000000..319be0e --- /dev/null +++ b/src/adapters/store_migration/canonical_format_marker.rs @@ -0,0 +1,36 @@ +//! This boundary module owns canonical version-2 store-format marker bytes. + +use super::{StoreFormatMarkerDigest, format_marker_decoder, format_marker_encoder}; + +/// Owned canonical version-2 store-format marker. +#[must_use] +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct CanonicalStoreFormatMarker { + encoded: [u8; format_marker_decoder::ENCODED_LENGTH], + digest: StoreFormatMarkerDigest, +} + +impl CanonicalStoreFormatMarker { + /// Constructs the one registered version-2 marker. + pub fn version_two() -> Self { + format_marker_encoder::version_two() + } + + /// Returns the canonical bytes. + #[must_use] + pub const fn encoded(&self) -> &[u8] { + &self.encoded + } + + /// Returns the identity of all marker bytes. + pub const fn digest(&self) -> StoreFormatMarkerDigest { + self.digest + } + + pub(super) const fn admitted( + encoded: [u8; format_marker_decoder::ENCODED_LENGTH], + digest: StoreFormatMarkerDigest, + ) -> Self { + Self { encoded, digest } + } +} diff --git a/src/adapters/store_migration/format_definition_digest.rs b/src/adapters/store_migration/format_definition_digest.rs new file mode 100644 index 0000000..f9b29bb --- /dev/null +++ b/src/adapters/store_migration/format_definition_digest.rs @@ -0,0 +1,25 @@ +//! This module owns the registered version-2 format-definition digest. + +/// Identity of one registered store-format definition. +#[must_use] +#[derive(Clone, Copy, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)] +pub struct StoreFormatDefinitionDigest([u8; 32]); + +impl StoreFormatDefinitionDigest { + /// Digest of the frozen `keep.segment-store/v2` definition. + pub const VERSION_TWO: Self = Self([ + 0x32, 0x38, 0x1f, 0x1a, 0xc3, 0x32, 0xd1, 0x27, 0x7a, 0x7e, 0x1f, 0xaf, 0x8f, 0x11, 0x57, + 0x69, 0x93, 0xcb, 0x55, 0xb7, 0xe8, 0x5d, 0x2a, 0x11, 0x0b, 0x74, 0xdc, 0x9c, 0x3b, 0x87, + 0x34, 0x27, + ]); + + /// Returns the raw digest bytes. + #[must_use] + pub const fn as_bytes(&self) -> &[u8; 32] { + &self.0 + } + + pub(super) const fn from_hash(hash: [u8; 32]) -> Self { + Self(hash) + } +} diff --git a/src/adapters/store_migration/format_marker_decode_error.rs b/src/adapters/store_migration/format_marker_decode_error.rs new file mode 100644 index 0000000..5102209 --- /dev/null +++ b/src/adapters/store_migration/format_marker_decode_error.rs @@ -0,0 +1,63 @@ +//! This boundary module owns store-format marker decoding failures. + +/// Failure to decode and admit one version-2 store-format marker. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum StoreFormatMarkerDecodeError { + /// The input was not exactly one complete marker. + WrongLength { + /// Required fixed width. + expected: usize, + /// Observed input width. + observed: usize, + }, + /// The fixed record magic was not canonical. + InvalidMagic { + /// Observed 16 magic bytes. + observed: [u8; 16], + }, + /// The format version is unsupported. + UnsupportedVersion { + /// Supported version. + expected: u16, + /// Observed version. + observed: u16, + }, + /// The fixed record length field was noncanonical. + InvalidRecordLength { + /// Required record length. + expected: u16, + /// Observed record length. + observed: u16, + }, + /// The marker carried unsupported flags. + UnsupportedFlags { + /// Observed flag bits. + observed: u32, + }, + /// Reserved bytes were nonzero. + NonZeroReserved { + /// Observed reserved field. + observed: u32, + }, + /// The checksum did not match the exact prefix. + ChecksumMismatch { + /// Computed canonical checksum. + expected: [u8; 32], + /// Checksum stored in the record. + observed: [u8; 32], + }, + /// The format-definition digest was not the registered version-2 value. + DefinitionDigestMismatch { + /// Registered version-2 definition digest. + expected: [u8; 32], + /// Observed definition digest. + observed: [u8; 32], + }, + /// The maximum namespace count was noncanonical. + InvalidMaximumNamespaceCount { + /// Required namespace bound. + expected: u32, + /// Observed namespace bound. + observed: u32, + }, +} diff --git a/src/adapters/store_migration/format_marker_decode_error_display.rs b/src/adapters/store_migration/format_marker_decode_error_display.rs new file mode 100644 index 0000000..435d2e3 --- /dev/null +++ b/src/adapters/store_migration/format_marker_decode_error_display.rs @@ -0,0 +1,50 @@ +//! This boundary module owns store-format marker decode diagnostics. + +use std::{error::Error, fmt}; + +use super::StoreFormatMarkerDecodeError; + +impl fmt::Display for StoreFormatMarkerDecodeError { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::WrongLength { expected, observed } => write!( + formatter, + "store-format marker has {observed} bytes; expected {expected}" + ), + Self::InvalidMagic { observed } => { + write!( + formatter, + "invalid store-format marker magic {observed:02x?}" + ) + } + Self::UnsupportedVersion { expected, observed } => write!( + formatter, + "unsupported store-format marker version {observed}; expected {expected}" + ), + Self::InvalidRecordLength { expected, observed } => write!( + formatter, + "store-format marker record length {observed}; expected {expected}" + ), + Self::UnsupportedFlags { observed } => write!( + formatter, + "unsupported store-format marker flags {observed:#010x}" + ), + Self::NonZeroReserved { observed } => write!( + formatter, + "store-format marker reserved field is nonzero: {observed:#010x}" + ), + Self::ChecksumMismatch { .. } => { + formatter.write_str("store-format marker checksum mismatch") + } + Self::DefinitionDigestMismatch { .. } => { + formatter.write_str("store-format definition digest mismatch") + } + Self::InvalidMaximumNamespaceCount { expected, observed } => write!( + formatter, + "store-format maximum namespace count {observed}; expected {expected}" + ), + } + } +} + +impl Error for StoreFormatMarkerDecodeError {} diff --git a/src/adapters/store_migration/format_marker_decoder.rs b/src/adapters/store_migration/format_marker_decoder.rs new file mode 100644 index 0000000..75ca8f1 --- /dev/null +++ b/src/adapters/store_migration/format_marker_decoder.rs @@ -0,0 +1,137 @@ +//! This boundary module owns store-format marker decoding order. + +use super::{ + AdmittedStoreFormatMarker, StoreFormatDefinitionDigest, StoreFormatMarkerDecodeError, + StoreFormatMarkerDigest, +}; +use crate::RetentionManifest; + +pub(super) const ENCODED_LENGTH: usize = 96; +pub(super) const CHECKSUM_OFFSET: usize = 64; +pub(super) const MAGIC: [u8; 16] = *b"KEEP:STORE:V2\0\0\0"; +pub(super) const VERSION: u16 = 2; +pub(super) const RECORD_LENGTH: u16 = 96; +const CHECKSUM_DOMAIN: &[u8] = b"keep.segment-store-marker-checksum/v2\0"; +const DIGEST_DOMAIN: &[u8] = b"keep.store-format-marker/v2\0"; + +pub(super) fn decode( + encoded: &[u8], +) -> Result, StoreFormatMarkerDecodeError> { + require_length(encoded)?; + validate_fixed_fields(encoded)?; + verify_checksum(encoded)?; + let definition_hash = read_array(encoded, 24)?; + let definition_digest = StoreFormatDefinitionDigest::from_hash(definition_hash); + if definition_digest != StoreFormatDefinitionDigest::VERSION_TWO { + return Err(StoreFormatMarkerDecodeError::DefinitionDigestMismatch { + expected: *StoreFormatDefinitionDigest::VERSION_TWO.as_bytes(), + observed: definition_hash, + }); + } + let maximum_namespace_count = read_u32(encoded, 56)?; + if maximum_namespace_count != RetentionManifest::MAXIMUM_ENTRY_COUNT { + return Err(StoreFormatMarkerDecodeError::InvalidMaximumNamespaceCount { + expected: RetentionManifest::MAXIMUM_ENTRY_COUNT, + observed: maximum_namespace_count, + }); + } + Ok(AdmittedStoreFormatMarker::admitted( + encoded, + definition_digest, + digest(encoded), + )) +} + +fn validate_fixed_fields(encoded: &[u8]) -> Result<(), StoreFormatMarkerDecodeError> { + let magic = read_array(encoded, 0)?; + if magic != MAGIC { + return Err(StoreFormatMarkerDecodeError::InvalidMagic { observed: magic }); + } + let version = read_u16(encoded, 16)?; + if version != VERSION { + return Err(StoreFormatMarkerDecodeError::UnsupportedVersion { + expected: VERSION, + observed: version, + }); + } + let record_length = read_u16(encoded, 18)?; + if record_length != RECORD_LENGTH { + return Err(StoreFormatMarkerDecodeError::InvalidRecordLength { + expected: RECORD_LENGTH, + observed: record_length, + }); + } + let flags = read_u32(encoded, 20)?; + if flags != 0 { + return Err(StoreFormatMarkerDecodeError::UnsupportedFlags { observed: flags }); + } + let reserved = read_u32(encoded, 60)?; + if reserved != 0 { + return Err(StoreFormatMarkerDecodeError::NonZeroReserved { observed: reserved }); + } + Ok(()) +} + +fn verify_checksum(encoded: &[u8]) -> Result<(), StoreFormatMarkerDecodeError> { + let preimage = encoded + .get(..CHECKSUM_OFFSET) + .ok_or_else(|| wrong_length(encoded))?; + let observed = read_array(encoded, CHECKSUM_OFFSET)?; + let expected = checksum(preimage); + if observed == expected { + Ok(()) + } else { + Err(StoreFormatMarkerDecodeError::ChecksumMismatch { expected, observed }) + } +} + +pub(super) fn checksum(preimage: &[u8]) -> [u8; 32] { + hash(CHECKSUM_DOMAIN, preimage) +} + +pub(super) fn digest(encoded: &[u8]) -> StoreFormatMarkerDigest { + StoreFormatMarkerDigest::from_hash(hash(DIGEST_DOMAIN, encoded)) +} + +fn hash(domain: &[u8], bytes: &[u8]) -> [u8; 32] { + let mut hasher = blake3::Hasher::new(); + hasher.update(domain); + hasher.update(bytes); + *hasher.finalize().as_bytes() +} + +const fn require_length(encoded: &[u8]) -> Result<(), StoreFormatMarkerDecodeError> { + if encoded.len() == ENCODED_LENGTH { + Ok(()) + } else { + Err(wrong_length(encoded)) + } +} + +const fn wrong_length(encoded: &[u8]) -> StoreFormatMarkerDecodeError { + StoreFormatMarkerDecodeError::WrongLength { + expected: ENCODED_LENGTH, + observed: encoded.len(), + } +} + +fn read_u16(encoded: &[u8], offset: usize) -> Result { + read_array(encoded, offset).map(u16::from_be_bytes) +} + +fn read_u32(encoded: &[u8], offset: usize) -> Result { + read_array(encoded, offset).map(u32::from_be_bytes) +} + +fn read_array( + encoded: &[u8], + offset: usize, +) -> Result<[u8; WIDTH], StoreFormatMarkerDecodeError> { + let Some(end) = offset.checked_add(WIDTH) else { + return Err(wrong_length(encoded)); + }; + let bytes = encoded + .get(offset..end) + .ok_or_else(|| wrong_length(encoded))?; + <[u8; WIDTH]>::try_from(bytes).map_err(|_| wrong_length(encoded)) +} diff --git a/src/adapters/store_migration/format_marker_digest.rs b/src/adapters/store_migration/format_marker_digest.rs new file mode 100644 index 0000000..39fc208 --- /dev/null +++ b/src/adapters/store_migration/format_marker_digest.rs @@ -0,0 +1,18 @@ +//! This module owns version-2 store-format marker identity. + +/// Domain-separated identity of all canonical store-format marker bytes. +#[must_use] +#[derive(Clone, Copy, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)] +pub struct StoreFormatMarkerDigest([u8; 32]); + +impl StoreFormatMarkerDigest { + /// Returns the raw digest bytes. + #[must_use] + pub const fn as_bytes(&self) -> &[u8; 32] { + &self.0 + } + + pub(super) const fn from_hash(hash: [u8; 32]) -> Self { + Self(hash) + } +} diff --git a/src/adapters/store_migration/format_marker_encoder.rs b/src/adapters/store_migration/format_marker_encoder.rs new file mode 100644 index 0000000..14223b8 --- /dev/null +++ b/src/adapters/store_migration/format_marker_encoder.rs @@ -0,0 +1,27 @@ +//! This boundary module owns canonical version-2 format-marker encoding. + +use super::{ + CanonicalStoreFormatMarker, StoreFormatDefinitionDigest, format_marker_decoder as format, +}; +use crate::RetentionManifest; + +pub(super) fn version_two() -> CanonicalStoreFormatMarker { + let mut encoded = [0_u8; format::ENCODED_LENGTH]; + let (preimage, checksum_slot) = encoded.split_at_mut(format::CHECKSUM_OFFSET); + let (magic, remaining) = preimage.split_at_mut(16); + magic.copy_from_slice(&format::MAGIC); + let (version, remaining) = remaining.split_at_mut(2); + version.copy_from_slice(&format::VERSION.to_be_bytes()); + let (record_length, remaining) = remaining.split_at_mut(2); + record_length.copy_from_slice(&format::RECORD_LENGTH.to_be_bytes()); + let (flags, remaining) = remaining.split_at_mut(4); + flags.copy_from_slice(&0_u32.to_be_bytes()); + let (definition_digest, remaining) = remaining.split_at_mut(32); + definition_digest.copy_from_slice(StoreFormatDefinitionDigest::VERSION_TWO.as_bytes()); + let (maximum_namespace_count, remaining) = remaining.split_at_mut(4); + maximum_namespace_count.copy_from_slice(&RetentionManifest::MAXIMUM_ENTRY_COUNT.to_be_bytes()); + let (_reserved, _complete) = remaining.split_at_mut(4); + checksum_slot.copy_from_slice(&format::checksum(preimage)); + let digest = format::digest(&encoded); + CanonicalStoreFormatMarker::admitted(encoded, digest) +} diff --git a/src/lib.rs b/src/lib.rs index 91cd10a..118d258 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -29,8 +29,10 @@ //! with a blocking storage capability port are available. Storage-independent //! preparation binds preflight to exact canonical manifest and head successors. //! Ordered publication revalidates authority, executes all durability phases, -//! and returns a complete receipt. Filesystem execution, recovery, and garbage -//! collection remain intentionally absent. +//! and returns a complete receipt. The exact version-2 store-format marker has +//! canonical encoding, registered-definition admission, checksum verification, +//! and domain-separated identity. Filesystem migration, retention execution, +//! recovery, and garbage collection remain intentionally absent. #[cfg(test)] extern crate self as keep; @@ -49,41 +51,41 @@ mod retention; pub use adapters::RepositoryInitializationStorage; pub use adapters::{ AdmittedCatalog, AdmittedRecoveryStageBytes, AdmittedSegment, AdmittedSegmentRecord, - BlobIdBinaryParseError, BlobIdTextParseError, CanonicalCatalog, CanonicalLayoutRecord, - CanonicalPublicationHead, CatalogAdmissionError, CatalogAllocationPhase, CatalogDecodeError, - CatalogEncodeError, CatalogEntryDecodeError, CatalogPublicationError, - CatalogPublicationExpectation, CatalogPublicationOutcome, CatalogPublicationPhase, - CatalogPublicationReadiness, CatalogPublicationReceipt, CatalogPublicationStorage, - CatalogRestartArtifact, CatalogRestartByteLimit, CatalogRestartByteLimitError, - CatalogRestartError, CatalogRestartPhase, CatalogRestartPolicy, CatalogSnapshot, - CatalogSnapshotError, CatalogSuccessor, CatalogTransitionError, ChecksummedCatalog, - ChecksummedPublicationHead, ChecksummedSegmentRecord, ClosedSegment, - FilesystemCatalogPublicationError, FilesystemCatalogPublisher, FilesystemCatalogSnapshot, - FilesystemPlatformAdmission, FilesystemPlatformAdmissionError, - FilesystemRecoveryInventoryReader, FilesystemRecoveryNextHeadFinalizationOpenError, - FilesystemRecoveryNextHeadFinalizer, FilesystemRecoverySegmentResumeOpenError, - FilesystemRecoverySegmentResumer, FilesystemRecoverySegmentStage, - FilesystemRecoveryStageCompleter, FilesystemRecoveryStageCompletionOpenError, - FilesystemRecoveryStageDiscardOpenError, FilesystemRecoveryStageDiscarder, - FilesystemRecoveryStageError, FilesystemSegmentStage, FilesystemWriterLock, LayoutDecodeError, - LayoutDecodePolicy, LayoutEncodeError, LayoutIdBinaryParseError, LayoutIdTextParseError, - OpenedReusableSegment, PublicationHeadDecodeError, RecoveryCatalogStage, - RecoveryCatalogStageError, RecoveryEntryName, RecoveryEntryNameError, RecoveryEntryRole, - RecoveryInventory, RecoveryInventoryEntry, RecoveryInventoryError, RecoveryInventoryLimit, - RecoveryInventoryLimitError, RecoveryInventoryOperation, RecoveryInventoryStorage, - RecoveryNameClassificationError, RecoveryNameManifest, RecoveryNamedEntry, RecoveryNamespace, - RecoveryNextHeadFinalizationError, RecoveryNextHeadFinalizationOutcome, - RecoveryNextHeadFinalizationPlanError, RecoveryNextHeadFinalizationReadiness, - RecoveryNextHeadFinalizationReceipt, RecoveryNextHeadFinalizationRequest, - RecoveryNextHeadFinalizationStorage, RecoveryNextHeadFinalizationStorageError, - RecoveryNextHeadFinalizationTarget, RecoveryNextHeadStage, RecoveryNextHeadStageError, - RecoveryPoolNameError, RecoveryRequiredEntry, RecoverySegmentResumeError, - RecoverySegmentResumePlanError, RecoverySegmentResumeRequest, RecoverySegmentResumeStorage, - RecoverySegmentResumeStorageError, RecoverySegmentStage, RecoverySegmentStageError, - RecoverySegmentTruncation, RecoveryStage, RecoveryStageAssessment, - RecoveryStageAssessmentError, RecoveryStageByteAdmissionError, RecoveryStageCompletionError, - RecoveryStageCompletionPlanError, RecoveryStageCompletionPool, RecoveryStageCompletionReceipt, - RecoveryStageCompletionRequest, RecoveryStageCompletionStorage, + AdmittedStoreFormatMarker, BlobIdBinaryParseError, BlobIdTextParseError, CanonicalCatalog, + CanonicalLayoutRecord, CanonicalPublicationHead, CanonicalStoreFormatMarker, + CatalogAdmissionError, CatalogAllocationPhase, CatalogDecodeError, CatalogEncodeError, + CatalogEntryDecodeError, CatalogPublicationError, CatalogPublicationExpectation, + CatalogPublicationOutcome, CatalogPublicationPhase, CatalogPublicationReadiness, + CatalogPublicationReceipt, CatalogPublicationStorage, CatalogRestartArtifact, + CatalogRestartByteLimit, CatalogRestartByteLimitError, CatalogRestartError, + CatalogRestartPhase, CatalogRestartPolicy, CatalogSnapshot, CatalogSnapshotError, + CatalogSuccessor, CatalogTransitionError, ChecksummedCatalog, ChecksummedPublicationHead, + ChecksummedSegmentRecord, ClosedSegment, FilesystemCatalogPublicationError, + FilesystemCatalogPublisher, FilesystemCatalogSnapshot, FilesystemPlatformAdmission, + FilesystemPlatformAdmissionError, FilesystemRecoveryInventoryReader, + FilesystemRecoveryNextHeadFinalizationOpenError, FilesystemRecoveryNextHeadFinalizer, + FilesystemRecoverySegmentResumeOpenError, FilesystemRecoverySegmentResumer, + FilesystemRecoverySegmentStage, FilesystemRecoveryStageCompleter, + FilesystemRecoveryStageCompletionOpenError, FilesystemRecoveryStageDiscardOpenError, + FilesystemRecoveryStageDiscarder, FilesystemRecoveryStageError, FilesystemSegmentStage, + FilesystemWriterLock, LayoutDecodeError, LayoutDecodePolicy, LayoutEncodeError, + LayoutIdBinaryParseError, LayoutIdTextParseError, OpenedReusableSegment, + PublicationHeadDecodeError, RecoveryCatalogStage, RecoveryCatalogStageError, RecoveryEntryName, + RecoveryEntryNameError, RecoveryEntryRole, RecoveryInventory, RecoveryInventoryEntry, + RecoveryInventoryError, RecoveryInventoryLimit, RecoveryInventoryLimitError, + RecoveryInventoryOperation, RecoveryInventoryStorage, RecoveryNameClassificationError, + RecoveryNameManifest, RecoveryNamedEntry, RecoveryNamespace, RecoveryNextHeadFinalizationError, + RecoveryNextHeadFinalizationOutcome, RecoveryNextHeadFinalizationPlanError, + RecoveryNextHeadFinalizationReadiness, RecoveryNextHeadFinalizationReceipt, + RecoveryNextHeadFinalizationRequest, RecoveryNextHeadFinalizationStorage, + RecoveryNextHeadFinalizationStorageError, RecoveryNextHeadFinalizationTarget, + RecoveryNextHeadStage, RecoveryNextHeadStageError, RecoveryPoolNameError, + RecoveryRequiredEntry, RecoverySegmentResumeError, RecoverySegmentResumePlanError, + RecoverySegmentResumeRequest, RecoverySegmentResumeStorage, RecoverySegmentResumeStorageError, + RecoverySegmentStage, RecoverySegmentStageError, RecoverySegmentTruncation, RecoveryStage, + RecoveryStageAssessment, RecoveryStageAssessmentError, RecoveryStageByteAdmissionError, + RecoveryStageCompletionError, RecoveryStageCompletionPlanError, RecoveryStageCompletionPool, + RecoveryStageCompletionReceipt, RecoveryStageCompletionRequest, RecoveryStageCompletionStorage, RecoveryStageCompletionStorageError, RecoveryStageCompletionTarget, RecoveryStageDiscardError, RecoveryStageDiscardOutcome, RecoveryStageDiscardPlanError, RecoveryStageDiscardReason, RecoveryStageDiscardReceipt, RecoveryStageDiscardRequest, RecoveryStageDiscardStorage, @@ -98,6 +100,7 @@ pub use adapters::{ SegmentRecordLength, SegmentRecordLimit, SegmentRecordLimitError, SegmentRecordPayloadLength, SegmentRecords, SegmentSeal, SegmentSealError, SegmentStage, SegmentStageCreateError, SegmentWriteError, SegmentWritePhase, StagedSegment, StorageProfileIdParseError, + StoreFormatDefinitionDigest, StoreFormatMarkerDecodeError, StoreFormatMarkerDigest, StoreInitializationError, StoreInitializationPhase, StoreInitializationReceipt, StoreInitializationStorage, WriterLockAcquireError, WriterLockAcquirePhase, admit_recovery_stage_bytes, assess_recovery_stage, classify_recovery_catalog_stage, diff --git a/tests/store_format_marker.rs b/tests/store_format_marker.rs new file mode 100644 index 0000000..bc23f60 --- /dev/null +++ b/tests/store_format_marker.rs @@ -0,0 +1,168 @@ +//! Canonical version-2 store-format marker laws. + +mod support; + +use std::io; + +use keep::{ + AdmittedStoreFormatMarker, CanonicalStoreFormatMarker, StoreFormatDefinitionDigest, + StoreFormatMarkerDecodeError, +}; + +const FORMAT_MARKER: &str = include_str!("../conformance/segment-store/v2/format-marker.hex"); +const MARKER_DIGEST: [u8; 32] = [ + 0x4b, 0x06, 0x3c, 0x32, 0x90, 0x85, 0xab, 0xde, 0xbe, 0x86, 0xb2, 0x56, 0xd5, 0x31, 0xb1, 0x12, + 0xc7, 0xea, 0x33, 0xcb, 0x2f, 0x54, 0x5c, 0xaa, 0x40, 0xa7, 0xa8, 0x69, 0xff, 0x33, 0x37, 0xce, +]; + +#[test] +fn marker_reproduces_the_frozen_version_two_record() -> Result<(), Box> { + let bytes = fixture_bytes()?; + let admitted = AdmittedStoreFormatMarker::decode(&bytes)?; + let canonical = CanonicalStoreFormatMarker::version_two(); + + assert_eq!(admitted.encoded(), bytes); + assert_eq!( + admitted.definition_digest(), + StoreFormatDefinitionDigest::VERSION_TWO + ); + assert_eq!(admitted.digest().as_bytes(), &MARKER_DIGEST); + assert_eq!(canonical.encoded(), bytes); + assert_eq!(canonical.digest(), admitted.digest()); + Ok(()) +} + +#[test] +fn marker_framing_has_exact_first_refusals() -> Result<(), Box> { + let bytes = fixture_bytes()?; + let mut truncated = bytes.clone(); + assert!(truncated.pop().is_some()); + assert!(matches!( + AdmittedStoreFormatMarker::decode(&truncated), + Err(StoreFormatMarkerDecodeError::WrongLength { + expected: 96, + observed: 95, + }) + )); + + assert_fixed_refusal( + 0, + StoreFormatMarkerDecodeError::InvalidMagic { + observed: mutated_array::<16>(&bytes, 0, 0)?, + }, + )?; + assert_fixed_refusal( + 17, + StoreFormatMarkerDecodeError::UnsupportedVersion { + expected: 2, + observed: 3, + }, + )?; + assert_fixed_refusal( + 19, + StoreFormatMarkerDecodeError::InvalidRecordLength { + expected: 96, + observed: 97, + }, + )?; + assert_fixed_refusal( + 23, + StoreFormatMarkerDecodeError::UnsupportedFlags { observed: 1 }, + )?; + assert_fixed_refusal( + 63, + StoreFormatMarkerDecodeError::NonZeroReserved { observed: 1 }, + )?; + Ok(()) +} + +#[test] +fn checksum_precedes_registered_marker_semantics() -> Result<(), Box> { + let mut definition = fixture_bytes()?; + flip_byte(&mut definition, 24)?; + assert!(matches!( + AdmittedStoreFormatMarker::decode(&definition), + Err(StoreFormatMarkerDecodeError::ChecksumMismatch { .. }) + )); + refresh_checksum(&mut definition)?; + assert!(matches!( + AdmittedStoreFormatMarker::decode(&definition), + Err(StoreFormatMarkerDecodeError::DefinitionDigestMismatch { .. }) + )); + + let mut namespace_limit = fixture_bytes()?; + namespace_limit + .get_mut(56..60) + .ok_or_else(|| io::Error::other("marker lacks namespace limit"))? + .copy_from_slice(&4_095_u32.to_be_bytes()); + refresh_checksum(&mut namespace_limit)?; + assert_eq!( + AdmittedStoreFormatMarker::decode(&namespace_limit), + Err(StoreFormatMarkerDecodeError::InvalidMaximumNamespaceCount { + expected: 4_096, + observed: 4_095, + }) + ); + + let mut checksum = fixture_bytes()?; + flip_byte(&mut checksum, 95)?; + assert!(matches!( + AdmittedStoreFormatMarker::decode(&checksum), + Err(StoreFormatMarkerDecodeError::ChecksumMismatch { .. }) + )); + Ok(()) +} + +fn assert_fixed_refusal( + offset: usize, + expected: StoreFormatMarkerDecodeError, +) -> Result<(), Box> { + let mut bytes = fixture_bytes()?; + flip_byte(&mut bytes, offset)?; + assert_eq!(AdmittedStoreFormatMarker::decode(&bytes), Err(expected)); + Ok(()) +} + +fn mutated_array( + bytes: &[u8], + offset: usize, + relative: usize, +) -> Result<[u8; WIDTH], io::Error> { + let end = offset + .checked_add(WIDTH) + .ok_or_else(|| io::Error::other("marker field offset overflow"))?; + let mut observed = <[u8; WIDTH]>::try_from( + bytes + .get(offset..end) + .ok_or_else(|| io::Error::other("marker lacks fixed field"))?, + ) + .map_err(|_| io::Error::other("marker field width mismatch"))?; + let byte = observed + .get_mut(relative) + .ok_or_else(|| io::Error::other("marker mutation offset is out of bounds"))?; + *byte ^= 1; + Ok(observed) +} + +fn flip_byte(bytes: &mut [u8], offset: usize) -> Result<(), io::Error> { + let byte = bytes + .get_mut(offset) + .ok_or_else(|| io::Error::other("marker mutation offset is out of bounds"))?; + *byte ^= 1; + Ok(()) +} + +fn refresh_checksum(bytes: &mut [u8]) -> Result<(), io::Error> { + let (preimage, checksum) = bytes + .split_at_mut_checked(64) + .ok_or_else(|| io::Error::other("marker lacks checksum boundary"))?; + let mut hasher = blake3::Hasher::new(); + hasher.update(b"keep.segment-store-marker-checksum/v2\0"); + hasher.update(preimage); + checksum.copy_from_slice(hasher.finalize().as_bytes()); + Ok(()) +} + +fn fixture_bytes() -> Result, io::Error> { + support::decode_hex(FORMAT_MARKER.trim_end()) +} From fc20097f4e0cc5cd88d3ea6e03d539a030e515f8 Mon Sep 17 00:00:00 2001 From: James Ross Date: Thu, 30 Jul 2026 02:04:19 -0700 Subject: [PATCH 031/111] Add: Admit store migration intents --- CHANGELOG.md | 12 +- README.md | 22 +- docs/formats/segment-store-v2/recovery.md | 8 +- docs/formats/segment-store-v2/requirements.md | 2 +- src/adapters/store_migration.rs | 17 ++ .../admitted_migration_intent.rs | 122 +++++++++++ .../immutable_pool_inventory_digest.rs | 20 ++ .../store_migration/migration_intent_bytes.rs | 54 +++++ .../migration_intent_decode_error.rs | 84 ++++++++ .../migration_intent_decode_error_display.rs | 64 ++++++ .../migration_intent_decoder.rs | 181 ++++++++++++++++ .../migration_intent_digest.rs | 18 ++ .../store_migration/store_identifier.rs | 18 ++ .../store_migration/store_root_identity.rs | 38 ++++ src/lib.rs | 48 +++-- tests/store_migration_intent.rs | 200 ++++++++++++++++++ tests/store_migration_intent/fixture.rs | 33 +++ 17 files changed, 898 insertions(+), 43 deletions(-) create mode 100644 src/adapters/store_migration/admitted_migration_intent.rs create mode 100644 src/adapters/store_migration/immutable_pool_inventory_digest.rs create mode 100644 src/adapters/store_migration/migration_intent_bytes.rs create mode 100644 src/adapters/store_migration/migration_intent_decode_error.rs create mode 100644 src/adapters/store_migration/migration_intent_decode_error_display.rs create mode 100644 src/adapters/store_migration/migration_intent_decoder.rs create mode 100644 src/adapters/store_migration/migration_intent_digest.rs create mode 100644 src/adapters/store_migration/store_identifier.rs create mode 100644 src/adapters/store_migration/store_root_identity.rs create mode 100644 tests/store_migration_intent.rs create mode 100644 tests/store_migration_intent/fixture.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index 2afa12d..beb0ef4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,12 +10,12 @@ after its public API and format compatibility policies are established. ### Changed -- The version-2 store-format marker now has exact canonical encoding, - registered-definition admission, checksum verification, and domain-separated - identity. Retention transition preflight combines expected-generation - planning with deterministic closure verification against one pinned catalog; - authority-revalidated 17-phase orchestration returns an unforgeable - complete-coordinate receipt after durable cleanup. +- The version-2 store-format marker now has exact canonical encoding and + admission, while migration intents admit exact catalog, predecessor, root, + definition, store-identity, checksum, and digest coordinates. Retention + preflight combines expected-generation planning with deterministic closure + verification; authority-revalidated 17-phase orchestration returns an + unforgeable complete-coordinate receipt after durable cleanup. - Repository crash-matrix execution now terminates isolated writer process groups at all 105 canonical before/during/after coordinates, retains open writer and stage authority until termination, executes production diff --git a/README.md b/README.md index 00f27e8..7406736 100644 --- a/README.md +++ b/README.md @@ -115,17 +115,17 @@ bytes, checks hard-link identity and writer-lock release, runs the production recovery classifiers and immutable-artifact admission, and reconstructs the exact published generation and visible one-zero chunk when `HEAD` exists. This matrix proves application process-death behavior; it does not simulate host -power loss. Canonical version-2 store-format marker encoding and admission are -implemented. Version-2 retention values; canonical in-memory root, global -manifest, and retention-head codecs; storage-independent expected-state -transition planning; deterministic bounded closure verification against a -pinned catalog; a combined transition preflight proof; and the exact 17-phase -publication vocabulary with a blocking storage capability port are -implemented. Private-field proofs retain every receipt coordinate. Ordered -storage-port orchestration revalidates current authority, executes all 17 -durability phases, and returns a consequential complete-coordinate receipt. -Filesystem migration, retention execution, recovery, compaction, and garbage -collection remain planned. +power loss. Canonical version-2 store-format marker encoding and exact +migration-intent admission are implemented. Version-2 retention values; +canonical in-memory root, global manifest, and retention-head codecs; +storage-independent expected-state transition planning; deterministic bounded +closure verification against a pinned catalog; a combined transition preflight +proof; and the exact 17-phase publication vocabulary with a blocking storage +capability port are implemented. Private-field proofs retain every receipt +coordinate. Ordered storage-port orchestration revalidates current authority, +executes all 17 durability phases, and returns a consequential +complete-coordinate receipt. Filesystem migration, retention execution, +recovery, compaction, and garbage collection remain planned. Presence in the reference CAS does not claim durable retention or crash recovery. diff --git a/docs/formats/segment-store-v2/recovery.md b/docs/formats/segment-store-v2/recovery.md index ee9bcb3..4f7ad6b 100644 --- a/docs/formats/segment-store-v2/recovery.md +++ b/docs/formats/segment-store-v2/recovery.md @@ -60,10 +60,10 @@ The format-definition digest is BLAKE3-256 of its domain followed by the exact corpus `definition.tsv` bytes. The format-marker digest is BLAKE3-256 of `keep.store-format-marker/v2\0` followed by all 96 marker bytes. -`CanonicalStoreFormatMarker` produces the one registered marker, and -`AdmittedStoreFormatMarker` admits exact canonical bytes only after framing, -checksum, definition, and namespace-bound validation. Store detection and -filesystem migration remain absent. +`CanonicalStoreFormatMarker` produces the registered marker; `AdmittedStoreFormatMarker` admits its framing, checksum, definition, and namespace bound. +`AdmittedStoreMigrationIntent` admits the exact intent framing, checksum, catalog coordinates, predecessor law, definition, and derived store identity. +These record boundaries do not prove the named live inventory or physical root, +detect the store version, or execute filesystem migration. ## Reader fence diff --git a/docs/formats/segment-store-v2/requirements.md b/docs/formats/segment-store-v2/requirements.md index c84c9cf..afb0409 100644 --- a/docs/formats/segment-store-v2/requirements.md +++ b/docs/formats/segment-store-v2/requirements.md @@ -29,7 +29,7 @@ case is not evidence. | ID | Requirement | Evidence | Status | | --- | --- | --- | --- | | `KEEP-MIGRATION-001` | Exact version-1 stores remain admitted until a durable migration artifact exists | compatibility fixtures | Planned in #19 | -| `KEEP-MIGRATION-002` | Format marker, intent, and receipt have complete fixed byte tables, named domains, bounds, checksums, deterministic store identity, and exact initial-state digests | exact format-marker codec and `tests/store_format_marker.rs`; intent and receipt admission remain | In progress in #19 | +| `KEEP-MIGRATION-002` | Format marker, intent, and receipt have complete fixed byte tables, named domains, bounds, checksums, deterministic store identity, and exact initial-state digests | exact marker and intent admission in `tests/store_format_marker.rs` and `tests/store_migration_intent.rs`; receipt admission remains | In progress in #19 | | `KEEP-MIGRATION-003` | Migration revalidates version-1 head, catalog, pools, root identity, and writer authority before mutation | capability-relative integration tests | Planned in #19 | | `KEEP-MIGRATION-004` | Every partial migration prefix continues idempotently under writer authority | state-machine and recovery tests | Planned in #19 | | `KEEP-MIGRATION-005` | Unknown, out-of-order, substituted, corrupt, conflicting, or changed evidence is unrecoverable ambiguity | corruption and mutation matrix | Planned in #19 | diff --git a/src/adapters/store_migration.rs b/src/adapters/store_migration.rs index 87d54bc..19effb7 100644 --- a/src/adapters/store_migration.rs +++ b/src/adapters/store_migration.rs @@ -1,6 +1,7 @@ //! Canonical version-2 store migration record adapters. mod admitted_format_marker; +mod admitted_migration_intent; mod canonical_format_marker; mod format_definition_digest; mod format_marker_decode_error; @@ -8,9 +9,25 @@ mod format_marker_decode_error_display; mod format_marker_decoder; mod format_marker_digest; mod format_marker_encoder; +mod immutable_pool_inventory_digest; +mod migration_intent_bytes; +mod migration_intent_decode_error; +mod migration_intent_decode_error_display; +mod migration_intent_decoder; +mod migration_intent_digest; +mod store_identifier; +mod store_root_identity; pub use admitted_format_marker::AdmittedStoreFormatMarker; +pub use admitted_migration_intent::AdmittedStoreMigrationIntent; pub use canonical_format_marker::CanonicalStoreFormatMarker; pub use format_definition_digest::StoreFormatDefinitionDigest; pub use format_marker_decode_error::StoreFormatMarkerDecodeError; pub use format_marker_digest::StoreFormatMarkerDigest; +pub use immutable_pool_inventory_digest::ImmutablePoolInventoryDigest; +pub use migration_intent_decode_error::StoreMigrationIntentDecodeError; +pub use migration_intent_digest::StoreMigrationIntentDigest; +pub use store_identifier::StoreIdentifier; +pub use store_root_identity::{ + StoreRootDeviceIdentity, StoreRootFileIdentity, StoreRootMountIdentity, +}; diff --git a/src/adapters/store_migration/admitted_migration_intent.rs b/src/adapters/store_migration/admitted_migration_intent.rs new file mode 100644 index 0000000..fa625b1 --- /dev/null +++ b/src/adapters/store_migration/admitted_migration_intent.rs @@ -0,0 +1,122 @@ +//! This boundary module owns admitted store-migration intent evidence. + +use super::{ + ImmutablePoolInventoryDigest, StoreFormatDefinitionDigest, StoreIdentifier, + StoreMigrationIntentDecodeError, StoreMigrationIntentDigest, StoreRootDeviceIdentity, + StoreRootFileIdentity, StoreRootMountIdentity, migration_intent_decoder, +}; +use crate::{CatalogDigest, CatalogGeneration, CatalogLength}; + +/// Semantic fields admitted from one canonical migration intent. +#[derive(Clone, Debug, Eq, PartialEq)] +pub(super) struct StoreMigrationIntentFields { + pub(super) catalog_generation: CatalogGeneration, + pub(super) catalog_length: CatalogLength, + pub(super) catalog_digest: CatalogDigest, + pub(super) predecessor_catalog_digest: Option, + pub(super) inventory_digest: ImmutablePoolInventoryDigest, + pub(super) root_device_identity: StoreRootDeviceIdentity, + pub(super) root_mount_identity: StoreRootMountIdentity, + pub(super) root_file_identity: StoreRootFileIdentity, + pub(super) target_definition_digest: StoreFormatDefinitionDigest, + pub(super) store_identifier: StoreIdentifier, +} + +/// Borrowed canonical version-2 store-migration intent. +/// +/// Admission proves record framing, integrity, internal generation laws, the +/// registered target definition, and deterministic store identity. It does not +/// prove that the named catalog, inventory, or physical root is current. +#[must_use] +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct AdmittedStoreMigrationIntent<'encoded> { + encoded: &'encoded [u8], + fields: StoreMigrationIntentFields, + digest: StoreMigrationIntentDigest, +} + +impl<'encoded> AdmittedStoreMigrationIntent<'encoded> { + /// Decodes and admits one exact canonical migration intent. + /// + /// # Errors + /// + /// Returns [`StoreMigrationIntentDecodeError`] for invalid framing, + /// integrity, catalog coordinates, predecessor state, definition identity, + /// or store identity. + pub fn decode(encoded: &'encoded [u8]) -> Result { + migration_intent_decoder::decode(encoded) + } + + /// Returns the exact borrowed canonical bytes. + #[must_use] + pub const fn encoded(&self) -> &'encoded [u8] { + self.encoded + } + + /// Returns the positive catalog generation named by the intent. + pub const fn catalog_generation(&self) -> CatalogGeneration { + self.fields.catalog_generation + } + + /// Returns the exact admitted catalog byte length. + pub const fn catalog_length(&self) -> CatalogLength { + self.fields.catalog_length + } + + /// Returns the catalog digest named by the intent. + pub const fn catalog_digest(&self) -> CatalogDigest { + self.fields.catalog_digest + } + + /// Returns the generation-relative predecessor digest. + pub const fn predecessor_catalog_digest(&self) -> Option { + self.fields.predecessor_catalog_digest + } + + /// Returns the immutable-pool inventory digest named by the intent. + pub const fn inventory_digest(&self) -> ImmutablePoolInventoryDigest { + self.fields.inventory_digest + } + + /// Returns the serialized root device coordinate named by the intent. + pub const fn root_device_identity(&self) -> StoreRootDeviceIdentity { + self.fields.root_device_identity + } + + /// Returns the serialized root mount coordinate named by the intent. + pub const fn root_mount_identity(&self) -> StoreRootMountIdentity { + self.fields.root_mount_identity + } + + /// Returns the serialized root file coordinate named by the intent. + pub const fn root_file_identity(&self) -> StoreRootFileIdentity { + self.fields.root_file_identity + } + + /// Returns the registered target format-definition digest. + pub const fn target_definition_digest(&self) -> StoreFormatDefinitionDigest { + self.fields.target_definition_digest + } + + /// Returns the deterministic logical store identity. + pub const fn store_identifier(&self) -> StoreIdentifier { + self.fields.store_identifier + } + + /// Returns the domain-separated identity of all intent bytes. + pub const fn digest(&self) -> StoreMigrationIntentDigest { + self.digest + } + + pub(super) const fn admitted( + encoded: &'encoded [u8], + fields: StoreMigrationIntentFields, + digest: StoreMigrationIntentDigest, + ) -> Self { + Self { + encoded, + fields, + digest, + } + } +} diff --git a/src/adapters/store_migration/immutable_pool_inventory_digest.rs b/src/adapters/store_migration/immutable_pool_inventory_digest.rs new file mode 100644 index 0000000..188c0aa --- /dev/null +++ b/src/adapters/store_migration/immutable_pool_inventory_digest.rs @@ -0,0 +1,20 @@ +//! This module owns immutable-pool inventory identity. + +/// Digest coordinate naming one canonical complete immutable-pool inventory. +/// +/// Intent admission does not prove that a current inventory has this digest. +#[must_use] +#[derive(Clone, Copy, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)] +pub struct ImmutablePoolInventoryDigest([u8; 32]); + +impl ImmutablePoolInventoryDigest { + /// Returns the exact digest bytes. + #[must_use] + pub const fn as_bytes(&self) -> &[u8; 32] { + &self.0 + } + + pub(super) const fn from_admitted(bytes: [u8; 32]) -> Self { + Self(bytes) + } +} diff --git a/src/adapters/store_migration/migration_intent_bytes.rs b/src/adapters/store_migration/migration_intent_bytes.rs new file mode 100644 index 0000000..4f6906b --- /dev/null +++ b/src/adapters/store_migration/migration_intent_bytes.rs @@ -0,0 +1,54 @@ +//! This boundary module owns fixed-width migration-intent field access. + +use super::StoreMigrationIntentDecodeError; + +pub(super) const ENCODED_LENGTH: usize = 256; + +pub(super) const fn require_length(encoded: &[u8]) -> Result<(), StoreMigrationIntentDecodeError> { + if encoded.len() == ENCODED_LENGTH { + Ok(()) + } else { + Err(wrong_length(encoded)) + } +} + +pub(super) const fn wrong_length(encoded: &[u8]) -> StoreMigrationIntentDecodeError { + StoreMigrationIntentDecodeError::WrongLength { + expected: ENCODED_LENGTH, + observed: encoded.len(), + } +} + +pub(super) fn read_u16( + encoded: &[u8], + offset: usize, +) -> Result { + read_array(encoded, offset).map(u16::from_be_bytes) +} + +pub(super) fn read_u32( + encoded: &[u8], + offset: usize, +) -> Result { + read_array(encoded, offset).map(u32::from_be_bytes) +} + +pub(super) fn read_u64( + encoded: &[u8], + offset: usize, +) -> Result { + read_array(encoded, offset).map(u64::from_be_bytes) +} + +pub(super) fn read_array( + encoded: &[u8], + offset: usize, +) -> Result<[u8; WIDTH], StoreMigrationIntentDecodeError> { + let Some(end) = offset.checked_add(WIDTH) else { + return Err(wrong_length(encoded)); + }; + let bytes = encoded + .get(offset..end) + .ok_or_else(|| wrong_length(encoded))?; + <[u8; WIDTH]>::try_from(bytes).map_err(|_| wrong_length(encoded)) +} diff --git a/src/adapters/store_migration/migration_intent_decode_error.rs b/src/adapters/store_migration/migration_intent_decode_error.rs new file mode 100644 index 0000000..5d6c1e3 --- /dev/null +++ b/src/adapters/store_migration/migration_intent_decode_error.rs @@ -0,0 +1,84 @@ +//! This boundary module owns store-migration intent decoding failures. + +use crate::{CatalogGenerationError, CatalogLengthError}; + +/// Failure to decode and admit one version-2 migration intent. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum StoreMigrationIntentDecodeError { + /// The input was not exactly one complete intent. + WrongLength { + /// Required fixed width. + expected: usize, + /// Observed input width. + observed: usize, + }, + /// The fixed record magic was not canonical. + InvalidMagic { + /// Observed 16 magic bytes. + observed: [u8; 16], + }, + /// The format version is unsupported. + UnsupportedVersion { + /// Supported version. + expected: u16, + /// Observed version. + observed: u16, + }, + /// The record-length field was noncanonical. + InvalidRecordLength { + /// Required record length. + expected: u16, + /// Observed record length. + observed: u16, + }, + /// The intent carried unsupported flags. + UnsupportedFlags { + /// Observed flag bits. + observed: u32, + }, + /// The checksum did not match the exact prefix. + ChecksumMismatch { + /// Computed canonical checksum. + expected: [u8; 32], + /// Checksum stored in the record. + observed: [u8; 32], + }, + /// The catalog generation was not positive. + InvalidCatalogGeneration { + /// Observed generation. + observed: u64, + /// Precise generation refusal. + source: CatalogGenerationError, + }, + /// The catalog length was outside the canonical version-1 grammar. + InvalidCatalogLength { + /// Observed length. + observed: u64, + /// Precise catalog-length refusal. + source: CatalogLengthError, + }, + /// Generation 1 carried a forbidden predecessor. + NonZeroInitialPredecessor { + /// Observed nonzero predecessor digest. + observed: [u8; 32], + }, + /// A later generation omitted its required predecessor. + MissingSuccessorPredecessor { + /// Observed later generation. + generation: u64, + }, + /// The target definition was not the registered version-2 definition. + DefinitionDigestMismatch { + /// Registered version-2 definition digest. + expected: [u8; 32], + /// Observed target definition digest. + observed: [u8; 32], + }, + /// The stored identifier did not match the deterministic derivation. + StoreIdentifierMismatch { + /// Identifier derived from the admitted semantic fields. + expected: [u8; 32], + /// Identifier stored in the record. + observed: [u8; 32], + }, +} diff --git a/src/adapters/store_migration/migration_intent_decode_error_display.rs b/src/adapters/store_migration/migration_intent_decode_error_display.rs new file mode 100644 index 0000000..daeebd2 --- /dev/null +++ b/src/adapters/store_migration/migration_intent_decode_error_display.rs @@ -0,0 +1,64 @@ +//! This boundary module owns migration-intent error formatting and sources. + +use std::error::Error; +use std::fmt; + +use super::StoreMigrationIntentDecodeError; + +impl fmt::Display for StoreMigrationIntentDecodeError { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::WrongLength { expected, observed } => write!( + formatter, + "migration intent requires {expected} bytes, observed {observed}" + ), + Self::InvalidMagic { .. } => formatter.write_str("invalid migration-intent magic"), + Self::UnsupportedVersion { expected, observed } => write!( + formatter, + "unsupported migration-intent version {observed}; expected {expected}" + ), + Self::InvalidRecordLength { expected, observed } => write!( + formatter, + "migration-intent record length {observed}; expected {expected}" + ), + Self::UnsupportedFlags { observed } => { + write!( + formatter, + "unsupported migration-intent flags {observed:#010x}" + ) + } + Self::ChecksumMismatch { .. } => { + formatter.write_str("migration-intent checksum mismatch") + } + Self::InvalidCatalogGeneration { observed, .. } => { + write!(formatter, "invalid migration catalog generation {observed}") + } + Self::InvalidCatalogLength { observed, .. } => { + write!(formatter, "invalid migration catalog length {observed}") + } + Self::NonZeroInitialPredecessor { .. } => { + formatter.write_str("initial migration catalog forbids a predecessor") + } + Self::MissingSuccessorPredecessor { generation } => write!( + formatter, + "migration catalog generation {generation} requires a predecessor" + ), + Self::DefinitionDigestMismatch { .. } => { + formatter.write_str("migration target definition digest mismatch") + } + Self::StoreIdentifierMismatch { .. } => { + formatter.write_str("migration store identifier mismatch") + } + } + } +} + +impl Error for StoreMigrationIntentDecodeError { + fn source(&self) -> Option<&(dyn Error + 'static)> { + match self { + Self::InvalidCatalogGeneration { source, .. } => Some(source), + Self::InvalidCatalogLength { source, .. } => Some(source), + _ => None, + } + } +} diff --git a/src/adapters/store_migration/migration_intent_decoder.rs b/src/adapters/store_migration/migration_intent_decoder.rs new file mode 100644 index 0000000..0108c7a --- /dev/null +++ b/src/adapters/store_migration/migration_intent_decoder.rs @@ -0,0 +1,181 @@ +//! This boundary module owns store-migration intent decoding order. + +use super::admitted_migration_intent::StoreMigrationIntentFields; +use super::migration_intent_bytes::{ + read_array, read_u16, read_u32, read_u64, require_length, wrong_length, +}; +use super::{ + AdmittedStoreMigrationIntent, ImmutablePoolInventoryDigest, StoreFormatDefinitionDigest, + StoreIdentifier, StoreMigrationIntentDecodeError, StoreMigrationIntentDigest, + StoreRootDeviceIdentity, StoreRootFileIdentity, StoreRootMountIdentity, +}; +use crate::{CatalogDigest, CatalogGeneration, CatalogLength}; + +const CHECKSUM_OFFSET: usize = 224; +const MAGIC: [u8; 16] = *b"KEEP:MIG:INT2\0\0\0"; +const VERSION: u16 = 2; +const RECORD_LENGTH: u16 = 256; +const CHECKSUM_DOMAIN: &[u8] = b"keep.store-migration-intent-checksum/v2\0"; +const DIGEST_DOMAIN: &[u8] = b"keep.store-migration-intent/v2\0"; +const STORE_IDENTIFIER_DOMAIN: &[u8] = b"keep.store-identifier/v2\0"; +const ZERO_DIGEST: [u8; 32] = [0; 32]; + +pub(super) fn decode( + encoded: &[u8], +) -> Result, StoreMigrationIntentDecodeError> { + require_length(encoded)?; + validate_fixed_fields(encoded)?; + verify_checksum(encoded)?; + let catalog_generation = read_catalog_generation(encoded)?; + let catalog_length = read_catalog_length(encoded)?; + let catalog_digest = CatalogDigest::from_validated(read_array(encoded, 40)?); + let predecessor_catalog_digest = + read_predecessor(catalog_generation, read_array(encoded, 72)?)?; + let fields = StoreMigrationIntentFields { + catalog_generation, + catalog_length, + catalog_digest, + predecessor_catalog_digest, + inventory_digest: ImmutablePoolInventoryDigest::from_admitted(read_array(encoded, 104)?), + root_device_identity: StoreRootDeviceIdentity::from_admitted(read_u64(encoded, 136)?), + root_mount_identity: StoreRootMountIdentity::from_admitted(read_u64(encoded, 144)?), + root_file_identity: StoreRootFileIdentity::from_admitted(read_u64(encoded, 152)?), + target_definition_digest: read_definition_digest(encoded)?, + store_identifier: StoreIdentifier::from_hash(read_array(encoded, 192)?), + }; + verify_store_identifier(&fields)?; + Ok(AdmittedStoreMigrationIntent::admitted( + encoded, + fields, + digest(encoded), + )) +} + +fn validate_fixed_fields(encoded: &[u8]) -> Result<(), StoreMigrationIntentDecodeError> { + let magic = read_array(encoded, 0)?; + if magic != MAGIC { + return Err(StoreMigrationIntentDecodeError::InvalidMagic { observed: magic }); + } + let version = read_u16(encoded, 16)?; + if version != VERSION { + return Err(StoreMigrationIntentDecodeError::UnsupportedVersion { + expected: VERSION, + observed: version, + }); + } + let record_length = read_u16(encoded, 18)?; + if record_length != RECORD_LENGTH { + return Err(StoreMigrationIntentDecodeError::InvalidRecordLength { + expected: RECORD_LENGTH, + observed: record_length, + }); + } + let flags = read_u32(encoded, 20)?; + if flags != 0 { + return Err(StoreMigrationIntentDecodeError::UnsupportedFlags { observed: flags }); + } + Ok(()) +} + +fn verify_checksum(encoded: &[u8]) -> Result<(), StoreMigrationIntentDecodeError> { + let preimage = encoded + .get(..CHECKSUM_OFFSET) + .ok_or_else(|| wrong_length(encoded))?; + let observed = read_array(encoded, CHECKSUM_OFFSET)?; + let expected = hash(CHECKSUM_DOMAIN, &[preimage]); + if observed == expected { + Ok(()) + } else { + Err(StoreMigrationIntentDecodeError::ChecksumMismatch { expected, observed }) + } +} + +fn read_catalog_generation( + encoded: &[u8], +) -> Result { + let observed = read_u64(encoded, 24)?; + CatalogGeneration::new(observed).map_err(|source| { + StoreMigrationIntentDecodeError::InvalidCatalogGeneration { observed, source } + }) +} + +fn read_catalog_length(encoded: &[u8]) -> Result { + let observed = read_u64(encoded, 32)?; + CatalogLength::new(observed).map_err(|source| { + StoreMigrationIntentDecodeError::InvalidCatalogLength { observed, source } + }) +} + +fn read_predecessor( + generation: CatalogGeneration, + observed: [u8; 32], +) -> Result, StoreMigrationIntentDecodeError> { + if generation.get() == 1 { + return if observed == ZERO_DIGEST { + Ok(None) + } else { + Err(StoreMigrationIntentDecodeError::NonZeroInitialPredecessor { observed }) + }; + } + if observed == ZERO_DIGEST { + return Err( + StoreMigrationIntentDecodeError::MissingSuccessorPredecessor { + generation: generation.get(), + }, + ); + } + Ok(Some(CatalogDigest::from_validated(observed))) +} + +fn read_definition_digest( + encoded: &[u8], +) -> Result { + let observed = read_array(encoded, 160)?; + if observed == *StoreFormatDefinitionDigest::VERSION_TWO.as_bytes() { + Ok(StoreFormatDefinitionDigest::VERSION_TWO) + } else { + Err(StoreMigrationIntentDecodeError::DefinitionDigestMismatch { + expected: *StoreFormatDefinitionDigest::VERSION_TWO.as_bytes(), + observed, + }) + } +} + +fn verify_store_identifier( + fields: &StoreMigrationIntentFields, +) -> Result<(), StoreMigrationIntentDecodeError> { + let predecessor = fields + .predecessor_catalog_digest + .as_ref() + .map_or(&ZERO_DIGEST, CatalogDigest::as_bytes); + let expected = hash( + STORE_IDENTIFIER_DOMAIN, + &[ + &fields.catalog_generation.get().to_be_bytes(), + &fields.catalog_length.get().to_be_bytes(), + fields.catalog_digest.as_bytes(), + predecessor, + fields.inventory_digest.as_bytes(), + fields.target_definition_digest.as_bytes(), + ], + ); + let observed = *fields.store_identifier.as_bytes(); + if observed == expected { + Ok(()) + } else { + Err(StoreMigrationIntentDecodeError::StoreIdentifierMismatch { expected, observed }) + } +} + +fn digest(encoded: &[u8]) -> StoreMigrationIntentDigest { + StoreMigrationIntentDigest::from_hash(hash(DIGEST_DOMAIN, &[encoded])) +} + +fn hash(domain: &[u8], fields: &[&[u8]]) -> [u8; 32] { + let mut hasher = blake3::Hasher::new(); + hasher.update(domain); + for field in fields { + hasher.update(field); + } + *hasher.finalize().as_bytes() +} diff --git a/src/adapters/store_migration/migration_intent_digest.rs b/src/adapters/store_migration/migration_intent_digest.rs new file mode 100644 index 0000000..8cce8fc --- /dev/null +++ b/src/adapters/store_migration/migration_intent_digest.rs @@ -0,0 +1,18 @@ +//! This module owns canonical migration-intent identity. + +/// Domain-separated digest of one complete canonical migration intent. +#[must_use] +#[derive(Clone, Copy, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)] +pub struct StoreMigrationIntentDigest([u8; 32]); + +impl StoreMigrationIntentDigest { + /// Returns the exact digest bytes. + #[must_use] + pub const fn as_bytes(&self) -> &[u8; 32] { + &self.0 + } + + pub(super) const fn from_hash(hash: [u8; 32]) -> Self { + Self(hash) + } +} diff --git a/src/adapters/store_migration/store_identifier.rs b/src/adapters/store_migration/store_identifier.rs new file mode 100644 index 0000000..27f6197 --- /dev/null +++ b/src/adapters/store_migration/store_identifier.rs @@ -0,0 +1,18 @@ +//! This module owns deterministic logical store identity. + +/// Logical identity derived from admitted version-1 state and the v2 format. +#[must_use] +#[derive(Clone, Copy, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)] +pub struct StoreIdentifier([u8; 32]); + +impl StoreIdentifier { + /// Returns the exact identifier bytes. + #[must_use] + pub const fn as_bytes(&self) -> &[u8; 32] { + &self.0 + } + + pub(super) const fn from_hash(hash: [u8; 32]) -> Self { + Self(hash) + } +} diff --git a/src/adapters/store_migration/store_root_identity.rs b/src/adapters/store_migration/store_root_identity.rs new file mode 100644 index 0000000..39bc4bf --- /dev/null +++ b/src/adapters/store_migration/store_root_identity.rs @@ -0,0 +1,38 @@ +//! This module owns physical store-root recovery coordinates. + +macro_rules! root_identity { + ($name:ident, $documentation:literal) => { + #[doc = $documentation] + #[must_use] + #[derive(Clone, Copy, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)] + pub struct $name(u64); + + impl $name { + /// Returns the exact serialized platform coordinate. + /// + /// This value remains a comparison coordinate until a platform + /// adapter revalidates it against the opened store root. + #[must_use] + pub const fn get(self) -> u64 { + self.0 + } + + pub(super) const fn from_admitted(value: u64) -> Self { + Self(value) + } + } + }; +} + +root_identity!( + StoreRootDeviceIdentity, + "Platform device identity bound into a migration intent." +); +root_identity!( + StoreRootMountIdentity, + "Platform mount identity bound into a migration intent." +); +root_identity!( + StoreRootFileIdentity, + "Platform file identity bound into a migration intent." +); diff --git a/src/lib.rs b/src/lib.rs index 118d258..c09acfc 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -31,7 +31,10 @@ //! Ordered publication revalidates authority, executes all durability phases, //! and returns a complete receipt. The exact version-2 store-format marker has //! canonical encoding, registered-definition admission, checksum verification, -//! and domain-separated identity. Filesystem migration, retention execution, +//! and domain-separated identity. Migration-intent admission validates its +//! framing, checksum, catalog and predecessor grammar, registered definition, +//! deterministic store identity, and typed recovery coordinates. Live +//! inventory and root revalidation, filesystem migration, retention execution, //! recovery, and garbage collection remain intentionally absent. #[cfg(test)] @@ -51,24 +54,25 @@ mod retention; pub use adapters::RepositoryInitializationStorage; pub use adapters::{ AdmittedCatalog, AdmittedRecoveryStageBytes, AdmittedSegment, AdmittedSegmentRecord, - AdmittedStoreFormatMarker, BlobIdBinaryParseError, BlobIdTextParseError, CanonicalCatalog, - CanonicalLayoutRecord, CanonicalPublicationHead, CanonicalStoreFormatMarker, - CatalogAdmissionError, CatalogAllocationPhase, CatalogDecodeError, CatalogEncodeError, - CatalogEntryDecodeError, CatalogPublicationError, CatalogPublicationExpectation, - CatalogPublicationOutcome, CatalogPublicationPhase, CatalogPublicationReadiness, - CatalogPublicationReceipt, CatalogPublicationStorage, CatalogRestartArtifact, - CatalogRestartByteLimit, CatalogRestartByteLimitError, CatalogRestartError, - CatalogRestartPhase, CatalogRestartPolicy, CatalogSnapshot, CatalogSnapshotError, - CatalogSuccessor, CatalogTransitionError, ChecksummedCatalog, ChecksummedPublicationHead, - ChecksummedSegmentRecord, ClosedSegment, FilesystemCatalogPublicationError, - FilesystemCatalogPublisher, FilesystemCatalogSnapshot, FilesystemPlatformAdmission, - FilesystemPlatformAdmissionError, FilesystemRecoveryInventoryReader, - FilesystemRecoveryNextHeadFinalizationOpenError, FilesystemRecoveryNextHeadFinalizer, - FilesystemRecoverySegmentResumeOpenError, FilesystemRecoverySegmentResumer, - FilesystemRecoverySegmentStage, FilesystemRecoveryStageCompleter, - FilesystemRecoveryStageCompletionOpenError, FilesystemRecoveryStageDiscardOpenError, - FilesystemRecoveryStageDiscarder, FilesystemRecoveryStageError, FilesystemSegmentStage, - FilesystemWriterLock, LayoutDecodeError, LayoutDecodePolicy, LayoutEncodeError, + AdmittedStoreFormatMarker, AdmittedStoreMigrationIntent, BlobIdBinaryParseError, + BlobIdTextParseError, CanonicalCatalog, CanonicalLayoutRecord, CanonicalPublicationHead, + CanonicalStoreFormatMarker, CatalogAdmissionError, CatalogAllocationPhase, CatalogDecodeError, + CatalogEncodeError, CatalogEntryDecodeError, CatalogPublicationError, + CatalogPublicationExpectation, CatalogPublicationOutcome, CatalogPublicationPhase, + CatalogPublicationReadiness, CatalogPublicationReceipt, CatalogPublicationStorage, + CatalogRestartArtifact, CatalogRestartByteLimit, CatalogRestartByteLimitError, + CatalogRestartError, CatalogRestartPhase, CatalogRestartPolicy, CatalogSnapshot, + CatalogSnapshotError, CatalogSuccessor, CatalogTransitionError, ChecksummedCatalog, + ChecksummedPublicationHead, ChecksummedSegmentRecord, ClosedSegment, + FilesystemCatalogPublicationError, FilesystemCatalogPublisher, FilesystemCatalogSnapshot, + FilesystemPlatformAdmission, FilesystemPlatformAdmissionError, + FilesystemRecoveryInventoryReader, FilesystemRecoveryNextHeadFinalizationOpenError, + FilesystemRecoveryNextHeadFinalizer, FilesystemRecoverySegmentResumeOpenError, + FilesystemRecoverySegmentResumer, FilesystemRecoverySegmentStage, + FilesystemRecoveryStageCompleter, FilesystemRecoveryStageCompletionOpenError, + FilesystemRecoveryStageDiscardOpenError, FilesystemRecoveryStageDiscarder, + FilesystemRecoveryStageError, FilesystemSegmentStage, FilesystemWriterLock, + ImmutablePoolInventoryDigest, LayoutDecodeError, LayoutDecodePolicy, LayoutEncodeError, LayoutIdBinaryParseError, LayoutIdTextParseError, OpenedReusableSegment, PublicationHeadDecodeError, RecoveryCatalogStage, RecoveryCatalogStageError, RecoveryEntryName, RecoveryEntryNameError, RecoveryEntryRole, RecoveryInventory, RecoveryInventoryEntry, @@ -101,8 +105,10 @@ pub use adapters::{ SegmentRecords, SegmentSeal, SegmentSealError, SegmentStage, SegmentStageCreateError, SegmentWriteError, SegmentWritePhase, StagedSegment, StorageProfileIdParseError, StoreFormatDefinitionDigest, StoreFormatMarkerDecodeError, StoreFormatMarkerDigest, - StoreInitializationError, StoreInitializationPhase, StoreInitializationReceipt, - StoreInitializationStorage, WriterLockAcquireError, WriterLockAcquirePhase, + StoreIdentifier, StoreInitializationError, StoreInitializationPhase, + StoreInitializationReceipt, StoreInitializationStorage, StoreMigrationIntentDecodeError, + StoreMigrationIntentDigest, StoreRootDeviceIdentity, StoreRootFileIdentity, + StoreRootMountIdentity, WriterLockAcquireError, WriterLockAcquirePhase, admit_recovery_stage_bytes, assess_recovery_stage, classify_recovery_catalog_stage, classify_recovery_names, classify_recovery_next_head_stage, classify_recovery_segment_stage, execute_recovery_next_head_finalization, execute_recovery_segment_resume, diff --git a/tests/store_migration_intent.rs b/tests/store_migration_intent.rs new file mode 100644 index 0000000..2040286 --- /dev/null +++ b/tests/store_migration_intent.rs @@ -0,0 +1,200 @@ +//! Canonical version-2 store-migration intent laws. + +#[path = "store_migration_intent/fixture.rs"] +mod fixture; +mod support; + +use std::io; + +use fixture::{CATALOG_DIGEST, INTENT_DIGEST, INVENTORY_DIGEST, STORE_IDENTIFIER, fixture_bytes}; +use keep::{ + AdmittedStoreMigrationIntent, CatalogGeneration, CatalogGenerationError, CatalogLength, + CatalogLengthError, StoreFormatDefinitionDigest, StoreMigrationIntentDecodeError, +}; + +#[test] +fn intent_admits_every_frozen_coordinate() -> Result<(), Box> { + let bytes = fixture_bytes()?; + let intent = AdmittedStoreMigrationIntent::decode(&bytes)?; + + assert_eq!(intent.encoded(), bytes); + assert_eq!(intent.catalog_generation(), CatalogGeneration::new(1)?); + assert_eq!(intent.catalog_length(), CatalogLength::new(352)?); + assert_eq!(intent.catalog_digest().as_bytes(), &CATALOG_DIGEST); + assert_eq!(intent.predecessor_catalog_digest(), None); + assert_eq!(intent.inventory_digest().as_bytes(), &INVENTORY_DIGEST); + assert_eq!(intent.root_device_identity().get(), 1); + assert_eq!(intent.root_mount_identity().get(), 2); + assert_eq!(intent.root_file_identity().get(), 3); + assert_eq!( + intent.target_definition_digest(), + StoreFormatDefinitionDigest::VERSION_TWO + ); + assert_eq!(intent.store_identifier().as_bytes(), &STORE_IDENTIFIER); + assert_eq!(intent.digest().as_bytes(), &INTENT_DIGEST); + Ok(()) +} + +#[test] +fn intent_framing_has_exact_first_refusals() -> Result<(), Box> { + let bytes = fixture_bytes()?; + let mut truncated = bytes.clone(); + assert!(truncated.pop().is_some()); + assert_eq!( + AdmittedStoreMigrationIntent::decode(&truncated), + Err(StoreMigrationIntentDecodeError::WrongLength { + expected: 256, + observed: 255, + }) + ); + let mut extended = bytes.clone(); + extended.push(0); + assert_eq!( + AdmittedStoreMigrationIntent::decode(&extended), + Err(StoreMigrationIntentDecodeError::WrongLength { + expected: 256, + observed: 257, + }) + ); + assert_fixed_refusal( + 0, + StoreMigrationIntentDecodeError::InvalidMagic { + observed: mutated_array(&bytes, 0, 0)?, + }, + )?; + assert_fixed_refusal( + 17, + StoreMigrationIntentDecodeError::UnsupportedVersion { + expected: 2, + observed: 3, + }, + )?; + assert_fixed_refusal( + 19, + StoreMigrationIntentDecodeError::InvalidRecordLength { + expected: 256, + observed: 257, + }, + )?; + assert_fixed_refusal( + 23, + StoreMigrationIntentDecodeError::UnsupportedFlags { observed: 1 }, + )?; + Ok(()) +} + +#[test] +fn checksum_and_semantic_laws_have_exact_precedence() -> Result<(), Box> { + let mut bytes = fixture_bytes()?; + flip_byte(&mut bytes, 31)?; + assert!(matches!( + AdmittedStoreMigrationIntent::decode(&bytes), + Err(StoreMigrationIntentDecodeError::ChecksumMismatch { .. }) + )); + refresh_checksum(&mut bytes)?; + assert_eq!( + AdmittedStoreMigrationIntent::decode(&bytes), + Err(StoreMigrationIntentDecodeError::InvalidCatalogGeneration { + observed: 0, + source: CatalogGenerationError::Zero, + }) + ); + + assert_semantic_refusal( + 39, + StoreMigrationIntentDecodeError::InvalidCatalogLength { + observed: 353, + source: CatalogLengthError::NotCongruent { observed: 353 }, + }, + )?; + assert_semantic_refusal( + 103, + StoreMigrationIntentDecodeError::NonZeroInitialPredecessor { + observed: mutated_array(&fixture_bytes()?, 72, 31)?, + }, + )?; + + let mut successor = fixture_bytes()?; + let generation = successor + .get_mut(31) + .ok_or_else(|| io::Error::other("intent lacks generation field"))?; + *generation = 2; + refresh_checksum(&mut successor)?; + assert_eq!( + AdmittedStoreMigrationIntent::decode(&successor), + Err(StoreMigrationIntentDecodeError::MissingSuccessorPredecessor { generation: 2 }) + ); + + assert_semantic_refusal( + 160, + StoreMigrationIntentDecodeError::DefinitionDigestMismatch { + expected: *StoreFormatDefinitionDigest::VERSION_TWO.as_bytes(), + observed: mutated_array(&fixture_bytes()?, 160, 0)?, + }, + )?; + assert_semantic_refusal( + 223, + StoreMigrationIntentDecodeError::StoreIdentifierMismatch { + expected: STORE_IDENTIFIER, + observed: mutated_array(&fixture_bytes()?, 192, 31)?, + }, + )?; + Ok(()) +} + +fn assert_fixed_refusal( + offset: usize, + expected: StoreMigrationIntentDecodeError, +) -> Result<(), Box> { + let mut bytes = fixture_bytes()?; + flip_byte(&mut bytes, offset)?; + assert_eq!(AdmittedStoreMigrationIntent::decode(&bytes), Err(expected)); + Ok(()) +} + +fn assert_semantic_refusal( + offset: usize, + expected: StoreMigrationIntentDecodeError, +) -> Result<(), Box> { + let mut bytes = fixture_bytes()?; + flip_byte(&mut bytes, offset)?; + refresh_checksum(&mut bytes)?; + assert_eq!(AdmittedStoreMigrationIntent::decode(&bytes), Err(expected)); + Ok(()) +} + +fn mutated_array( + bytes: &[u8], + offset: usize, + relative: usize, +) -> Result<[u8; WIDTH], io::Error> { + let end = offset + .checked_add(WIDTH) + .ok_or_else(|| io::Error::other("intent field offset overflow"))?; + let field = bytes + .get(offset..end) + .ok_or_else(|| io::Error::other("intent lacks fixed field"))?; + let mut observed = <[u8; WIDTH]>::try_from(field) + .map_err(|_| io::Error::other("intent field width mismatch"))?; + flip_byte(&mut observed, relative)?; + Ok(observed) +} + +fn flip_byte(bytes: &mut [u8], offset: usize) -> Result<(), io::Error> { + let byte = bytes + .get_mut(offset) + .ok_or_else(|| io::Error::other("intent mutation offset is out of bounds"))?; + *byte ^= 1; + Ok(()) +} + +fn refresh_checksum(bytes: &mut [u8]) -> Result<(), io::Error> { + let (preimage, checksum) = bytes + .split_at_mut_checked(224) + .ok_or_else(|| io::Error::other("intent lacks checksum boundary"))?; + let mut hasher = blake3::Hasher::new(); + hasher.update(b"keep.store-migration-intent-checksum/v2\0"); + hasher.update(preimage); + checksum.copy_from_slice(hasher.finalize().as_bytes()); + Ok(()) +} diff --git a/tests/store_migration_intent/fixture.rs b/tests/store_migration_intent/fixture.rs new file mode 100644 index 0000000..83da59d --- /dev/null +++ b/tests/store_migration_intent/fixture.rs @@ -0,0 +1,33 @@ +//! This module owns frozen migration-intent fixture values. +#![allow( + clippy::redundant_pub_crate, + reason = "the parent integration-test module consumes this private fixture" +)] + +use std::io; + +use super::support; + +const MIGRATION_INTENT: &str = + include_str!("../../conformance/segment-store/v2/migration-intent.hex"); + +pub(super) const CATALOG_DIGEST: [u8; 32] = [ + 0x04, 0xb8, 0x25, 0x19, 0xb0, 0x39, 0x9b, 0xae, 0xfd, 0x0b, 0x9c, 0x0f, 0x32, 0xa8, 0x71, 0x05, + 0x2e, 0x4c, 0x47, 0xe3, 0xa0, 0x02, 0x26, 0xab, 0x03, 0xb2, 0x16, 0x61, 0x47, 0x0f, 0x73, 0x20, +]; +pub(super) const INVENTORY_DIGEST: [u8; 32] = [ + 0x40, 0xbf, 0x5d, 0x49, 0xc3, 0x48, 0x47, 0xac, 0x9c, 0xf4, 0x6a, 0x25, 0x6f, 0x34, 0x3c, 0xee, + 0x80, 0xcd, 0x98, 0x0d, 0x14, 0x05, 0xd2, 0xdd, 0x02, 0xce, 0xff, 0x8f, 0x58, 0xd6, 0x74, 0xf9, +]; +pub(super) const STORE_IDENTIFIER: [u8; 32] = [ + 0x0c, 0xd9, 0xd3, 0xdf, 0xbe, 0xc9, 0xb3, 0x49, 0xfe, 0x42, 0xd2, 0x14, 0x75, 0x27, 0x1b, 0x0e, + 0x8d, 0xe2, 0x3c, 0x04, 0x34, 0x40, 0xd6, 0x42, 0x7a, 0x1c, 0x37, 0x89, 0x8a, 0xd1, 0xdd, 0x79, +]; +pub(super) const INTENT_DIGEST: [u8; 32] = [ + 0xa1, 0x5a, 0x00, 0x00, 0x02, 0x19, 0xdf, 0x20, 0x97, 0x9d, 0xa3, 0x64, 0x19, 0x04, 0x6e, 0xae, + 0x9a, 0x0b, 0xa9, 0x98, 0x64, 0x5f, 0xbf, 0xe3, 0x08, 0xea, 0x43, 0x35, 0xa8, 0x32, 0x6b, 0x44, +]; + +pub(super) fn fixture_bytes() -> Result, io::Error> { + support::decode_hex(MIGRATION_INTENT.trim_end()) +} From 94092ac0ad8742dfeb15fc36c70f848225cf69fd Mon Sep 17 00:00:00 2001 From: James Ross Date: Thu, 30 Jul 2026 02:26:18 -0700 Subject: [PATCH 032/111] Add: Admit store migration receipts --- CHANGELOG.md | 12 +- README.md | 16 +- docs/formats/segment-store-v2/recovery.md | 6 +- docs/formats/segment-store-v2/requirements.md | 2 +- src/adapters/store_migration.rs | 17 +- .../admitted_migration_receipt.rs | 97 +++++++++++ .../empty_disposition_set_digest.rs | 18 ++ .../initial_gc_state_digest.rs | 18 ++ .../initial_retention_state_digest.rs | 18 ++ .../store_migration/migration_intent_bytes.rs | 54 ------ .../migration_intent_decoder.rs | 2 +- .../migration_receipt_decode_error.rs | 100 +++++++++++ .../migration_receipt_decode_error_display.rs | 63 +++++++ .../migration_receipt_decoder.rs | 157 ++++++++++++++++++ .../migration_receipt_initial_state.rs | 61 +++++++ .../store_migration/migration_record_bytes.rs | 69 ++++++++ .../migration_synchronization_mask.rs | 20 +++ src/lib.rs | 41 ++--- tests/store_migration_receipt.rs | 96 +++++++++++ tests/store_migration_receipt/binding_laws.rs | 104 ++++++++++++ tests/store_migration_receipt/fixture.rs | 38 +++++ tests/store_migration_receipt/harness.rs | 107 ++++++++++++ 22 files changed, 1023 insertions(+), 93 deletions(-) create mode 100644 src/adapters/store_migration/admitted_migration_receipt.rs create mode 100644 src/adapters/store_migration/empty_disposition_set_digest.rs create mode 100644 src/adapters/store_migration/initial_gc_state_digest.rs create mode 100644 src/adapters/store_migration/initial_retention_state_digest.rs delete mode 100644 src/adapters/store_migration/migration_intent_bytes.rs create mode 100644 src/adapters/store_migration/migration_receipt_decode_error.rs create mode 100644 src/adapters/store_migration/migration_receipt_decode_error_display.rs create mode 100644 src/adapters/store_migration/migration_receipt_decoder.rs create mode 100644 src/adapters/store_migration/migration_receipt_initial_state.rs create mode 100644 src/adapters/store_migration/migration_record_bytes.rs create mode 100644 src/adapters/store_migration/migration_synchronization_mask.rs create mode 100644 tests/store_migration_receipt.rs create mode 100644 tests/store_migration_receipt/binding_laws.rs create mode 100644 tests/store_migration_receipt/fixture.rs create mode 100644 tests/store_migration_receipt/harness.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index beb0ef4..a105add 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,12 +10,12 @@ after its public API and format compatibility policies are established. ### Changed -- The version-2 store-format marker now has exact canonical encoding and - admission, while migration intents admit exact catalog, predecessor, root, - definition, store-identity, checksum, and digest coordinates. Retention - preflight combines expected-generation planning with deterministic closure - verification; authority-revalidated 17-phase orchestration returns an - unforgeable complete-coordinate receipt after durable cleanup. +- Version-2 marker, migration-intent, and completion-receipt admission now bind + exact catalog, predecessor, root, definition, store, empty-state, checksum, + digest, and synchronization-mask coordinates. Retention preflight combines + expected-generation planning with deterministic closure verification; + authority-revalidated 17-phase orchestration returns an unforgeable + complete-coordinate receipt after durable cleanup. - Repository crash-matrix execution now terminates isolated writer process groups at all 105 canonical before/during/after coordinates, retains open writer and stage authority until termination, executes production diff --git a/README.md b/README.md index 7406736..499bc51 100644 --- a/README.md +++ b/README.md @@ -116,14 +116,14 @@ recovery classifiers and immutable-artifact admission, and reconstructs the exact published generation and visible one-zero chunk when `HEAD` exists. This matrix proves application process-death behavior; it does not simulate host power loss. Canonical version-2 store-format marker encoding and exact -migration-intent admission are implemented. Version-2 retention values; -canonical in-memory root, global manifest, and retention-head codecs; -storage-independent expected-state transition planning; deterministic bounded -closure verification against a pinned catalog; a combined transition preflight -proof; and the exact 17-phase publication vocabulary with a blocking storage -capability port are implemented. Private-field proofs retain every receipt -coordinate. Ordered storage-port orchestration revalidates current authority, -executes all 17 durability phases, and returns a consequential +migration-intent and completion-receipt admission are implemented. Version-2 +retention values; canonical in-memory root, global manifest, and retention-head +codecs; storage-independent expected-state transition planning; deterministic +bounded closure verification against a pinned catalog; a combined transition +preflight proof; and the exact 17-phase publication vocabulary with a blocking +storage capability port are implemented. Private-field proofs retain every +receipt coordinate. Ordered storage-port orchestration revalidates current +authority, executes all 17 durability phases, and returns a consequential complete-coordinate receipt. Filesystem migration, retention execution, recovery, compaction, and garbage collection remain planned. Presence in the reference CAS does not claim durable retention or crash diff --git a/docs/formats/segment-store-v2/recovery.md b/docs/formats/segment-store-v2/recovery.md index 4f7ad6b..f631c23 100644 --- a/docs/formats/segment-store-v2/recovery.md +++ b/docs/formats/segment-store-v2/recovery.md @@ -61,9 +61,9 @@ corpus `definition.tsv` bytes. The format-marker digest is BLAKE3-256 of `keep.store-format-marker/v2\0` followed by all 96 marker bytes. `CanonicalStoreFormatMarker` produces the registered marker; `AdmittedStoreFormatMarker` admits its framing, checksum, definition, and namespace bound. -`AdmittedStoreMigrationIntent` admits the exact intent framing, checksum, catalog coordinates, predecessor law, definition, and derived store identity. -These record boundaries do not prove the named live inventory or physical root, -detect the store version, or execute filesystem migration. +`AdmittedStoreMigrationIntent` admits intent integrity and identity; `AdmittedStoreMigrationReceipt` binds that intent, the marker, registered empty states, and all synchronization bits. +These record boundaries do not prove the named live inventory, physical root, +store version, or execution of filesystem migration. ## Reader fence diff --git a/docs/formats/segment-store-v2/requirements.md b/docs/formats/segment-store-v2/requirements.md index afb0409..9d85a84 100644 --- a/docs/formats/segment-store-v2/requirements.md +++ b/docs/formats/segment-store-v2/requirements.md @@ -29,7 +29,7 @@ case is not evidence. | ID | Requirement | Evidence | Status | | --- | --- | --- | --- | | `KEEP-MIGRATION-001` | Exact version-1 stores remain admitted until a durable migration artifact exists | compatibility fixtures | Planned in #19 | -| `KEEP-MIGRATION-002` | Format marker, intent, and receipt have complete fixed byte tables, named domains, bounds, checksums, deterministic store identity, and exact initial-state digests | exact marker and intent admission in `tests/store_format_marker.rs` and `tests/store_migration_intent.rs`; receipt admission remains | In progress in #19 | +| `KEEP-MIGRATION-002` | Format marker, intent, and receipt have complete fixed byte tables, named domains, bounds, checksums, deterministic store identity, and exact initial-state digests | exact admission in `tests/store_format_marker.rs`, `tests/store_migration_intent.rs`, and `tests/store_migration_receipt.rs` | Implemented | | `KEEP-MIGRATION-003` | Migration revalidates version-1 head, catalog, pools, root identity, and writer authority before mutation | capability-relative integration tests | Planned in #19 | | `KEEP-MIGRATION-004` | Every partial migration prefix continues idempotently under writer authority | state-machine and recovery tests | Planned in #19 | | `KEEP-MIGRATION-005` | Unknown, out-of-order, substituted, corrupt, conflicting, or changed evidence is unrecoverable ambiguity | corruption and mutation matrix | Planned in #19 | diff --git a/src/adapters/store_migration.rs b/src/adapters/store_migration.rs index 19effb7..2021c2a 100644 --- a/src/adapters/store_migration.rs +++ b/src/adapters/store_migration.rs @@ -2,7 +2,9 @@ mod admitted_format_marker; mod admitted_migration_intent; +mod admitted_migration_receipt; mod canonical_format_marker; +mod empty_disposition_set_digest; mod format_definition_digest; mod format_marker_decode_error; mod format_marker_decode_error_display; @@ -10,23 +12,36 @@ mod format_marker_decoder; mod format_marker_digest; mod format_marker_encoder; mod immutable_pool_inventory_digest; -mod migration_intent_bytes; +mod initial_gc_state_digest; +mod initial_retention_state_digest; mod migration_intent_decode_error; mod migration_intent_decode_error_display; mod migration_intent_decoder; mod migration_intent_digest; +mod migration_receipt_decode_error; +mod migration_receipt_decode_error_display; +mod migration_receipt_decoder; +mod migration_receipt_initial_state; +mod migration_record_bytes; +mod migration_synchronization_mask; mod store_identifier; mod store_root_identity; pub use admitted_format_marker::AdmittedStoreFormatMarker; pub use admitted_migration_intent::AdmittedStoreMigrationIntent; +pub use admitted_migration_receipt::AdmittedStoreMigrationReceipt; pub use canonical_format_marker::CanonicalStoreFormatMarker; +pub use empty_disposition_set_digest::EmptyDispositionSetDigest; pub use format_definition_digest::StoreFormatDefinitionDigest; pub use format_marker_decode_error::StoreFormatMarkerDecodeError; pub use format_marker_digest::StoreFormatMarkerDigest; pub use immutable_pool_inventory_digest::ImmutablePoolInventoryDigest; +pub use initial_gc_state_digest::InitialGcStateDigest; +pub use initial_retention_state_digest::InitialRetentionStateDigest; pub use migration_intent_decode_error::StoreMigrationIntentDecodeError; pub use migration_intent_digest::StoreMigrationIntentDigest; +pub use migration_receipt_decode_error::StoreMigrationReceiptDecodeError; +pub use migration_synchronization_mask::MigrationSynchronizationMask; pub use store_identifier::StoreIdentifier; pub use store_root_identity::{ StoreRootDeviceIdentity, StoreRootFileIdentity, StoreRootMountIdentity, diff --git a/src/adapters/store_migration/admitted_migration_receipt.rs b/src/adapters/store_migration/admitted_migration_receipt.rs new file mode 100644 index 0000000..6abf88c --- /dev/null +++ b/src/adapters/store_migration/admitted_migration_receipt.rs @@ -0,0 +1,97 @@ +//! This boundary module owns admitted store-migration completion evidence. + +use super::{ + AdmittedStoreFormatMarker, AdmittedStoreMigrationIntent, EmptyDispositionSetDigest, + InitialGcStateDigest, InitialRetentionStateDigest, MigrationSynchronizationMask, + StoreFormatMarkerDigest, StoreIdentifier, StoreMigrationIntentDigest, + StoreMigrationReceiptDecodeError, migration_receipt_decoder, +}; + +/// Semantic fields admitted from one canonical migration receipt. +#[derive(Clone, Debug, Eq, PartialEq)] +pub(super) struct StoreMigrationReceiptFields { + pub(super) intent_digest: StoreMigrationIntentDigest, + pub(super) store_identifier: StoreIdentifier, + pub(super) format_marker_digest: StoreFormatMarkerDigest, + pub(super) initial_retention_state_digest: InitialRetentionStateDigest, + pub(super) initial_gc_state_digest: InitialGcStateDigest, + pub(super) empty_disposition_set_digest: EmptyDispositionSetDigest, + pub(super) synchronization_mask: MigrationSynchronizationMask, +} + +/// Borrowed canonical version-2 store-migration receipt. +/// +/// Admission proves record integrity, exact binding to caller-supplied admitted +/// intent and marker evidence, registered empty-state digests, and the complete +/// synchronization mask. It does not prove that filesystem transitions +/// actually occurred; production recovery must establish that separately. +#[must_use] +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct AdmittedStoreMigrationReceipt<'encoded> { + encoded: &'encoded [u8], + fields: StoreMigrationReceiptFields, +} + +impl<'encoded> AdmittedStoreMigrationReceipt<'encoded> { + /// Decodes and admits one exact receipt bound to `intent` and `marker`. + /// + /// # Errors + /// + /// Returns [`StoreMigrationReceiptDecodeError`] for invalid framing, + /// integrity, binding, registered state digest, or synchronization bits. + pub fn decode( + encoded: &'encoded [u8], + intent: &AdmittedStoreMigrationIntent<'_>, + marker: &AdmittedStoreFormatMarker<'_>, + ) -> Result { + migration_receipt_decoder::decode(encoded, intent, marker) + } + + /// Returns the exact borrowed canonical bytes. + #[must_use] + pub const fn encoded(&self) -> &'encoded [u8] { + self.encoded + } + + /// Returns the exact bound migration-intent digest. + pub const fn intent_digest(&self) -> StoreMigrationIntentDigest { + self.fields.intent_digest + } + + /// Returns the exact bound logical store identity. + pub const fn store_identifier(&self) -> StoreIdentifier { + self.fields.store_identifier + } + + /// Returns the exact bound format-marker digest. + pub const fn format_marker_digest(&self) -> StoreFormatMarkerDigest { + self.fields.format_marker_digest + } + + /// Returns the registered empty retention-state digest. + pub const fn initial_retention_state_digest(&self) -> InitialRetentionStateDigest { + self.fields.initial_retention_state_digest + } + + /// Returns the registered empty garbage-collection-state digest. + pub const fn initial_gc_state_digest(&self) -> InitialGcStateDigest { + self.fields.initial_gc_state_digest + } + + /// Returns the registered empty recovery-disposition-set digest. + pub const fn empty_disposition_set_digest(&self) -> EmptyDispositionSetDigest { + self.fields.empty_disposition_set_digest + } + + /// Returns the complete admitted synchronization mask. + pub const fn synchronization_mask(&self) -> MigrationSynchronizationMask { + self.fields.synchronization_mask + } + + pub(super) const fn admitted( + encoded: &'encoded [u8], + fields: StoreMigrationReceiptFields, + ) -> Self { + Self { encoded, fields } + } +} diff --git a/src/adapters/store_migration/empty_disposition_set_digest.rs b/src/adapters/store_migration/empty_disposition_set_digest.rs new file mode 100644 index 0000000..ad59041 --- /dev/null +++ b/src/adapters/store_migration/empty_disposition_set_digest.rs @@ -0,0 +1,18 @@ +//! This module owns the registered empty recovery-disposition-set identity. + +/// Registered identity of an empty version-2 recovery-disposition set. +#[must_use] +#[derive(Clone, Copy, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)] +pub struct EmptyDispositionSetDigest([u8; 32]); + +impl EmptyDispositionSetDigest { + /// Returns the exact digest bytes. + #[must_use] + pub const fn as_bytes(&self) -> &[u8; 32] { + &self.0 + } + + pub(super) const fn from_hash(hash: [u8; 32]) -> Self { + Self(hash) + } +} diff --git a/src/adapters/store_migration/initial_gc_state_digest.rs b/src/adapters/store_migration/initial_gc_state_digest.rs new file mode 100644 index 0000000..394c875 --- /dev/null +++ b/src/adapters/store_migration/initial_gc_state_digest.rs @@ -0,0 +1,18 @@ +//! This module owns the registered initial garbage-collection-state identity. + +/// Registered identity of empty version-2 garbage-collection state. +#[must_use] +#[derive(Clone, Copy, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)] +pub struct InitialGcStateDigest([u8; 32]); + +impl InitialGcStateDigest { + /// Returns the exact digest bytes. + #[must_use] + pub const fn as_bytes(&self) -> &[u8; 32] { + &self.0 + } + + pub(super) const fn from_hash(hash: [u8; 32]) -> Self { + Self(hash) + } +} diff --git a/src/adapters/store_migration/initial_retention_state_digest.rs b/src/adapters/store_migration/initial_retention_state_digest.rs new file mode 100644 index 0000000..2dc9cd2 --- /dev/null +++ b/src/adapters/store_migration/initial_retention_state_digest.rs @@ -0,0 +1,18 @@ +//! This module owns the registered initial retention-state identity. + +/// Registered identity of empty version-2 retention state. +#[must_use] +#[derive(Clone, Copy, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)] +pub struct InitialRetentionStateDigest([u8; 32]); + +impl InitialRetentionStateDigest { + /// Returns the exact digest bytes. + #[must_use] + pub const fn as_bytes(&self) -> &[u8; 32] { + &self.0 + } + + pub(super) const fn from_hash(hash: [u8; 32]) -> Self { + Self(hash) + } +} diff --git a/src/adapters/store_migration/migration_intent_bytes.rs b/src/adapters/store_migration/migration_intent_bytes.rs deleted file mode 100644 index 4f6906b..0000000 --- a/src/adapters/store_migration/migration_intent_bytes.rs +++ /dev/null @@ -1,54 +0,0 @@ -//! This boundary module owns fixed-width migration-intent field access. - -use super::StoreMigrationIntentDecodeError; - -pub(super) const ENCODED_LENGTH: usize = 256; - -pub(super) const fn require_length(encoded: &[u8]) -> Result<(), StoreMigrationIntentDecodeError> { - if encoded.len() == ENCODED_LENGTH { - Ok(()) - } else { - Err(wrong_length(encoded)) - } -} - -pub(super) const fn wrong_length(encoded: &[u8]) -> StoreMigrationIntentDecodeError { - StoreMigrationIntentDecodeError::WrongLength { - expected: ENCODED_LENGTH, - observed: encoded.len(), - } -} - -pub(super) fn read_u16( - encoded: &[u8], - offset: usize, -) -> Result { - read_array(encoded, offset).map(u16::from_be_bytes) -} - -pub(super) fn read_u32( - encoded: &[u8], - offset: usize, -) -> Result { - read_array(encoded, offset).map(u32::from_be_bytes) -} - -pub(super) fn read_u64( - encoded: &[u8], - offset: usize, -) -> Result { - read_array(encoded, offset).map(u64::from_be_bytes) -} - -pub(super) fn read_array( - encoded: &[u8], - offset: usize, -) -> Result<[u8; WIDTH], StoreMigrationIntentDecodeError> { - let Some(end) = offset.checked_add(WIDTH) else { - return Err(wrong_length(encoded)); - }; - let bytes = encoded - .get(offset..end) - .ok_or_else(|| wrong_length(encoded))?; - <[u8; WIDTH]>::try_from(bytes).map_err(|_| wrong_length(encoded)) -} diff --git a/src/adapters/store_migration/migration_intent_decoder.rs b/src/adapters/store_migration/migration_intent_decoder.rs index 0108c7a..7dabc5c 100644 --- a/src/adapters/store_migration/migration_intent_decoder.rs +++ b/src/adapters/store_migration/migration_intent_decoder.rs @@ -1,7 +1,7 @@ //! This boundary module owns store-migration intent decoding order. use super::admitted_migration_intent::StoreMigrationIntentFields; -use super::migration_intent_bytes::{ +use super::migration_record_bytes::{ read_array, read_u16, read_u32, read_u64, require_length, wrong_length, }; use super::{ diff --git a/src/adapters/store_migration/migration_receipt_decode_error.rs b/src/adapters/store_migration/migration_receipt_decode_error.rs new file mode 100644 index 0000000..609a38c --- /dev/null +++ b/src/adapters/store_migration/migration_receipt_decode_error.rs @@ -0,0 +1,100 @@ +//! This boundary module owns store-migration receipt decoding failures. + +/// Failure to decode and admit one version-2 migration receipt. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum StoreMigrationReceiptDecodeError { + /// The input was not exactly one complete receipt. + WrongLength { + /// Required fixed width. + expected: usize, + /// Observed input width. + observed: usize, + }, + /// The fixed record magic was not canonical. + InvalidMagic { + /// Observed 16 magic bytes. + observed: [u8; 16], + }, + /// The format version is unsupported. + UnsupportedVersion { + /// Supported version. + expected: u16, + /// Observed version. + observed: u16, + }, + /// The record-length field was noncanonical. + InvalidRecordLength { + /// Required record length. + expected: u16, + /// Observed record length. + observed: u16, + }, + /// The receipt carried unsupported flags. + UnsupportedFlags { + /// Observed flag bits. + observed: u32, + }, + /// The checksum did not match the exact prefix. + ChecksumMismatch { + /// Computed canonical checksum. + expected: [u8; 32], + /// Checksum stored in the record. + observed: [u8; 32], + }, + /// The receipt did not bind the supplied admitted intent. + IntentDigestMismatch { + /// Supplied intent digest. + expected: [u8; 32], + /// Receipt intent digest. + observed: [u8; 32], + }, + /// The receipt did not bind the intent's store identifier. + StoreIdentifierMismatch { + /// Supplied intent store identifier. + expected: [u8; 32], + /// Receipt store identifier. + observed: [u8; 32], + }, + /// The receipt did not bind the supplied admitted marker. + FormatMarkerDigestMismatch { + /// Supplied marker digest. + expected: [u8; 32], + /// Receipt marker digest. + observed: [u8; 32], + }, + /// The initial retention-state digest was not registered. + InitialRetentionStateDigestMismatch { + /// Registered empty-state digest. + expected: [u8; 32], + /// Receipt digest. + observed: [u8; 32], + }, + /// The initial garbage-collection-state digest was not registered. + InitialGcStateDigestMismatch { + /// Registered empty-state digest. + expected: [u8; 32], + /// Receipt digest. + observed: [u8; 32], + }, + /// The empty recovery-disposition-set digest was not registered. + EmptyDispositionSetDigestMismatch { + /// Registered empty-set digest. + expected: [u8; 32], + /// Receipt digest. + observed: [u8; 32], + }, + /// The synchronization mask carried unknown bits. + UnsupportedSynchronizationBits { + /// Complete supported bit set. + supported: u64, + /// Observed mask. + observed: u64, + }, + /// The synchronization mask omitted one or more mandatory bits. + IncompleteSynchronizationMask { + /// Complete required bit set. + required: u64, + /// Observed mask. + observed: u64, + }, +} diff --git a/src/adapters/store_migration/migration_receipt_decode_error_display.rs b/src/adapters/store_migration/migration_receipt_decode_error_display.rs new file mode 100644 index 0000000..16cc473 --- /dev/null +++ b/src/adapters/store_migration/migration_receipt_decode_error_display.rs @@ -0,0 +1,63 @@ +//! This boundary module owns migration-receipt error formatting. + +use std::error::Error; +use std::fmt; + +use super::StoreMigrationReceiptDecodeError; + +impl fmt::Display for StoreMigrationReceiptDecodeError { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::WrongLength { expected, observed } => write!( + formatter, + "migration receipt requires {expected} bytes, observed {observed}" + ), + Self::InvalidMagic { .. } => formatter.write_str("invalid migration-receipt magic"), + Self::UnsupportedVersion { expected, observed } => write!( + formatter, + "unsupported migration-receipt version {observed}; expected {expected}" + ), + Self::InvalidRecordLength { expected, observed } => write!( + formatter, + "migration-receipt record length {observed}; expected {expected}" + ), + Self::UnsupportedFlags { observed } => { + write!( + formatter, + "unsupported migration-receipt flags {observed:#010x}" + ) + } + Self::ChecksumMismatch { .. } => { + formatter.write_str("migration-receipt checksum mismatch") + } + Self::IntentDigestMismatch { .. } => { + formatter.write_str("migration-receipt intent digest mismatch") + } + Self::StoreIdentifierMismatch { .. } => { + formatter.write_str("migration-receipt store identifier mismatch") + } + Self::FormatMarkerDigestMismatch { .. } => { + formatter.write_str("migration-receipt format-marker digest mismatch") + } + Self::InitialRetentionStateDigestMismatch { .. } => { + formatter.write_str("migration-receipt initial retention-state digest mismatch") + } + Self::InitialGcStateDigestMismatch { .. } => { + formatter.write_str("migration-receipt initial GC-state digest mismatch") + } + Self::EmptyDispositionSetDigestMismatch { .. } => { + formatter.write_str("migration-receipt empty disposition-set digest mismatch") + } + Self::UnsupportedSynchronizationBits { observed, .. } => write!( + formatter, + "migration-receipt synchronization mask has unknown bits: {observed:#018x}" + ), + Self::IncompleteSynchronizationMask { observed, .. } => write!( + formatter, + "migration-receipt synchronization mask is incomplete: {observed:#018x}" + ), + } + } +} + +impl Error for StoreMigrationReceiptDecodeError {} diff --git a/src/adapters/store_migration/migration_receipt_decoder.rs b/src/adapters/store_migration/migration_receipt_decoder.rs new file mode 100644 index 0000000..308308d --- /dev/null +++ b/src/adapters/store_migration/migration_receipt_decoder.rs @@ -0,0 +1,157 @@ +//! This boundary module owns store-migration receipt decoding order. + +use super::admitted_migration_receipt::StoreMigrationReceiptFields; +use super::migration_receipt_initial_state::{ + read_empty_disposition_digest, read_initial_gc_digest, read_initial_retention_digest, +}; +use super::migration_record_bytes::{ + read_array, read_u16, read_u32, read_u64, require_length, wrong_length, +}; +use super::{ + AdmittedStoreFormatMarker, AdmittedStoreMigrationIntent, AdmittedStoreMigrationReceipt, + MigrationSynchronizationMask, StoreMigrationReceiptDecodeError, +}; + +const CHECKSUM_OFFSET: usize = 224; +const MAGIC: [u8; 16] = *b"KEEP:MIG:REC2\0\0\0"; +const VERSION: u16 = 2; +const RECORD_LENGTH: u16 = 256; +const CHECKSUM_DOMAIN: &[u8] = b"keep.store-migration-receipt-checksum/v2\0"; + +pub(super) fn decode<'encoded>( + encoded: &'encoded [u8], + intent: &AdmittedStoreMigrationIntent<'_>, + marker: &AdmittedStoreFormatMarker<'_>, +) -> Result, StoreMigrationReceiptDecodeError> { + require_length(encoded)?; + validate_fixed_fields(encoded)?; + verify_checksum(encoded)?; + let intent_digest = bind_intent_digest(encoded, intent)?; + let store_identifier = bind_store_identifier(encoded, intent)?; + let format_marker_digest = bind_marker_digest(encoded, marker)?; + let initial_retention_state_digest = read_initial_retention_digest(encoded)?; + let initial_gc_state_digest = read_initial_gc_digest(encoded)?; + let empty_disposition_set_digest = read_empty_disposition_digest(encoded)?; + let synchronization_mask = read_synchronization_mask(encoded)?; + Ok(AdmittedStoreMigrationReceipt::admitted( + encoded, + StoreMigrationReceiptFields { + intent_digest, + store_identifier, + format_marker_digest, + initial_retention_state_digest, + initial_gc_state_digest, + empty_disposition_set_digest, + synchronization_mask, + }, + )) +} + +fn validate_fixed_fields(encoded: &[u8]) -> Result<(), StoreMigrationReceiptDecodeError> { + let magic = read_array(encoded, 0)?; + if magic != MAGIC { + return Err(StoreMigrationReceiptDecodeError::InvalidMagic { observed: magic }); + } + let version = read_u16(encoded, 16)?; + if version != VERSION { + return Err(StoreMigrationReceiptDecodeError::UnsupportedVersion { + expected: VERSION, + observed: version, + }); + } + let record_length = read_u16(encoded, 18)?; + if record_length != RECORD_LENGTH { + return Err(StoreMigrationReceiptDecodeError::InvalidRecordLength { + expected: RECORD_LENGTH, + observed: record_length, + }); + } + let flags = read_u32(encoded, 20)?; + if flags != 0 { + return Err(StoreMigrationReceiptDecodeError::UnsupportedFlags { observed: flags }); + } + Ok(()) +} + +fn verify_checksum(encoded: &[u8]) -> Result<(), StoreMigrationReceiptDecodeError> { + let preimage = encoded + .get(..CHECKSUM_OFFSET) + .ok_or_else(|| wrong_length(encoded))?; + let observed = read_array(encoded, CHECKSUM_OFFSET)?; + let expected = hash(CHECKSUM_DOMAIN, preimage); + if observed == expected { + Ok(()) + } else { + Err(StoreMigrationReceiptDecodeError::ChecksumMismatch { expected, observed }) + } +} + +fn bind_intent_digest( + encoded: &[u8], + intent: &AdmittedStoreMigrationIntent<'_>, +) -> Result { + let expected = *intent.digest().as_bytes(); + let observed = read_array(encoded, 24)?; + if observed == expected { + Ok(intent.digest()) + } else { + Err(StoreMigrationReceiptDecodeError::IntentDigestMismatch { expected, observed }) + } +} + +fn bind_store_identifier( + encoded: &[u8], + intent: &AdmittedStoreMigrationIntent<'_>, +) -> Result { + let expected = *intent.store_identifier().as_bytes(); + let observed = read_array(encoded, 56)?; + if observed == expected { + Ok(intent.store_identifier()) + } else { + Err(StoreMigrationReceiptDecodeError::StoreIdentifierMismatch { expected, observed }) + } +} + +fn bind_marker_digest( + encoded: &[u8], + marker: &AdmittedStoreFormatMarker<'_>, +) -> Result { + let expected = *marker.digest().as_bytes(); + let observed = read_array(encoded, 88)?; + if observed == expected { + Ok(marker.digest()) + } else { + Err(StoreMigrationReceiptDecodeError::FormatMarkerDigestMismatch { expected, observed }) + } +} + +fn read_synchronization_mask( + encoded: &[u8], +) -> Result { + let observed = read_u64(encoded, 216)?; + let supported = MigrationSynchronizationMask::COMPLETE_BITS; + if observed & !supported != 0 { + return Err( + StoreMigrationReceiptDecodeError::UnsupportedSynchronizationBits { + supported, + observed, + }, + ); + } + if observed != supported { + return Err( + StoreMigrationReceiptDecodeError::IncompleteSynchronizationMask { + required: supported, + observed, + }, + ); + } + Ok(MigrationSynchronizationMask::complete()) +} + +fn hash(domain: &[u8], bytes: &[u8]) -> [u8; 32] { + let mut hasher = blake3::Hasher::new(); + hasher.update(domain); + hasher.update(bytes); + *hasher.finalize().as_bytes() +} diff --git a/src/adapters/store_migration/migration_receipt_initial_state.rs b/src/adapters/store_migration/migration_receipt_initial_state.rs new file mode 100644 index 0000000..01a43d5 --- /dev/null +++ b/src/adapters/store_migration/migration_receipt_initial_state.rs @@ -0,0 +1,61 @@ +//! This boundary module owns registered empty-state receipt admission. + +use super::migration_record_bytes::read_array; +use super::{ + EmptyDispositionSetDigest, InitialGcStateDigest, InitialRetentionStateDigest, + StoreMigrationReceiptDecodeError, +}; + +const INITIAL_RETENTION_DOMAIN: &[u8] = b"keep.initial-retention-state/v2\0"; +const INITIAL_GC_DOMAIN: &[u8] = b"keep.initial-gc-state/v2\0"; +const EMPTY_DISPOSITION_DOMAIN: &[u8] = b"keep.empty-disposition-set/v2\0"; + +pub(super) fn read_initial_retention_digest( + encoded: &[u8], +) -> Result { + let expected = digest(INITIAL_RETENTION_DOMAIN); + let observed = read_array(encoded, 120)?; + if observed == expected { + Ok(InitialRetentionStateDigest::from_hash(expected)) + } else { + Err( + StoreMigrationReceiptDecodeError::InitialRetentionStateDigestMismatch { + expected, + observed, + }, + ) + } +} + +pub(super) fn read_initial_gc_digest( + encoded: &[u8], +) -> Result { + let expected = digest(INITIAL_GC_DOMAIN); + let observed = read_array(encoded, 152)?; + if observed == expected { + Ok(InitialGcStateDigest::from_hash(expected)) + } else { + Err(StoreMigrationReceiptDecodeError::InitialGcStateDigestMismatch { expected, observed }) + } +} + +pub(super) fn read_empty_disposition_digest( + encoded: &[u8], +) -> Result { + let expected = digest(EMPTY_DISPOSITION_DOMAIN); + let observed = read_array(encoded, 184)?; + if observed == expected { + Ok(EmptyDispositionSetDigest::from_hash(expected)) + } else { + Err( + StoreMigrationReceiptDecodeError::EmptyDispositionSetDigestMismatch { + expected, + observed, + }, + ) + } +} + +fn digest(domain: &[u8]) -> [u8; 32] { + *blake3::hash(domain).as_bytes() +} diff --git a/src/adapters/store_migration/migration_record_bytes.rs b/src/adapters/store_migration/migration_record_bytes.rs new file mode 100644 index 0000000..774469d --- /dev/null +++ b/src/adapters/store_migration/migration_record_bytes.rs @@ -0,0 +1,69 @@ +//! This boundary module owns fixed-width store-migration record field access. + +use super::{StoreMigrationIntentDecodeError, StoreMigrationReceiptDecodeError}; + +const ENCODED_LENGTH: usize = 256; + +pub(super) trait MigrationRecordDecodeError: Sized { + fn wrong_length(expected: usize, observed: usize) -> Self; +} + +impl MigrationRecordDecodeError for StoreMigrationIntentDecodeError { + fn wrong_length(expected: usize, observed: usize) -> Self { + Self::WrongLength { expected, observed } + } +} + +impl MigrationRecordDecodeError for StoreMigrationReceiptDecodeError { + fn wrong_length(expected: usize, observed: usize) -> Self { + Self::WrongLength { expected, observed } + } +} + +pub(super) fn require_length( + encoded: &[u8], +) -> Result<(), Error> { + if encoded.len() == ENCODED_LENGTH { + Ok(()) + } else { + Err(wrong_length(encoded)) + } +} + +pub(super) fn wrong_length(encoded: &[u8]) -> Error { + Error::wrong_length(ENCODED_LENGTH, encoded.len()) +} + +pub(super) fn read_u16( + encoded: &[u8], + offset: usize, +) -> Result { + read_array(encoded, offset).map(u16::from_be_bytes) +} + +pub(super) fn read_u32( + encoded: &[u8], + offset: usize, +) -> Result { + read_array(encoded, offset).map(u32::from_be_bytes) +} + +pub(super) fn read_u64( + encoded: &[u8], + offset: usize, +) -> Result { + read_array(encoded, offset).map(u64::from_be_bytes) +} + +pub(super) fn read_array( + encoded: &[u8], + offset: usize, +) -> Result<[u8; WIDTH], Error> { + let Some(end) = offset.checked_add(WIDTH) else { + return Err(wrong_length(encoded)); + }; + let bytes = encoded + .get(offset..end) + .ok_or_else(|| wrong_length(encoded))?; + <[u8; WIDTH]>::try_from(bytes).map_err(|_| wrong_length(encoded)) +} diff --git a/src/adapters/store_migration/migration_synchronization_mask.rs b/src/adapters/store_migration/migration_synchronization_mask.rs new file mode 100644 index 0000000..47c46cd --- /dev/null +++ b/src/adapters/store_migration/migration_synchronization_mask.rs @@ -0,0 +1,20 @@ +//! This module owns completed migration synchronization evidence. + +/// Closed set of mandatory migration synchronization transitions. +#[must_use] +#[derive(Clone, Copy, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)] +pub struct MigrationSynchronizationMask(u64); + +impl MigrationSynchronizationMask { + pub(super) const COMPLETE_BITS: u64 = 0x03ff; + + /// Returns the complete synchronization bit set. + #[must_use] + pub const fn bits(self) -> u64 { + self.0 + } + + pub(super) const fn complete() -> Self { + Self(Self::COMPLETE_BITS) + } +} diff --git a/src/lib.rs b/src/lib.rs index c09acfc..fb536f3 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -33,9 +33,11 @@ //! canonical encoding, registered-definition admission, checksum verification, //! and domain-separated identity. Migration-intent admission validates its //! framing, checksum, catalog and predecessor grammar, registered definition, -//! deterministic store identity, and typed recovery coordinates. Live -//! inventory and root revalidation, filesystem migration, retention execution, -//! recovery, and garbage collection remain intentionally absent. +//! deterministic store identity, and typed recovery coordinates. Completion +//! receipts bind an admitted intent and marker, registered empty-state digests, +//! and the complete synchronization mask. Live inventory and root revalidation, +//! filesystem migration, retention execution, recovery, and garbage collection +//! remain intentionally absent. #[cfg(test)] extern crate self as keep; @@ -54,26 +56,27 @@ mod retention; pub use adapters::RepositoryInitializationStorage; pub use adapters::{ AdmittedCatalog, AdmittedRecoveryStageBytes, AdmittedSegment, AdmittedSegmentRecord, - AdmittedStoreFormatMarker, AdmittedStoreMigrationIntent, BlobIdBinaryParseError, - BlobIdTextParseError, CanonicalCatalog, CanonicalLayoutRecord, CanonicalPublicationHead, - CanonicalStoreFormatMarker, CatalogAdmissionError, CatalogAllocationPhase, CatalogDecodeError, - CatalogEncodeError, CatalogEntryDecodeError, CatalogPublicationError, - CatalogPublicationExpectation, CatalogPublicationOutcome, CatalogPublicationPhase, - CatalogPublicationReadiness, CatalogPublicationReceipt, CatalogPublicationStorage, - CatalogRestartArtifact, CatalogRestartByteLimit, CatalogRestartByteLimitError, - CatalogRestartError, CatalogRestartPhase, CatalogRestartPolicy, CatalogSnapshot, - CatalogSnapshotError, CatalogSuccessor, CatalogTransitionError, ChecksummedCatalog, - ChecksummedPublicationHead, ChecksummedSegmentRecord, ClosedSegment, - FilesystemCatalogPublicationError, FilesystemCatalogPublisher, FilesystemCatalogSnapshot, - FilesystemPlatformAdmission, FilesystemPlatformAdmissionError, + AdmittedStoreFormatMarker, AdmittedStoreMigrationIntent, AdmittedStoreMigrationReceipt, + BlobIdBinaryParseError, BlobIdTextParseError, CanonicalCatalog, CanonicalLayoutRecord, + CanonicalPublicationHead, CanonicalStoreFormatMarker, CatalogAdmissionError, + CatalogAllocationPhase, CatalogDecodeError, CatalogEncodeError, CatalogEntryDecodeError, + CatalogPublicationError, CatalogPublicationExpectation, CatalogPublicationOutcome, + CatalogPublicationPhase, CatalogPublicationReadiness, CatalogPublicationReceipt, + CatalogPublicationStorage, CatalogRestartArtifact, CatalogRestartByteLimit, + CatalogRestartByteLimitError, CatalogRestartError, CatalogRestartPhase, CatalogRestartPolicy, + CatalogSnapshot, CatalogSnapshotError, CatalogSuccessor, CatalogTransitionError, + ChecksummedCatalog, ChecksummedPublicationHead, ChecksummedSegmentRecord, ClosedSegment, + EmptyDispositionSetDigest, FilesystemCatalogPublicationError, FilesystemCatalogPublisher, + FilesystemCatalogSnapshot, FilesystemPlatformAdmission, FilesystemPlatformAdmissionError, FilesystemRecoveryInventoryReader, FilesystemRecoveryNextHeadFinalizationOpenError, FilesystemRecoveryNextHeadFinalizer, FilesystemRecoverySegmentResumeOpenError, FilesystemRecoverySegmentResumer, FilesystemRecoverySegmentStage, FilesystemRecoveryStageCompleter, FilesystemRecoveryStageCompletionOpenError, FilesystemRecoveryStageDiscardOpenError, FilesystemRecoveryStageDiscarder, FilesystemRecoveryStageError, FilesystemSegmentStage, FilesystemWriterLock, - ImmutablePoolInventoryDigest, LayoutDecodeError, LayoutDecodePolicy, LayoutEncodeError, - LayoutIdBinaryParseError, LayoutIdTextParseError, OpenedReusableSegment, + ImmutablePoolInventoryDigest, InitialGcStateDigest, InitialRetentionStateDigest, + LayoutDecodeError, LayoutDecodePolicy, LayoutEncodeError, LayoutIdBinaryParseError, + LayoutIdTextParseError, MigrationSynchronizationMask, OpenedReusableSegment, PublicationHeadDecodeError, RecoveryCatalogStage, RecoveryCatalogStageError, RecoveryEntryName, RecoveryEntryNameError, RecoveryEntryRole, RecoveryInventory, RecoveryInventoryEntry, RecoveryInventoryError, RecoveryInventoryLimit, RecoveryInventoryLimitError, @@ -107,8 +110,8 @@ pub use adapters::{ StoreFormatDefinitionDigest, StoreFormatMarkerDecodeError, StoreFormatMarkerDigest, StoreIdentifier, StoreInitializationError, StoreInitializationPhase, StoreInitializationReceipt, StoreInitializationStorage, StoreMigrationIntentDecodeError, - StoreMigrationIntentDigest, StoreRootDeviceIdentity, StoreRootFileIdentity, - StoreRootMountIdentity, WriterLockAcquireError, WriterLockAcquirePhase, + StoreMigrationIntentDigest, StoreMigrationReceiptDecodeError, StoreRootDeviceIdentity, + StoreRootFileIdentity, StoreRootMountIdentity, WriterLockAcquireError, WriterLockAcquirePhase, admit_recovery_stage_bytes, assess_recovery_stage, classify_recovery_catalog_stage, classify_recovery_names, classify_recovery_next_head_stage, classify_recovery_segment_stage, execute_recovery_next_head_finalization, execute_recovery_segment_resume, diff --git a/tests/store_migration_receipt.rs b/tests/store_migration_receipt.rs new file mode 100644 index 0000000..bbacfd2 --- /dev/null +++ b/tests/store_migration_receipt.rs @@ -0,0 +1,96 @@ +//! Canonical version-2 store-migration receipt laws. + +#[path = "store_migration_receipt/binding_laws.rs"] +mod binding_laws; +#[path = "store_migration_receipt/fixture.rs"] +mod fixture; +#[path = "store_migration_receipt/harness.rs"] +mod harness; +mod support; + +use fixture::{ + DISPOSITION_DIGEST, INITIAL_GC_DIGEST, INITIAL_RETENTION_DIGEST, intent_bytes, marker_bytes, + receipt_bytes, +}; +use harness::{assert_fixed_refusal, assert_receipt_refusal, mutated_array}; +use keep::{ + AdmittedStoreFormatMarker, AdmittedStoreMigrationIntent, AdmittedStoreMigrationReceipt, + StoreMigrationReceiptDecodeError, +}; + +#[test] +fn receipt_admits_every_frozen_completion_coordinate() -> Result<(), Box> { + let intent_bytes = intent_bytes()?; + let marker_bytes = marker_bytes()?; + let receipt_bytes = receipt_bytes()?; + let intent = AdmittedStoreMigrationIntent::decode(&intent_bytes)?; + let marker = AdmittedStoreFormatMarker::decode(&marker_bytes)?; + let receipt = AdmittedStoreMigrationReceipt::decode(&receipt_bytes, &intent, &marker)?; + + assert_eq!(receipt.encoded(), receipt_bytes); + assert_eq!(receipt.intent_digest(), intent.digest()); + assert_eq!(receipt.store_identifier(), intent.store_identifier()); + assert_eq!(receipt.format_marker_digest(), marker.digest()); + assert_eq!( + receipt.initial_retention_state_digest().as_bytes(), + &INITIAL_RETENTION_DIGEST + ); + assert_eq!( + receipt.initial_gc_state_digest().as_bytes(), + &INITIAL_GC_DIGEST + ); + assert_eq!( + receipt.empty_disposition_set_digest().as_bytes(), + &DISPOSITION_DIGEST + ); + assert_eq!(receipt.synchronization_mask().bits(), 0x03ff); + Ok(()) +} + +#[test] +fn receipt_framing_has_exact_first_refusals() -> Result<(), Box> { + let bytes = receipt_bytes()?; + let mut truncated = bytes.clone(); + assert!(truncated.pop().is_some()); + assert_receipt_refusal( + &truncated, + StoreMigrationReceiptDecodeError::WrongLength { + expected: 256, + observed: 255, + }, + )?; + let mut extended = bytes.clone(); + extended.push(0); + assert_receipt_refusal( + &extended, + StoreMigrationReceiptDecodeError::WrongLength { + expected: 256, + observed: 257, + }, + )?; + assert_fixed_refusal( + 0, + StoreMigrationReceiptDecodeError::InvalidMagic { + observed: mutated_array(&bytes, 0, 0)?, + }, + )?; + assert_fixed_refusal( + 17, + StoreMigrationReceiptDecodeError::UnsupportedVersion { + expected: 2, + observed: 3, + }, + )?; + assert_fixed_refusal( + 19, + StoreMigrationReceiptDecodeError::InvalidRecordLength { + expected: 256, + observed: 257, + }, + )?; + assert_fixed_refusal( + 23, + StoreMigrationReceiptDecodeError::UnsupportedFlags { observed: 1 }, + )?; + Ok(()) +} diff --git a/tests/store_migration_receipt/binding_laws.rs b/tests/store_migration_receipt/binding_laws.rs new file mode 100644 index 0000000..c7a8cc0 --- /dev/null +++ b/tests/store_migration_receipt/binding_laws.rs @@ -0,0 +1,104 @@ +//! This module owns migration-receipt integrity and binding laws. + +use keep::{ + AdmittedStoreFormatMarker, AdmittedStoreMigrationIntent, AdmittedStoreMigrationReceipt, + StoreMigrationReceiptDecodeError, +}; + +use super::fixture::{ + DISPOSITION_DIGEST, INITIAL_GC_DIGEST, INITIAL_RETENTION_DIGEST, intent_bytes, marker_bytes, + receipt_bytes, +}; +use super::harness::{ + assert_semantic_refusal, decode_receipt, digest_intent, flip_byte, mutated_array, + refresh_checksum, +}; + +#[test] +fn receipt_integrity_and_binding_have_exact_precedence() -> Result<(), Box> { + let mut checksum = receipt_bytes()?; + flip_byte(&mut checksum, 24)?; + assert!(matches!( + decode_receipt(&checksum)?, + Err(StoreMigrationReceiptDecodeError::ChecksumMismatch { .. }) + )); + + let intent = intent_bytes()?; + let marker = marker_bytes()?; + assert_semantic_refusal( + 24, + StoreMigrationReceiptDecodeError::IntentDigestMismatch { + expected: digest_intent(&intent), + observed: mutated_array(&receipt_bytes()?, 24, 0)?, + }, + )?; + assert_semantic_refusal( + 56, + StoreMigrationReceiptDecodeError::StoreIdentifierMismatch { + expected: *AdmittedStoreMigrationIntent::decode(&intent)? + .store_identifier() + .as_bytes(), + observed: mutated_array(&receipt_bytes()?, 56, 0)?, + }, + )?; + assert_semantic_refusal( + 88, + StoreMigrationReceiptDecodeError::FormatMarkerDigestMismatch { + expected: *AdmittedStoreFormatMarker::decode(&marker)? + .digest() + .as_bytes(), + observed: mutated_array(&receipt_bytes()?, 88, 0)?, + }, + )?; + assert_semantic_refusal( + 120, + StoreMigrationReceiptDecodeError::InitialRetentionStateDigestMismatch { + expected: INITIAL_RETENTION_DIGEST, + observed: mutated_array(&receipt_bytes()?, 120, 0)?, + }, + )?; + assert_semantic_refusal( + 152, + StoreMigrationReceiptDecodeError::InitialGcStateDigestMismatch { + expected: INITIAL_GC_DIGEST, + observed: mutated_array(&receipt_bytes()?, 152, 0)?, + }, + )?; + assert_semantic_refusal( + 184, + StoreMigrationReceiptDecodeError::EmptyDispositionSetDigestMismatch { + expected: DISPOSITION_DIGEST, + observed: mutated_array(&receipt_bytes()?, 184, 0)?, + }, + )?; + assert_semantic_refusal( + 221, + StoreMigrationReceiptDecodeError::UnsupportedSynchronizationBits { + supported: 0x03ff, + observed: 0x0001_03ff, + }, + )?; + assert_semantic_refusal( + 222, + StoreMigrationReceiptDecodeError::IncompleteSynchronizationMask { + required: 0x03ff, + observed: 0x02ff, + }, + )?; + + let mut alternative_intent = intent; + flip_byte(&mut alternative_intent, 159)?; + refresh_checksum( + &mut alternative_intent, + 224, + b"keep.store-migration-intent-checksum/v2\0", + )?; + let alternative = AdmittedStoreMigrationIntent::decode(&alternative_intent)?; + let receipt = receipt_bytes()?; + let marker = AdmittedStoreFormatMarker::decode(&marker)?; + assert!(matches!( + AdmittedStoreMigrationReceipt::decode(&receipt, &alternative, &marker), + Err(StoreMigrationReceiptDecodeError::IntentDigestMismatch { .. }) + )); + Ok(()) +} diff --git a/tests/store_migration_receipt/fixture.rs b/tests/store_migration_receipt/fixture.rs new file mode 100644 index 0000000..10a6ff2 --- /dev/null +++ b/tests/store_migration_receipt/fixture.rs @@ -0,0 +1,38 @@ +//! This module owns frozen migration-receipt fixture values. +#![allow( + clippy::redundant_pub_crate, + reason = "the parent integration-test module consumes this private fixture" +)] + +use std::io; + +use super::support; + +const RECEIPT: &str = include_str!("../../conformance/segment-store/v2/migration-receipt.hex"); +const INTENT: &str = include_str!("../../conformance/segment-store/v2/migration-intent.hex"); +const MARKER: &str = include_str!("../../conformance/segment-store/v2/format-marker.hex"); + +pub(super) const INITIAL_RETENTION_DIGEST: [u8; 32] = [ + 0xd5, 0x2f, 0x1f, 0x02, 0x2e, 0xdb, 0x1d, 0xe7, 0xb8, 0x40, 0xc5, 0xbf, 0x8f, 0xb5, 0x5d, 0xe7, + 0x93, 0x2c, 0xa6, 0x93, 0x70, 0xae, 0x85, 0xe2, 0xbe, 0xe4, 0x17, 0x91, 0x43, 0x79, 0x2b, 0xc3, +]; +pub(super) const INITIAL_GC_DIGEST: [u8; 32] = [ + 0xba, 0x0e, 0xa2, 0x00, 0xa5, 0xb0, 0x67, 0x41, 0x56, 0x4c, 0x43, 0xa7, 0x9a, 0x91, 0x94, 0x5b, + 0xef, 0x0b, 0x0f, 0xac, 0x51, 0xc9, 0x60, 0xea, 0x4f, 0x82, 0x07, 0x09, 0x4f, 0x3e, 0x1e, 0x31, +]; +pub(super) const DISPOSITION_DIGEST: [u8; 32] = [ + 0xa8, 0x02, 0x59, 0xfc, 0xd1, 0x23, 0x72, 0x03, 0xea, 0x6c, 0x6c, 0xc5, 0x06, 0x55, 0x14, 0xab, + 0xde, 0xb0, 0x1d, 0xa6, 0x03, 0xc3, 0x19, 0x4b, 0x09, 0x6a, 0x04, 0x5c, 0xf6, 0x94, 0xc9, 0x5a, +]; + +pub(super) fn receipt_bytes() -> Result, io::Error> { + support::decode_hex(RECEIPT.trim_end()) +} + +pub(super) fn intent_bytes() -> Result, io::Error> { + support::decode_hex(INTENT.trim_end()) +} + +pub(super) fn marker_bytes() -> Result, io::Error> { + support::decode_hex(MARKER.trim_end()) +} diff --git a/tests/store_migration_receipt/harness.rs b/tests/store_migration_receipt/harness.rs new file mode 100644 index 0000000..296b863 --- /dev/null +++ b/tests/store_migration_receipt/harness.rs @@ -0,0 +1,107 @@ +//! This module owns migration-receipt mutation and admission test mechanics. +#![allow( + clippy::redundant_pub_crate, + reason = "sibling private test modules consume this harness" +)] + +use std::io; + +use keep::{ + AdmittedStoreFormatMarker, AdmittedStoreMigrationIntent, AdmittedStoreMigrationReceipt, + StoreMigrationReceiptDecodeError, +}; + +use super::fixture::{intent_bytes, marker_bytes, receipt_bytes}; + +pub(super) fn assert_fixed_refusal( + offset: usize, + expected: StoreMigrationReceiptDecodeError, +) -> Result<(), Box> { + let mut bytes = receipt_bytes()?; + flip_byte(&mut bytes, offset)?; + assert_receipt_refusal(&bytes, expected) +} + +pub(super) fn assert_semantic_refusal( + offset: usize, + expected: StoreMigrationReceiptDecodeError, +) -> Result<(), Box> { + let mut bytes = receipt_bytes()?; + flip_byte(&mut bytes, offset)?; + refresh_checksum( + &mut bytes, + 224, + b"keep.store-migration-receipt-checksum/v2\0", + )?; + assert_receipt_refusal(&bytes, expected) +} + +pub(super) fn assert_receipt_refusal( + bytes: &[u8], + expected: StoreMigrationReceiptDecodeError, +) -> Result<(), Box> { + assert_eq!(decode_receipt(bytes)?, Err(expected)); + Ok(()) +} + +pub(super) fn decode_receipt( + bytes: &[u8], +) -> Result< + Result, StoreMigrationReceiptDecodeError>, + Box, +> { + let intent_bytes = intent_bytes()?; + let marker_bytes = marker_bytes()?; + let intent = AdmittedStoreMigrationIntent::decode(&intent_bytes)?; + let marker = AdmittedStoreFormatMarker::decode(&marker_bytes)?; + Ok(AdmittedStoreMigrationReceipt::decode( + bytes, &intent, &marker, + )) +} + +pub(super) fn mutated_array( + bytes: &[u8], + offset: usize, + relative: usize, +) -> Result<[u8; WIDTH], io::Error> { + let end = offset + .checked_add(WIDTH) + .ok_or_else(|| io::Error::other("receipt field offset overflow"))?; + let field = bytes + .get(offset..end) + .ok_or_else(|| io::Error::other("receipt lacks fixed field"))?; + let mut observed = <[u8; WIDTH]>::try_from(field) + .map_err(|_| io::Error::other("receipt field width mismatch"))?; + flip_byte(&mut observed, relative)?; + Ok(observed) +} + +pub(super) fn flip_byte(bytes: &mut [u8], offset: usize) -> Result<(), io::Error> { + let byte = bytes + .get_mut(offset) + .ok_or_else(|| io::Error::other("receipt mutation offset is out of bounds"))?; + *byte ^= 1; + Ok(()) +} + +pub(super) fn refresh_checksum( + bytes: &mut [u8], + offset: usize, + domain: &[u8], +) -> Result<(), io::Error> { + let (preimage, checksum) = bytes + .split_at_mut_checked(offset) + .ok_or_else(|| io::Error::other("record lacks checksum boundary"))?; + let mut hasher = blake3::Hasher::new(); + hasher.update(domain); + hasher.update(preimage); + checksum.copy_from_slice(hasher.finalize().as_bytes()); + Ok(()) +} + +pub(super) fn digest_intent(bytes: &[u8]) -> [u8; 32] { + let mut hasher = blake3::Hasher::new(); + hasher.update(b"keep.store-migration-intent/v2\0"); + hasher.update(bytes); + *hasher.finalize().as_bytes() +} From 5e541b0e84789afb7c1f0da972401182de1efc7f Mon Sep 17 00:00:00 2001 From: James Ross Date: Thu, 30 Jul 2026 02:39:40 -0700 Subject: [PATCH 033/111] Test: Fuzz store migration records --- CHANGELOG.md | 10 +- docs/formats/segment-store-v2/requirements.md | 2 +- fuzz/Cargo.toml | 7 ++ fuzz/README.md | 5 + fuzz/fuzz_targets/migration_format.rs | 52 ++++++++++ xtask/src/fuzz_campaign/target/tests.rs | 1 + xtask/src/fuzz_seed_corpus.rs | 3 + xtask/src/fuzz_seed_corpus/migration_seeds.rs | 94 +++++++++++++++++++ xtask/src/fuzz_seed_corpus/retention_seeds.rs | 25 +---- .../segment_store_v2_fixture.rs | 27 ++++++ .../fuzz_seed_corpus/tests/materialization.rs | 19 ++-- .../parser_fuzz_laws.rs | 23 +++++ 12 files changed, 233 insertions(+), 35 deletions(-) create mode 100644 fuzz/fuzz_targets/migration_format.rs create mode 100644 xtask/src/fuzz_seed_corpus/migration_seeds.rs create mode 100644 xtask/src/fuzz_seed_corpus/segment_store_v2_fixture.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index a105add..6b3bb30 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,12 +10,12 @@ after its public API and format compatibility policies are established. ### Changed -- Version-2 marker, migration-intent, and completion-receipt admission now bind - exact catalog, predecessor, root, definition, store, empty-state, checksum, - digest, and synchronization-mask coordinates. Retention preflight combines +- Version-2 marker, migration-intent, and completion-receipt admission now binds + exact catalog, predecessor, root, definition, store, empty-state, and checksum, + digest, and synchronization-mask coordinates; a seeded migration fuzz + surface drives all three exact decoders. Retention preflight combines expected-generation planning with deterministic closure verification; - authority-revalidated 17-phase orchestration returns an unforgeable - complete-coordinate receipt after durable cleanup. + authority-revalidated 17-phase orchestration returns its receipt after durable cleanup. - Repository crash-matrix execution now terminates isolated writer process groups at all 105 canonical before/during/after coordinates, retains open writer and stage authority until termination, executes production diff --git a/docs/formats/segment-store-v2/requirements.md b/docs/formats/segment-store-v2/requirements.md index 9d85a84..bff404f 100644 --- a/docs/formats/segment-store-v2/requirements.md +++ b/docs/formats/segment-store-v2/requirements.md @@ -29,7 +29,7 @@ case is not evidence. | ID | Requirement | Evidence | Status | | --- | --- | --- | --- | | `KEEP-MIGRATION-001` | Exact version-1 stores remain admitted until a durable migration artifact exists | compatibility fixtures | Planned in #19 | -| `KEEP-MIGRATION-002` | Format marker, intent, and receipt have complete fixed byte tables, named domains, bounds, checksums, deterministic store identity, and exact initial-state digests | exact admission in `tests/store_format_marker.rs`, `tests/store_migration_intent.rs`, and `tests/store_migration_receipt.rs` | Implemented | +| `KEEP-MIGRATION-002` | Format marker, intent, and receipt have complete fixed byte tables, named domains, bounds, checksums, deterministic store identity, and exact initial-state digests | exact admission in `tests/store_format_marker.rs`, `tests/store_migration_intent.rs`, and `tests/store_migration_receipt.rs`; seeded `migration_format` fuzz target | Implemented | | `KEEP-MIGRATION-003` | Migration revalidates version-1 head, catalog, pools, root identity, and writer authority before mutation | capability-relative integration tests | Planned in #19 | | `KEEP-MIGRATION-004` | Every partial migration prefix continues idempotently under writer authority | state-machine and recovery tests | Planned in #19 | | `KEEP-MIGRATION-005` | Unknown, out-of-order, substituted, corrupt, conflicting, or changed evidence is unrecoverable ambiguity | corruption and mutation matrix | Planned in #19 | diff --git a/fuzz/Cargo.toml b/fuzz/Cargo.toml index b041049..88feb48 100644 --- a/fuzz/Cargo.toml +++ b/fuzz/Cargo.toml @@ -117,6 +117,13 @@ test = false doc = false bench = false +[[bin]] +name = "migration_format" +path = "fuzz_targets/migration_format.rs" +test = false +doc = false +bench = false + [[bin]] name = "repository_json" path = "fuzz_targets/repository_json.rs" diff --git a/fuzz/README.md b/fuzz/README.md index 8b6af19..115dc98 100644 --- a/fuzz/README.md +++ b/fuzz/README.md @@ -70,6 +70,11 @@ retention-manifest, and retention-head decoders. The canonical one-root generation keeps mutations inside framing, semantic, ordering, checksum, and digest validation; every admitted value must retain its exact input bytes. +The `migration_format` seeds select the public format-marker, migration-intent, +and completion-receipt decoders. The receipt seed carries its exact marker and +intent dependencies so mutations exercise integrity and cross-record binding; +every admitted value must retain its exact input bytes. + The `segment_format` seeds select the public segment-header, record-header, complete-record, seal, and complete-segment boundaries. Canonical empty, one-record, and bundled segments keep mutations inside the nested parsers; diff --git a/fuzz/fuzz_targets/migration_format.rs b/fuzz/fuzz_targets/migration_format.rs new file mode 100644 index 0000000..739c906 --- /dev/null +++ b/fuzz/fuzz_targets/migration_format.rs @@ -0,0 +1,52 @@ +#![no_main] + +//! This target owns canonical store-migration record parser fuzzing. + +use keep::{ + AdmittedStoreFormatMarker, AdmittedStoreMigrationIntent, AdmittedStoreMigrationReceipt, +}; +use libfuzzer_sys::fuzz_target; + +const MARKER_BYTES: usize = 96; +const INTENT_BYTES: usize = 256; + +fuzz_target!(|bytes: &[u8]| { + let Some((&selector, input)) = bytes.split_first() else { + return; + }; + match selector { + 0 => marker(input), + 1 => intent(input), + _ => receipt(input), + } +}); + +fn marker(input: &[u8]) { + if let Ok(marker) = AdmittedStoreFormatMarker::decode(input) { + assert_eq!(marker.encoded(), input); + } +} + +fn intent(input: &[u8]) { + if let Ok(intent) = AdmittedStoreMigrationIntent::decode(input) { + assert_eq!(intent.encoded(), input); + } +} + +fn receipt(input: &[u8]) { + let Some((marker_bytes, remainder)) = input.split_at_checked(MARKER_BYTES) else { + return; + }; + let Some((intent_bytes, receipt_bytes)) = remainder.split_at_checked(INTENT_BYTES) else { + return; + }; + let (Ok(marker), Ok(intent)) = ( + AdmittedStoreFormatMarker::decode(marker_bytes), + AdmittedStoreMigrationIntent::decode(intent_bytes), + ) else { + return; + }; + if let Ok(receipt) = AdmittedStoreMigrationReceipt::decode(receipt_bytes, &intent, &marker) { + assert_eq!(receipt.encoded(), receipt_bytes); + } +} diff --git a/xtask/src/fuzz_campaign/target/tests.rs b/xtask/src/fuzz_campaign/target/tests.rs index 7b72f99..b43744d 100644 --- a/xtask/src/fuzz_campaign/target/tests.rs +++ b/xtask/src/fuzz_campaign/target/tests.rs @@ -30,6 +30,7 @@ fn checked_in_harness_set_is_exact_and_sorted() -> Result<(), Box> { "fast_cdc", "golden_protocol", "layout_record", + "migration_format", "repository_json", "retention_format", "segment_format", diff --git a/xtask/src/fuzz_seed_corpus.rs b/xtask/src/fuzz_seed_corpus.rs index 1455e04..3635322 100644 --- a/xtask/src/fuzz_seed_corpus.rs +++ b/xtask/src/fuzz_seed_corpus.rs @@ -5,8 +5,10 @@ mod cdc_seeds; mod filesystem; mod identity_seeds; mod layout_seeds; +mod migration_seeds; mod retention_seeds; mod segment_seeds; +mod segment_store_v2_fixture; use std::error::Error; use std::fmt; @@ -70,6 +72,7 @@ pub(super) fn prepare(repository_root: &Path) -> Result<(), FuzzSeedError> { seeds.extend(cdc_seeds::seeds()?); seeds.extend(golden_protocol_seeds_from(&files)?); seeds.extend(layout_seeds::seeds(&files)?); + seeds.extend(migration_seeds::seeds(&files)?); seeds.extend(retention_seeds::seeds(&files)?); seeds.extend(segment_seeds::seeds(&files)?); files.write_seeds(&seeds) diff --git a/xtask/src/fuzz_seed_corpus/migration_seeds.rs b/xtask/src/fuzz_seed_corpus/migration_seeds.rs new file mode 100644 index 0000000..30d01b8 --- /dev/null +++ b/xtask/src/fuzz_seed_corpus/migration_seeds.rs @@ -0,0 +1,94 @@ +//! This module owns canonical store-migration record fuzz seeds. + +use super::filesystem::RepositoryFiles; +use super::segment_store_v2_fixture; +use super::{FuzzSeedError, MAX_SEED_BYTES, Seed, prefixed}; + +const FORMAT_MARKER_FIXTURE: &str = "format-marker.hex"; +const MIGRATION_INTENT_FIXTURE: &str = "migration-intent.hex"; +const MIGRATION_RECEIPT_FIXTURE: &str = "migration-receipt.hex"; + +pub(super) const FIXTURES: [(u8, &str); 3] = [ + (0, FORMAT_MARKER_FIXTURE), + (1, MIGRATION_INTENT_FIXTURE), + (2, MIGRATION_RECEIPT_FIXTURE), +]; + +pub(super) fn seeds(files: &RepositoryFiles) -> Result, FuzzSeedError> { + let [ + (marker_selector, marker_fixture), + (intent_selector, intent_fixture), + (receipt_selector, receipt_fixture), + ] = FIXTURES; + let marker = segment_store_v2_fixture::read_hex(files, marker_fixture)?; + let intent = segment_store_v2_fixture::read_hex(files, intent_fixture)?; + let receipt = segment_store_v2_fixture::read_hex(files, receipt_fixture)?; + Ok(vec![ + Seed::new( + "migration_format", + "format-marker", + prefixed(marker_selector, &marker)?, + )?, + Seed::new( + "migration_format", + "migration-intent", + prefixed(intent_selector, &intent)?, + )?, + Seed::new( + "migration_format", + "migration-receipt", + receipt_seed(receipt_selector, &marker, &intent, &receipt)?, + )?, + ]) +} + +fn receipt_seed( + selector: u8, + marker: &[u8], + intent: &[u8], + receipt: &[u8], +) -> Result, FuzzSeedError> { + let payload_bytes = marker + .len() + .checked_add(intent.len()) + .and_then(|length| length.checked_add(receipt.len())) + .ok_or_else(|| FuzzSeedError::violation("migration receipt seed length overflow"))?; + let framed_bytes = payload_bytes + .checked_add(1) + .ok_or_else(|| FuzzSeedError::violation("migration receipt seed length overflow"))?; + if framed_bytes > MAX_SEED_BYTES { + return Err(FuzzSeedError::violation( + "migration receipt seed exceeds the input bound", + )); + } + let mut payload = Vec::with_capacity(payload_bytes); + payload.extend_from_slice(marker); + payload.extend_from_slice(intent); + payload.extend_from_slice(receipt); + prefixed(selector, &payload) +} + +#[cfg(test)] +mod tests { + use super::{FuzzSeedError, MAX_SEED_BYTES, receipt_seed}; + + #[test] + fn receipt_seed_frames_dependencies_before_the_receipt() -> Result<(), FuzzSeedError> { + let seed = receipt_seed(2, b"marker", b"intent", b"receipt")?; + assert_eq!(seed, b"\x02markerintentreceipt"); + Ok(()) + } + + #[test] + fn receipt_seed_refuses_before_allocating_above_the_seed_bound() -> Result<(), FuzzSeedError> { + let oversized_marker = vec![0; MAX_SEED_BYTES]; + let Err(FuzzSeedError::Violation(message)) = receipt_seed(2, &oversized_marker, &[], &[]) + else { + return Err(FuzzSeedError::violation( + "oversized migration receipt seed was admitted", + )); + }; + assert_eq!(message, "migration receipt seed exceeds the input bound"); + Ok(()) + } +} diff --git a/xtask/src/fuzz_seed_corpus/retention_seeds.rs b/xtask/src/fuzz_seed_corpus/retention_seeds.rs index 152dc16..4f16d57 100644 --- a/xtask/src/fuzz_seed_corpus/retention_seeds.rs +++ b/xtask/src/fuzz_seed_corpus/retention_seeds.rs @@ -1,12 +1,8 @@ //! This module owns canonical retention-record fuzz seeds. -use std::path::Path; - use super::filesystem::RepositoryFiles; -use super::{FuzzSeedError, MAX_SEED_BYTES, Seed, prefixed}; -use xtask::protocol_admission::{EmptyHex, decode_lower_hex, framed_lines}; - -const SEGMENT_STORE_ROOT: &str = "conformance/segment-store/v2"; +use super::segment_store_v2_fixture; +use super::{FuzzSeedError, Seed, prefixed}; pub(super) const FIXTURES: [(u8, &str); 3] = [ (0, "one-anchor-root.hex"), @@ -20,7 +16,7 @@ pub(super) fn seeds(files: &RepositoryFiles) -> Result, FuzzSeedError> let name = fixture .strip_suffix(".hex") .ok_or_else(|| FuzzSeedError::violation("retention fixture lacks .hex suffix"))?; - let encoded = fixture_bytes(files, fixture)?; + let encoded = segment_store_v2_fixture::read_hex(files, fixture)?; seeds.push(Seed::new( "retention_format", name, @@ -29,18 +25,3 @@ pub(super) fn seeds(files: &RepositoryFiles) -> Result, FuzzSeedError> } Ok(seeds) } - -fn fixture_bytes(files: &RepositoryFiles, fixture: &'static str) -> Result, FuzzSeedError> { - let relative = Path::new(SEGMENT_STORE_ROOT).join(fixture); - let transport = files.read_bounded(&relative, MAX_SEED_BYTES)?; - let lines = framed_lines(&transport, MAX_SEED_BYTES) - .map_err(|source| FuzzSeedError::violation(format!("{fixture} framing moved: {source}")))?; - let [encoded] = lines.as_slice() else { - return Err(FuzzSeedError::violation(format!( - "{fixture} must contain exactly one hexadecimal line" - ))); - }; - decode_lower_hex(encoded, MAX_SEED_BYTES, EmptyHex::Refuse).map_err(|source| { - FuzzSeedError::violation(format!("{fixture} is not canonical hexadecimal: {source}")) - }) -} diff --git a/xtask/src/fuzz_seed_corpus/segment_store_v2_fixture.rs b/xtask/src/fuzz_seed_corpus/segment_store_v2_fixture.rs new file mode 100644 index 0000000..8788ac1 --- /dev/null +++ b/xtask/src/fuzz_seed_corpus/segment_store_v2_fixture.rs @@ -0,0 +1,27 @@ +//! This module owns bounded admission of version-2 hexadecimal seed fixtures. + +use std::path::Path; + +use super::filesystem::RepositoryFiles; +use super::{FuzzSeedError, MAX_SEED_BYTES}; +use xtask::protocol_admission::{EmptyHex, decode_lower_hex, framed_lines}; + +const SEGMENT_STORE_ROOT: &str = "conformance/segment-store/v2"; + +pub(super) fn read_hex( + files: &RepositoryFiles, + fixture: &'static str, +) -> Result, FuzzSeedError> { + let relative = Path::new(SEGMENT_STORE_ROOT).join(fixture); + let transport = files.read_bounded(&relative, MAX_SEED_BYTES)?; + let lines = framed_lines(&transport, MAX_SEED_BYTES) + .map_err(|source| FuzzSeedError::violation(format!("{fixture} framing moved: {source}")))?; + let [encoded] = lines.as_slice() else { + return Err(FuzzSeedError::violation(format!( + "{fixture} must contain exactly one hexadecimal line" + ))); + }; + decode_lower_hex(encoded, MAX_SEED_BYTES, EmptyHex::Refuse).map_err(|source| { + FuzzSeedError::violation(format!("{fixture} is not canonical hexadecimal: {source}")) + }) +} diff --git a/xtask/src/fuzz_seed_corpus/tests/materialization.rs b/xtask/src/fuzz_seed_corpus/tests/materialization.rs index 5ed813f..37dd51d 100644 --- a/xtask/src/fuzz_seed_corpus/tests/materialization.rs +++ b/xtask/src/fuzz_seed_corpus/tests/materialization.rs @@ -4,7 +4,8 @@ use std::collections::BTreeMap; use std::path::Path; use super::super::{ - FuzzSeedError, catalog_seeds, layout_seeds, prepare, retention_seeds, segment_seeds, + FuzzSeedError, catalog_seeds, layout_seeds, migration_seeds, prepare, retention_seeds, + segment_seeds, }; use crate::test_directory::TestDirectory; @@ -41,15 +42,16 @@ fn seed_preparation_materializes_the_complete_deterministic_set() copy_layout_fixtures(source_root, root)?; copy_segment_fixtures(source_root, root)?; copy_catalog_fixtures(source_root, root)?; - copy_retention_fixtures(source_root, root)?; + copy_version_two_fixtures(source_root, root)?; prepare(root)?; let corpus = root.join("fuzz/corpus"); let first = seed_contents(&corpus)?; - assert_eq!(first.len(), 43); + assert_eq!(first.len(), 46); assert_eq!(target_seed_count(&first, "catalog_format/"), 6); assert_eq!(target_seed_count(&first, "golden_protocol/"), 9); assert_eq!(target_seed_count(&first, "layout_record/"), 4); + assert_eq!(target_seed_count(&first, "migration_format/"), 3); assert_eq!(target_seed_count(&first, "retention_format/"), 3); assert_eq!(target_seed_count(&first, "segment_format/"), 8); prepare(root)?; @@ -116,24 +118,27 @@ fn copy_catalog_fixtures(source_root: &Path, root: &Path) -> Result<(), FuzzSeed Ok(()) } -fn copy_retention_fixtures(source_root: &Path, root: &Path) -> Result<(), FuzzSeedError> { +fn copy_version_two_fixtures(source_root: &Path, root: &Path) -> Result<(), FuzzSeedError> { use std::fs; let retention_directory = root.join("conformance/segment-store/v2"); fs::create_dir_all(&retention_directory).map_err(|source| { FuzzSeedError::io( - "create test retention conformance root", + "create test version-two conformance root", &retention_directory, source, ) })?; - for (_selector, fixture) in retention_seeds::FIXTURES { + let fixtures = retention_seeds::FIXTURES + .into_iter() + .chain(migration_seeds::FIXTURES); + for (_selector, fixture) in fixtures { let source_path = source_root .join("conformance/segment-store/v2") .join(fixture); let destination = retention_directory.join(fixture); fs::copy(&source_path, &destination) - .map_err(|source| FuzzSeedError::io("copy test retention", &destination, source))?; + .map_err(|source| FuzzSeedError::io("copy test version-two", &destination, source))?; } Ok(()) } diff --git a/xtask/tests/retention_store_v2_protocol_contract/parser_fuzz_laws.rs b/xtask/tests/retention_store_v2_protocol_contract/parser_fuzz_laws.rs index bc2919f..763d4db 100644 --- a/xtask/tests/retention_store_v2_protocol_contract/parser_fuzz_laws.rs +++ b/xtask/tests/retention_store_v2_protocol_contract/parser_fuzz_laws.rs @@ -29,3 +29,26 @@ fn retention_decoders_have_registered_seeded_fuzz_evidence() -> Result<(), Box Result<(), Box> { + let repository_root = Path::new(env!("CARGO_MANIFEST_DIR")) + .parent() + .ok_or("xtask manifest must have a repository parent")?; + + assert!( + repository_root + .join("fuzz/fuzz_targets/migration_format.rs") + .is_file() + ); + assert!( + repository_root + .join("xtask/src/fuzz_seed_corpus/migration_seeds.rs") + .is_file() + ); + assert!(FUZZ_MANIFEST.contains("name = \"migration_format\"")); + assert!(FUZZ_MANIFEST.contains("path = \"fuzz_targets/migration_format.rs\"")); + assert!(FUZZ_GUIDE.contains("The `migration_format` seeds")); + assert!(REQUIREMENTS.contains("`migration_format`")); + Ok(()) +} From 2c93701da16a973b8f74eb98f74cc7997de00888 Mon Sep 17 00:00:00 2001 From: James Ross Date: Thu, 30 Jul 2026 02:52:32 -0700 Subject: [PATCH 034/111] Add: Define store migration phases --- CHANGELOG.md | 4 +- .../segment-store-v2/migration-crash.md | 3 + docs/formats/segment-store-v2/requirements.md | 2 +- src/adapters/store_migration.rs | 2 + .../store_migration/migration_phase.rs | 114 ++++++++++++++++++ src/lib.rs | 15 +-- tests/store_migration_phase.rs | 101 ++++++++++++++++ 7 files changed, 231 insertions(+), 10 deletions(-) create mode 100644 src/adapters/store_migration/migration_phase.rs create mode 100644 tests/store_migration_phase.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index 6b3bb30..a6da96b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,8 +12,8 @@ after its public API and format compatibility policies are established. - Version-2 marker, migration-intent, and completion-receipt admission now binds exact catalog, predecessor, root, definition, store, empty-state, and checksum, - digest, and synchronization-mask coordinates; a seeded migration fuzz - surface drives all three exact decoders. Retention preflight combines + digest, and synchronization-mask coordinates; migration fuzzing drives all + three decoders, and `StoreMigrationPhase` freezes 21 transitions. Retention preflight combines expected-generation planning with deterministic closure verification; authority-revalidated 17-phase orchestration returns its receipt after durable cleanup. - Repository crash-matrix execution now terminates isolated writer process diff --git a/docs/formats/segment-store-v2/migration-crash.md b/docs/formats/segment-store-v2/migration-crash.md index 16127b0..b030fd8 100644 --- a/docs/formats/segment-store-v2/migration-crash.md +++ b/docs/formats/segment-store-v2/migration-crash.md @@ -102,3 +102,6 @@ Every identifier requires before, during, and after process-death evidence. prefix length. Restart must classify exact stages, canonical targets, namespace prefix, marker, receipt, and cleanup state without depending on a clock, filesystem iteration order, or file existence alone. + +`StoreMigrationPhase::ALL` freezes the 21 boundaries above in exact order. +Storage execution and process-death evidence remain unimplemented. diff --git a/docs/formats/segment-store-v2/requirements.md b/docs/formats/segment-store-v2/requirements.md index bff404f..e064fba 100644 --- a/docs/formats/segment-store-v2/requirements.md +++ b/docs/formats/segment-store-v2/requirements.md @@ -34,7 +34,7 @@ case is not evidence. | `KEEP-MIGRATION-004` | Every partial migration prefix continues idempotently under writer authority | state-machine and recovery tests | Planned in #19 | | `KEEP-MIGRATION-005` | Unknown, out-of-order, substituted, corrupt, conflicting, or changed evidence is unrecoverable ambiguity | corruption and mutation matrix | Planned in #19 | | `KEEP-MIGRATION-006` | Migration never rewrites or deletes admitted version-1 immutable bytes | byte-for-byte before/after witness | Planned in #19 | -| `KEEP-MIGRATION-007` | Process death around every intent stage, canonical link, namespace prefix, marker stage, receipt stage, cleanup, and synchronization boundary reaches a documented lawful state | `KEEP-CRASH-053..=073` crash-injection matrix | Planned in #19 | +| `KEEP-MIGRATION-007` | Process death around every intent stage, canonical link, namespace prefix, marker stage, receipt stage, cleanup, and synchronization boundary reaches a documented lawful state | ordered `StoreMigrationPhase` vocabulary in `tests/store_migration_phase.rs`; `KEEP-CRASH-053..=073` process-death matrix remains | In progress in #19 | | `KEEP-MIGRATION-008` | Version-1 admission refuses every version-2 or partial-migration artifact after migration begins | compatibility and fuzz tests | Planned in #19 | diff --git a/src/adapters/store_migration.rs b/src/adapters/store_migration.rs index 2021c2a..1ec3fbc 100644 --- a/src/adapters/store_migration.rs +++ b/src/adapters/store_migration.rs @@ -18,6 +18,7 @@ mod migration_intent_decode_error; mod migration_intent_decode_error_display; mod migration_intent_decoder; mod migration_intent_digest; +mod migration_phase; mod migration_receipt_decode_error; mod migration_receipt_decode_error_display; mod migration_receipt_decoder; @@ -40,6 +41,7 @@ pub use initial_gc_state_digest::InitialGcStateDigest; pub use initial_retention_state_digest::InitialRetentionStateDigest; pub use migration_intent_decode_error::StoreMigrationIntentDecodeError; pub use migration_intent_digest::StoreMigrationIntentDigest; +pub use migration_phase::StoreMigrationPhase; pub use migration_receipt_decode_error::StoreMigrationReceiptDecodeError; pub use migration_synchronization_mask::MigrationSynchronizationMask; pub use store_identifier::StoreIdentifier; diff --git a/src/adapters/store_migration/migration_phase.rs b/src/adapters/store_migration/migration_phase.rs new file mode 100644 index 0000000..cc14888 --- /dev/null +++ b/src/adapters/store_migration/migration_phase.rs @@ -0,0 +1,114 @@ +//! This boundary module owns exact store-migration durability phases. + +use std::fmt; + +/// Storage transition attempted by version-2 store migration. +/// +/// [`Self::ALL`] corresponds in order to `KEEP-CRASH-053` through +/// `KEEP-CRASH-073`. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum StoreMigrationPhase { + /// Write the complete canonical `migration.intent.next`. + WriteIntentStage, + /// Synchronize `migration.intent.next`. + SynchronizeIntentStage, + /// Link the synchronized intent stage to `migration.intent`. + LinkIntent, + /// Synchronize the store root after the intent link. + SynchronizeRootAfterIntent, + /// Remove the retained `migration.intent.next`. + RemoveIntentStage, + /// Synchronize the store root after intent-stage cleanup. + SynchronizeRootAfterIntentCleanup, + /// Create or exactly admit the persistent reader fence. + AdmitReaderFence, + /// Create or exactly admit the canonical version-2 directory prefix. + AdmitNamespacePrefix, + /// Synchronize the store root after namespace admission. + SynchronizeRootAfterNamespace, + /// Write the complete canonical `FORMAT.next`. + WriteMarkerStage, + /// Synchronize `FORMAT.next`. + SynchronizeMarkerStage, + /// Link the synchronized marker stage to `FORMAT`. + LinkMarker, + /// Synchronize the store root after the marker link. + SynchronizeRootAfterMarker, + /// Remove the retained `FORMAT.next`. + RemoveMarkerStage, + /// Synchronize the store root after marker-stage cleanup. + SynchronizeRootAfterMarkerCleanup, + /// Write the complete canonical `migration.receipt.next`. + WriteReceiptStage, + /// Synchronize `migration.receipt.next`. + SynchronizeReceiptStage, + /// Link the synchronized receipt stage to `migration.receipt`. + LinkReceipt, + /// Synchronize the store root after the receipt link. + SynchronizeRootAfterReceipt, + /// Remove the retained `migration.receipt.next`. + RemoveReceiptStage, + /// Synchronize the store root after receipt-stage cleanup. + SynchronizeRootAfterReceiptCleanup, +} + +impl StoreMigrationPhase { + /// Every migration phase in normative crash-boundary order. + pub const ALL: [Self; 21] = [ + Self::WriteIntentStage, + Self::SynchronizeIntentStage, + Self::LinkIntent, + Self::SynchronizeRootAfterIntent, + Self::RemoveIntentStage, + Self::SynchronizeRootAfterIntentCleanup, + Self::AdmitReaderFence, + Self::AdmitNamespacePrefix, + Self::SynchronizeRootAfterNamespace, + Self::WriteMarkerStage, + Self::SynchronizeMarkerStage, + Self::LinkMarker, + Self::SynchronizeRootAfterMarker, + Self::RemoveMarkerStage, + Self::SynchronizeRootAfterMarkerCleanup, + Self::WriteReceiptStage, + Self::SynchronizeReceiptStage, + Self::LinkReceipt, + Self::SynchronizeRootAfterReceipt, + Self::RemoveReceiptStage, + Self::SynchronizeRootAfterReceiptCleanup, + ]; +} + +impl fmt::Display for StoreMigrationPhase { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter.write_str(match self { + Self::WriteIntentStage => "migration-intent stage write", + Self::SynchronizeIntentStage => "migration-intent stage synchronization", + Self::LinkIntent => "migration-intent canonical link", + Self::SynchronizeRootAfterIntent => "store-root synchronization after intent link", + Self::RemoveIntentStage => "migration-intent stage removal", + Self::SynchronizeRootAfterIntentCleanup => { + "store-root synchronization after intent cleanup" + } + Self::AdmitReaderFence => "persistent reader-fence admission", + Self::AdmitNamespacePrefix => "canonical namespace-prefix admission", + Self::SynchronizeRootAfterNamespace => { + "store-root synchronization after namespace admission" + } + Self::WriteMarkerStage => "format-marker stage write", + Self::SynchronizeMarkerStage => "format-marker stage synchronization", + Self::LinkMarker => "format-marker canonical link", + Self::SynchronizeRootAfterMarker => "store-root synchronization after marker link", + Self::RemoveMarkerStage => "format-marker stage removal", + Self::SynchronizeRootAfterMarkerCleanup => { + "store-root synchronization after marker cleanup" + } + Self::WriteReceiptStage => "migration-receipt stage write", + Self::SynchronizeReceiptStage => "migration-receipt stage synchronization", + Self::LinkReceipt => "migration-receipt canonical link", + Self::SynchronizeRootAfterReceipt => "store-root synchronization after receipt link", + Self::RemoveReceiptStage => "migration-receipt stage removal", + Self::SynchronizeRootAfterReceiptCleanup => "final store-root synchronization", + }) + } +} diff --git a/src/lib.rs b/src/lib.rs index fb536f3..c4d86a8 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -110,13 +110,14 @@ pub use adapters::{ StoreFormatDefinitionDigest, StoreFormatMarkerDecodeError, StoreFormatMarkerDigest, StoreIdentifier, StoreInitializationError, StoreInitializationPhase, StoreInitializationReceipt, StoreInitializationStorage, StoreMigrationIntentDecodeError, - StoreMigrationIntentDigest, StoreMigrationReceiptDecodeError, StoreRootDeviceIdentity, - StoreRootFileIdentity, StoreRootMountIdentity, WriterLockAcquireError, WriterLockAcquirePhase, - admit_recovery_stage_bytes, assess_recovery_stage, classify_recovery_catalog_stage, - classify_recovery_names, classify_recovery_next_head_stage, classify_recovery_segment_stage, - execute_recovery_next_head_finalization, execute_recovery_segment_resume, - execute_recovery_stage_completion, execute_recovery_stage_discard, fingerprint_recovery_stage, - initialize_store, plan_recovery_next_head_finalization, plan_recovery_segment_resume, + StoreMigrationIntentDigest, StoreMigrationPhase, StoreMigrationReceiptDecodeError, + StoreRootDeviceIdentity, StoreRootFileIdentity, StoreRootMountIdentity, WriterLockAcquireError, + WriterLockAcquirePhase, admit_recovery_stage_bytes, assess_recovery_stage, + classify_recovery_catalog_stage, classify_recovery_names, classify_recovery_next_head_stage, + classify_recovery_segment_stage, execute_recovery_next_head_finalization, + execute_recovery_segment_resume, execute_recovery_stage_completion, + execute_recovery_stage_discard, fingerprint_recovery_stage, initialize_store, + plan_recovery_next_head_finalization, plan_recovery_segment_resume, plan_recovery_stage_completion, plan_recovery_stage_discard, publish_catalog_generation, read_recovery_inventory, }; diff --git a/tests/store_migration_phase.rs b/tests/store_migration_phase.rs new file mode 100644 index 0000000..978cad1 --- /dev/null +++ b/tests/store_migration_phase.rs @@ -0,0 +1,101 @@ +//! Ordered version-2 store-migration durability phase laws. + +use keep::StoreMigrationPhase; + +const EXPECTED: [(StoreMigrationPhase, &str); 21] = [ + ( + StoreMigrationPhase::WriteIntentStage, + "migration-intent stage write", + ), + ( + StoreMigrationPhase::SynchronizeIntentStage, + "migration-intent stage synchronization", + ), + ( + StoreMigrationPhase::LinkIntent, + "migration-intent canonical link", + ), + ( + StoreMigrationPhase::SynchronizeRootAfterIntent, + "store-root synchronization after intent link", + ), + ( + StoreMigrationPhase::RemoveIntentStage, + "migration-intent stage removal", + ), + ( + StoreMigrationPhase::SynchronizeRootAfterIntentCleanup, + "store-root synchronization after intent cleanup", + ), + ( + StoreMigrationPhase::AdmitReaderFence, + "persistent reader-fence admission", + ), + ( + StoreMigrationPhase::AdmitNamespacePrefix, + "canonical namespace-prefix admission", + ), + ( + StoreMigrationPhase::SynchronizeRootAfterNamespace, + "store-root synchronization after namespace admission", + ), + ( + StoreMigrationPhase::WriteMarkerStage, + "format-marker stage write", + ), + ( + StoreMigrationPhase::SynchronizeMarkerStage, + "format-marker stage synchronization", + ), + ( + StoreMigrationPhase::LinkMarker, + "format-marker canonical link", + ), + ( + StoreMigrationPhase::SynchronizeRootAfterMarker, + "store-root synchronization after marker link", + ), + ( + StoreMigrationPhase::RemoveMarkerStage, + "format-marker stage removal", + ), + ( + StoreMigrationPhase::SynchronizeRootAfterMarkerCleanup, + "store-root synchronization after marker cleanup", + ), + ( + StoreMigrationPhase::WriteReceiptStage, + "migration-receipt stage write", + ), + ( + StoreMigrationPhase::SynchronizeReceiptStage, + "migration-receipt stage synchronization", + ), + ( + StoreMigrationPhase::LinkReceipt, + "migration-receipt canonical link", + ), + ( + StoreMigrationPhase::SynchronizeRootAfterReceipt, + "store-root synchronization after receipt link", + ), + ( + StoreMigrationPhase::RemoveReceiptStage, + "migration-receipt stage removal", + ), + ( + StoreMigrationPhase::SynchronizeRootAfterReceiptCleanup, + "final store-root synchronization", + ), +]; + +#[test] +fn migration_phases_are_complete_ordered_and_stably_named() { + assert_eq!( + StoreMigrationPhase::ALL, + EXPECTED.map(|(phase, _name)| phase) + ); + for (phase, name) in EXPECTED { + assert_eq!(phase.to_string(), name); + } +} From 89429a5bbde841fa768547307d22ebb92a24cee9 Mon Sep 17 00:00:00 2001 From: James Ross Date: Thu, 30 Jul 2026 03:24:16 -0700 Subject: [PATCH 035/111] Add: Stream store migration inventory --- CHANGELOG.md | 3 +- .../segment-store-v2/migration-inventory.md | 6 + docs/formats/segment-store-v2/requirements.md | 2 +- src/adapters/admitted_segment.rs | 4 + src/adapters/store_migration.rs | 10 ++ .../migration_inventory_entry.rs | 51 +++++++ .../migration_inventory_entry_count.rs | 35 +++++ .../migration_inventory_entry_count_error.rs | 29 ++++ .../migration_inventory_error.rs | 65 +++++++++ .../migration_inventory_hasher.rs | 91 ++++++++++++ src/lib.rs | 4 +- tests/store_migration_inventory.rs | 135 ++++++++++++++++++ 12 files changed, 432 insertions(+), 3 deletions(-) create mode 100644 src/adapters/store_migration/migration_inventory_entry.rs create mode 100644 src/adapters/store_migration/migration_inventory_entry_count.rs create mode 100644 src/adapters/store_migration/migration_inventory_entry_count_error.rs create mode 100644 src/adapters/store_migration/migration_inventory_error.rs create mode 100644 src/adapters/store_migration/migration_inventory_hasher.rs create mode 100644 tests/store_migration_inventory.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index a6da96b..12d980e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,7 +13,8 @@ after its public API and format compatibility policies are established. - Version-2 marker, migration-intent, and completion-receipt admission now binds exact catalog, predecessor, root, definition, store, empty-state, and checksum, digest, and synchronization-mask coordinates; migration fuzzing drives all - three decoders, and `StoreMigrationPhase` freezes 21 transitions. Retention preflight combines + three decoders, streamed inventory is bounded, and `StoreMigrationPhase` + freezes 21 transitions. Retention preflight combines expected-generation planning with deterministic closure verification; authority-revalidated 17-phase orchestration returns its receipt after durable cleanup. - Repository crash-matrix execution now terminates isolated writer process diff --git a/docs/formats/segment-store-v2/migration-inventory.md b/docs/formats/segment-store-v2/migration-inventory.md index b50501e..f970abb 100644 --- a/docs/formats/segment-store-v2/migration-inventory.md +++ b/docs/formats/segment-store-v2/migration-inventory.md @@ -47,3 +47,9 @@ do not enter the digest. The exact one-segment, one-catalog input and its canonical entries are frozen in the version-2 corpus [`inventory.tsv`](../../../conformance/segment-store/v2/inventory.tsv). + +`StoreMigrationInventoryEntry` derives canonical bytes only from admitted +artifacts. `StoreMigrationInventoryHasher` requires the bounded entry count +before streaming, retains only the preceding entry, refuses duplicate or +out-of-order evidence, and reproduces the frozen digest. Capability-relative +filesystem inventory and mutation revalidation remain unimplemented. diff --git a/docs/formats/segment-store-v2/requirements.md b/docs/formats/segment-store-v2/requirements.md index e064fba..f282725 100644 --- a/docs/formats/segment-store-v2/requirements.md +++ b/docs/formats/segment-store-v2/requirements.md @@ -30,7 +30,7 @@ case is not evidence. | --- | --- | --- | --- | | `KEEP-MIGRATION-001` | Exact version-1 stores remain admitted until a durable migration artifact exists | compatibility fixtures | Planned in #19 | | `KEEP-MIGRATION-002` | Format marker, intent, and receipt have complete fixed byte tables, named domains, bounds, checksums, deterministic store identity, and exact initial-state digests | exact admission in `tests/store_format_marker.rs`, `tests/store_migration_intent.rs`, and `tests/store_migration_receipt.rs`; seeded `migration_format` fuzz target | Implemented | -| `KEEP-MIGRATION-003` | Migration revalidates version-1 head, catalog, pools, root identity, and writer authority before mutation | capability-relative integration tests | Planned in #19 | +| `KEEP-MIGRATION-003` | Migration revalidates version-1 head, catalog, pools, root identity, and writer authority before mutation | bounded canonical pool-inventory evidence in `tests/store_migration_inventory.rs`; capability-relative integration tests remain | In progress in #19 | | `KEEP-MIGRATION-004` | Every partial migration prefix continues idempotently under writer authority | state-machine and recovery tests | Planned in #19 | | `KEEP-MIGRATION-005` | Unknown, out-of-order, substituted, corrupt, conflicting, or changed evidence is unrecoverable ambiguity | corruption and mutation matrix | Planned in #19 | | `KEEP-MIGRATION-006` | Migration never rewrites or deletes admitted version-1 immutable bytes | byte-for-byte before/after witness | Planned in #19 | diff --git a/src/adapters/admitted_segment.rs b/src/adapters/admitted_segment.rs index f800819..862e8a2 100644 --- a/src/adapters/admitted_segment.rs +++ b/src/adapters/admitted_segment.rs @@ -51,6 +51,10 @@ impl<'a> AdmittedSegment<'a> { self.seal.digest() } + pub(super) const fn segment_length(&self) -> u64 { + self.seal.segment_length() + } + /// Returns a revalidating iterator over records in physical order. #[must_use] pub const fn records(&self) -> SegmentRecords<'a> { diff --git a/src/adapters/store_migration.rs b/src/adapters/store_migration.rs index 1ec3fbc..b1dbf9f 100644 --- a/src/adapters/store_migration.rs +++ b/src/adapters/store_migration.rs @@ -18,6 +18,11 @@ mod migration_intent_decode_error; mod migration_intent_decode_error_display; mod migration_intent_decoder; mod migration_intent_digest; +mod migration_inventory_entry; +mod migration_inventory_entry_count; +mod migration_inventory_entry_count_error; +mod migration_inventory_error; +mod migration_inventory_hasher; mod migration_phase; mod migration_receipt_decode_error; mod migration_receipt_decode_error_display; @@ -41,6 +46,11 @@ pub use initial_gc_state_digest::InitialGcStateDigest; pub use initial_retention_state_digest::InitialRetentionStateDigest; pub use migration_intent_decode_error::StoreMigrationIntentDecodeError; pub use migration_intent_digest::StoreMigrationIntentDigest; +pub use migration_inventory_entry::StoreMigrationInventoryEntry; +pub use migration_inventory_entry_count::StoreMigrationInventoryEntryCount; +pub use migration_inventory_entry_count_error::StoreMigrationInventoryEntryCountError; +pub use migration_inventory_error::StoreMigrationInventoryError; +pub use migration_inventory_hasher::StoreMigrationInventoryHasher; pub use migration_phase::StoreMigrationPhase; pub use migration_receipt_decode_error::StoreMigrationReceiptDecodeError; pub use migration_synchronization_mask::MigrationSynchronizationMask; diff --git a/src/adapters/store_migration/migration_inventory_entry.rs b/src/adapters/store_migration/migration_inventory_entry.rs new file mode 100644 index 0000000..1ce8bc0 --- /dev/null +++ b/src/adapters/store_migration/migration_inventory_entry.rs @@ -0,0 +1,51 @@ +//! This boundary module owns canonical migration inventory entries. + +use crate::{AdmittedCatalog, AdmittedSegment}; + +const SEGMENT_KIND: u8 = 1; +const CATALOG_KIND: u8 = 2; +const ENCODED_LENGTH: usize = 56; + +/// Canonical physical coordinate for one admitted version-1 pool artifact. +#[must_use] +#[derive(Clone, Copy, Debug, Eq, Ord, PartialEq, PartialOrd)] +pub struct StoreMigrationInventoryEntry([u8; ENCODED_LENGTH]); + +impl StoreMigrationInventoryEntry { + /// Constructs the canonical entry for one completely admitted segment. + pub const fn from_segment(segment: &AdmittedSegment<'_>) -> Self { + Self(encode( + SEGMENT_KIND, + 0, + segment.segment_length(), + segment.digest().as_bytes(), + )) + } + + /// Constructs the canonical entry for one completely admitted catalog. + pub const fn from_catalog(catalog: &AdmittedCatalog<'_, '_>) -> Self { + Self(encode( + CATALOG_KIND, + catalog.generation().get(), + catalog.length().get(), + catalog.digest().as_bytes(), + )) + } + + /// Returns the exact 56 canonical bytes. + pub const fn encoded(&self) -> &[u8; ENCODED_LENGTH] { + &self.0 + } +} + +const fn encode(kind: u8, generation: u64, length: u64, digest: &[u8; 32]) -> [u8; ENCODED_LENGTH] { + let mut encoded = [0_u8; ENCODED_LENGTH]; + let (kind_and_reserved, remainder) = encoded.split_at_mut(8); + kind_and_reserved.copy_from_slice(&[kind, 0, 0, 0, 0, 0, 0, 0]); + let (generation_bytes, remainder) = remainder.split_at_mut(8); + generation_bytes.copy_from_slice(&generation.to_be_bytes()); + let (length_bytes, digest_bytes) = remainder.split_at_mut(8); + length_bytes.copy_from_slice(&length.to_be_bytes()); + digest_bytes.copy_from_slice(digest); + encoded +} diff --git a/src/adapters/store_migration/migration_inventory_entry_count.rs b/src/adapters/store_migration/migration_inventory_entry_count.rs new file mode 100644 index 0000000..e0bc3c7 --- /dev/null +++ b/src/adapters/store_migration/migration_inventory_entry_count.rs @@ -0,0 +1,35 @@ +//! This module owns the bounded migration inventory entry count. + +use super::StoreMigrationInventoryEntryCountError; + +/// Exact number of canonical entries expected in one migration inventory. +#[must_use] +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub struct StoreMigrationInventoryEntryCount(u32); + +impl StoreMigrationInventoryEntryCount { + /// Largest number of immutable-pool entries admitted by version 2. + pub const MAXIMUM: u32 = 2_097_152; + + /// Admits one exact entry count, including an empty inventory. + /// + /// # Errors + /// + /// Returns [`StoreMigrationInventoryEntryCountError`] above + /// [`Self::MAXIMUM`]. + pub const fn new(value: u32) -> Result { + if value <= Self::MAXIMUM { + Ok(Self(value)) + } else { + Err(StoreMigrationInventoryEntryCountError::AboveMaximum { + observed: value, + maximum: Self::MAXIMUM, + }) + } + } + + /// Returns the exact admitted count. + pub const fn get(self) -> u32 { + self.0 + } +} diff --git a/src/adapters/store_migration/migration_inventory_entry_count_error.rs b/src/adapters/store_migration/migration_inventory_entry_count_error.rs new file mode 100644 index 0000000..6f44336 --- /dev/null +++ b/src/adapters/store_migration/migration_inventory_entry_count_error.rs @@ -0,0 +1,29 @@ +//! This boundary module owns migration inventory entry-count refusals. + +use std::error::Error; +use std::fmt; + +/// Failure to admit a bounded migration inventory entry count. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum StoreMigrationInventoryEntryCountError { + /// The requested count exceeds the immutable protocol maximum. + AboveMaximum { + /// Count supplied by the caller. + observed: u32, + /// Largest count admitted by the protocol. + maximum: u32, + }, +} + +impl fmt::Display for StoreMigrationInventoryEntryCountError { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::AboveMaximum { observed, maximum } => write!( + formatter, + "migration inventory entry count {observed} exceeds maximum {maximum}" + ), + } + } +} + +impl Error for StoreMigrationInventoryEntryCountError {} diff --git a/src/adapters/store_migration/migration_inventory_error.rs b/src/adapters/store_migration/migration_inventory_error.rs new file mode 100644 index 0000000..e1d35b7 --- /dev/null +++ b/src/adapters/store_migration/migration_inventory_error.rs @@ -0,0 +1,65 @@ +//! This boundary module owns streamed migration inventory refusals. + +use std::error::Error; +use std::fmt; + +use super::{StoreMigrationInventoryEntry, StoreMigrationInventoryEntryCount}; + +/// Failure to stream one bounded canonical migration inventory. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum StoreMigrationInventoryError { + /// An entry would exceed the declared inventory count. + EntryCountExceeded { + /// Exact count declared before hashing. + expected: StoreMigrationInventoryEntryCount, + /// Count that the attempted entry would produce. + observed: u32, + }, + /// The same canonical entry appeared more than once. + Duplicate { + /// Repeated canonical entry. + entry: StoreMigrationInventoryEntry, + }, + /// Canonical entry order moved backward. + OutOfOrder { + /// Last entry admitted before the refusal. + previous: StoreMigrationInventoryEntry, + /// Entry observed after `previous`. + observed: StoreMigrationInventoryEntry, + }, + /// Finalization observed fewer entries than declared. + Incomplete { + /// Exact count declared before hashing. + expected: StoreMigrationInventoryEntryCount, + /// Exact number of entries admitted. + observed: u32, + }, + /// Checked observed-count arithmetic overflowed. + EntryCountOverflow, +} + +impl fmt::Display for StoreMigrationInventoryError { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::EntryCountExceeded { expected, observed } => write!( + formatter, + "migration inventory expected {} entries but observed at least {observed}", + expected.get() + ), + Self::Duplicate { .. } => formatter.write_str("duplicate migration inventory entry"), + Self::OutOfOrder { .. } => { + formatter.write_str("migration inventory entries are out of canonical order") + } + Self::Incomplete { expected, observed } => write!( + formatter, + "migration inventory expected {} entries but observed {observed}", + expected.get() + ), + Self::EntryCountOverflow => { + formatter.write_str("migration inventory entry count overflow") + } + } + } +} + +impl Error for StoreMigrationInventoryError {} diff --git a/src/adapters/store_migration/migration_inventory_hasher.rs b/src/adapters/store_migration/migration_inventory_hasher.rs new file mode 100644 index 0000000..f802e35 --- /dev/null +++ b/src/adapters/store_migration/migration_inventory_hasher.rs @@ -0,0 +1,91 @@ +//! This boundary module owns streamed canonical migration inventory identity. + +use super::{ + ImmutablePoolInventoryDigest, StoreMigrationInventoryEntry, StoreMigrationInventoryEntryCount, + StoreMigrationInventoryError, +}; + +const DOMAIN: &[u8] = b"keep.store-v1-pool-inventory/v2\0"; + +/// In-progress bounded digest over one declared canonical pool inventory. +/// +/// Entries must be supplied in complete canonical byte order. The hasher +/// retains only the preceding entry and never materializes the complete +/// encoded inventory. +#[must_use] +pub struct StoreMigrationInventoryHasher { + expected: StoreMigrationInventoryEntryCount, + observed: u32, + previous: Option, + hasher: blake3::Hasher, +} + +impl StoreMigrationInventoryHasher { + /// Begins one inventory whose exact count is known before entry streaming. + pub fn new(expected: StoreMigrationInventoryEntryCount) -> Self { + let mut hasher = blake3::Hasher::new(); + hasher.update(DOMAIN); + hasher.update(&expected.get().to_be_bytes()); + Self { + expected, + observed: 0, + previous: None, + hasher, + } + } + + /// Admits and hashes the next exact canonical entry. + /// + /// # Errors + /// + /// Returns [`StoreMigrationInventoryError`] for count excess, a duplicate, + /// noncanonical order, or checked count overflow. + pub fn push( + &mut self, + entry: StoreMigrationInventoryEntry, + ) -> Result<(), StoreMigrationInventoryError> { + let observed = self + .observed + .checked_add(1) + .ok_or(StoreMigrationInventoryError::EntryCountOverflow)?; + if observed > self.expected.get() { + return Err(StoreMigrationInventoryError::EntryCountExceeded { + expected: self.expected, + observed, + }); + } + if let Some(previous) = self.previous { + if entry == previous { + return Err(StoreMigrationInventoryError::Duplicate { entry }); + } + if entry < previous { + return Err(StoreMigrationInventoryError::OutOfOrder { + previous, + observed: entry, + }); + } + } + self.hasher.update(entry.encoded()); + self.previous = Some(entry); + self.observed = observed; + Ok(()) + } + + /// Finalizes only after the declared number of entries was admitted. + /// + /// # Errors + /// + /// Returns [`StoreMigrationInventoryError::Incomplete`] when fewer entries + /// were supplied than declared. + pub fn finish(self) -> Result { + if self.observed != self.expected.get() { + return Err(StoreMigrationInventoryError::Incomplete { + expected: self.expected, + observed: self.observed, + }); + } + Ok(ImmutablePoolInventoryDigest::from_admitted( + *self.hasher.finalize().as_bytes(), + )) + } +} diff --git a/src/lib.rs b/src/lib.rs index c4d86a8..bacb7a1 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -110,7 +110,9 @@ pub use adapters::{ StoreFormatDefinitionDigest, StoreFormatMarkerDecodeError, StoreFormatMarkerDigest, StoreIdentifier, StoreInitializationError, StoreInitializationPhase, StoreInitializationReceipt, StoreInitializationStorage, StoreMigrationIntentDecodeError, - StoreMigrationIntentDigest, StoreMigrationPhase, StoreMigrationReceiptDecodeError, + StoreMigrationIntentDigest, StoreMigrationInventoryEntry, StoreMigrationInventoryEntryCount, + StoreMigrationInventoryEntryCountError, StoreMigrationInventoryError, + StoreMigrationInventoryHasher, StoreMigrationPhase, StoreMigrationReceiptDecodeError, StoreRootDeviceIdentity, StoreRootFileIdentity, StoreRootMountIdentity, WriterLockAcquireError, WriterLockAcquirePhase, admit_recovery_stage_bytes, assess_recovery_stage, classify_recovery_catalog_stage, classify_recovery_names, classify_recovery_next_head_stage, diff --git a/tests/store_migration_inventory.rs b/tests/store_migration_inventory.rs new file mode 100644 index 0000000..ffe49fc --- /dev/null +++ b/tests/store_migration_inventory.rs @@ -0,0 +1,135 @@ +//! Canonical version-1 immutable-pool migration inventory laws. + +mod support; + +use std::error::Error; + +use keep::{ + AdmittedSegment, ChecksummedCatalog, LayoutEntryLimit, SegmentReadPolicy, SegmentRecordLimit, + StoreMigrationInventoryEntry, StoreMigrationInventoryEntryCount, + StoreMigrationInventoryEntryCountError, StoreMigrationInventoryError, + StoreMigrationInventoryHasher, +}; +use support::decode_hex; + +const SEGMENT: &str = include_str!("../conformance/segment-store/v1/one-zero-segment.hex"); +const CATALOG: &str = include_str!("../conformance/segment-store/v1/one-zero-catalog.hex"); +const SEGMENT_ENTRY: &str = concat!( + "0100000000000000", + "0000000000000000", + "0000000000000151", + "b7542dced2ab770894a14d1d04b066e3a899942602c5986d35ba6df6c1a35cfc", +); +const CATALOG_ENTRY: &str = concat!( + "0200000000000000", + "0000000000000001", + "0000000000000160", + "04b82519b0399baefd0b9c0f32a871052e4c47e3a00226ab03b21661470f7320", +); +const INVENTORY_DIGEST: &str = "40bf5d49c34847ac9cf46a256f343cee80cd980d1405d2dd02ceff8f58d674f9"; + +#[test] +fn frozen_inventory_entries_and_digest_are_exact() -> Result<(), Box> { + let (segment, catalog) = frozen_entries()?; + assert_eq!(segment.encoded().as_slice(), decode_hex(SEGMENT_ENTRY)?); + assert_eq!(catalog.encoded().as_slice(), decode_hex(CATALOG_ENTRY)?); + + let count = StoreMigrationInventoryEntryCount::new(2)?; + let mut inventory = StoreMigrationInventoryHasher::new(count); + inventory.push(segment)?; + inventory.push(catalog)?; + let digest = inventory.finish()?; + assert_eq!(digest.as_bytes().as_slice(), decode_hex(INVENTORY_DIGEST)?); + Ok(()) +} + +#[test] +fn inventory_refuses_duplicate_and_out_of_order_entries() -> Result<(), Box> { + let (segment, catalog) = frozen_entries()?; + let count = StoreMigrationInventoryEntryCount::new(2)?; + + let mut duplicate = StoreMigrationInventoryHasher::new(count); + duplicate.push(segment)?; + assert_eq!( + duplicate.push(segment), + Err(StoreMigrationInventoryError::Duplicate { entry: segment }) + ); + + let mut out_of_order = StoreMigrationInventoryHasher::new(count); + out_of_order.push(catalog)?; + assert_eq!( + out_of_order.push(segment), + Err(StoreMigrationInventoryError::OutOfOrder { + previous: catalog, + observed: segment, + }) + ); + Ok(()) +} + +#[test] +fn inventory_refuses_count_overrun_and_incomplete_finalization() -> Result<(), Box> { + let (segment, catalog) = frozen_entries()?; + let one = StoreMigrationInventoryEntryCount::new(1)?; + let two = StoreMigrationInventoryEntryCount::new(2)?; + + let mut overrun = StoreMigrationInventoryHasher::new(one); + overrun.push(segment)?; + assert_eq!( + overrun.push(catalog), + Err(StoreMigrationInventoryError::EntryCountExceeded { + expected: one, + observed: 2, + }) + ); + + let mut incomplete = StoreMigrationInventoryHasher::new(two); + incomplete.push(segment)?; + assert_eq!( + incomplete.finish(), + Err(StoreMigrationInventoryError::Incomplete { + expected: two, + observed: 1, + }) + ); + Ok(()) +} + +#[test] +fn inventory_count_has_the_exact_protocol_bound() { + assert_eq!( + StoreMigrationInventoryEntryCount::new(0).map(StoreMigrationInventoryEntryCount::get), + Ok(0) + ); + assert_eq!( + StoreMigrationInventoryEntryCount::new(StoreMigrationInventoryEntryCount::MAXIMUM) + .map(StoreMigrationInventoryEntryCount::get), + Ok(StoreMigrationInventoryEntryCount::MAXIMUM) + ); + assert_eq!( + StoreMigrationInventoryEntryCount::new(2_097_153), + Err(StoreMigrationInventoryEntryCountError::AboveMaximum { + observed: 2_097_153, + maximum: 2_097_152, + }) + ); +} + +fn frozen_entries() +-> Result<(StoreMigrationInventoryEntry, StoreMigrationInventoryEntry), Box> { + let segment_bytes = fixture(SEGMENT)?; + let catalog_bytes = fixture(CATALOG)?; + let segment = AdmittedSegment::decode(&segment_bytes, maximum_policy())?; + let segment_entry = StoreMigrationInventoryEntry::from_segment(&segment); + let catalog = ChecksummedCatalog::decode(&catalog_bytes)?.admit(&[segment])?; + let catalog_entry = StoreMigrationInventoryEntry::from_catalog(&catalog); + Ok((segment_entry, catalog_entry)) +} + +const fn maximum_policy() -> SegmentReadPolicy { + SegmentReadPolicy::new(SegmentRecordLimit::MAXIMUM, LayoutEntryLimit::MAXIMUM) +} + +fn fixture(hex: &str) -> Result, Box> { + decode_hex(hex.strip_suffix('\n').ok_or("fixture must end in one LF")?).map_err(Into::into) +} From 72b09182254f5b549d94e02222787713f3acaec7 Mon Sep 17 00:00:00 2001 From: James Ross Date: Thu, 30 Jul 2026 03:40:33 -0700 Subject: [PATCH 036/111] Add: Construct store migration intents --- CHANGELOG.md | 2 +- docs/formats/segment-store-v2/recovery.md | 2 +- docs/formats/segment-store-v2/requirements.md | 2 +- src/adapters/store_migration.rs | 4 + .../canonical_migration_intent.rs | 67 +++++++++++++ .../migration_intent_decoder.rs | 79 ++++++--------- .../migration_intent_encoder.rs | 84 ++++++++++++++++ .../migration_intent_format.rs | 61 ++++++++++++ src/lib.rs | 21 ++-- tests/store_migration_intent_encoding.rs | 95 +++++++++++++++++++ 10 files changed, 352 insertions(+), 65 deletions(-) create mode 100644 src/adapters/store_migration/canonical_migration_intent.rs create mode 100644 src/adapters/store_migration/migration_intent_encoder.rs create mode 100644 src/adapters/store_migration/migration_intent_format.rs create mode 100644 tests/store_migration_intent_encoding.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index 12d980e..94ceadd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,7 +10,7 @@ after its public API and format compatibility policies are established. ### Changed -- Version-2 marker, migration-intent, and completion-receipt admission now binds +- Version-2 marker, canonical migration-intent construction, and record admission bind exact catalog, predecessor, root, definition, store, empty-state, and checksum, digest, and synchronization-mask coordinates; migration fuzzing drives all three decoders, streamed inventory is bounded, and `StoreMigrationPhase` diff --git a/docs/formats/segment-store-v2/recovery.md b/docs/formats/segment-store-v2/recovery.md index f631c23..b652509 100644 --- a/docs/formats/segment-store-v2/recovery.md +++ b/docs/formats/segment-store-v2/recovery.md @@ -61,7 +61,7 @@ corpus `definition.tsv` bytes. The format-marker digest is BLAKE3-256 of `keep.store-format-marker/v2\0` followed by all 96 marker bytes. `CanonicalStoreFormatMarker` produces the registered marker; `AdmittedStoreFormatMarker` admits its framing, checksum, definition, and namespace bound. -`AdmittedStoreMigrationIntent` admits intent integrity and identity; `AdmittedStoreMigrationReceipt` binds that intent, the marker, registered empty states, and all synchronization bits. +`CanonicalStoreMigrationIntent` reproduces intent bytes from typed coordinates; `AdmittedStoreMigrationIntent` admits integrity and identity; `AdmittedStoreMigrationReceipt` binds the intent, marker, empty states, and synchronization bits. These record boundaries do not prove the named live inventory, physical root, store version, or execution of filesystem migration. diff --git a/docs/formats/segment-store-v2/requirements.md b/docs/formats/segment-store-v2/requirements.md index f282725..072dc6c 100644 --- a/docs/formats/segment-store-v2/requirements.md +++ b/docs/formats/segment-store-v2/requirements.md @@ -29,7 +29,7 @@ case is not evidence. | ID | Requirement | Evidence | Status | | --- | --- | --- | --- | | `KEEP-MIGRATION-001` | Exact version-1 stores remain admitted until a durable migration artifact exists | compatibility fixtures | Planned in #19 | -| `KEEP-MIGRATION-002` | Format marker, intent, and receipt have complete fixed byte tables, named domains, bounds, checksums, deterministic store identity, and exact initial-state digests | exact admission in `tests/store_format_marker.rs`, `tests/store_migration_intent.rs`, and `tests/store_migration_receipt.rs`; seeded `migration_format` fuzz target | Implemented | +| `KEEP-MIGRATION-002` | Format marker, intent, and receipt have complete fixed byte tables, named domains, bounds, checksums, deterministic store identity, and exact initial-state digests | exact admission in `tests/store_format_marker.rs`, `tests/store_migration_intent.rs`, and `tests/store_migration_receipt.rs`; canonical intent construction in `tests/store_migration_intent_encoding.rs`; seeded `migration_format` fuzz target | Implemented | | `KEEP-MIGRATION-003` | Migration revalidates version-1 head, catalog, pools, root identity, and writer authority before mutation | bounded canonical pool-inventory evidence in `tests/store_migration_inventory.rs`; capability-relative integration tests remain | In progress in #19 | | `KEEP-MIGRATION-004` | Every partial migration prefix continues idempotently under writer authority | state-machine and recovery tests | Planned in #19 | | `KEEP-MIGRATION-005` | Unknown, out-of-order, substituted, corrupt, conflicting, or changed evidence is unrecoverable ambiguity | corruption and mutation matrix | Planned in #19 | diff --git a/src/adapters/store_migration.rs b/src/adapters/store_migration.rs index b1dbf9f..ee2ab13 100644 --- a/src/adapters/store_migration.rs +++ b/src/adapters/store_migration.rs @@ -4,6 +4,7 @@ mod admitted_format_marker; mod admitted_migration_intent; mod admitted_migration_receipt; mod canonical_format_marker; +mod canonical_migration_intent; mod empty_disposition_set_digest; mod format_definition_digest; mod format_marker_decode_error; @@ -18,6 +19,8 @@ mod migration_intent_decode_error; mod migration_intent_decode_error_display; mod migration_intent_decoder; mod migration_intent_digest; +mod migration_intent_encoder; +mod migration_intent_format; mod migration_inventory_entry; mod migration_inventory_entry_count; mod migration_inventory_entry_count_error; @@ -37,6 +40,7 @@ pub use admitted_format_marker::AdmittedStoreFormatMarker; pub use admitted_migration_intent::AdmittedStoreMigrationIntent; pub use admitted_migration_receipt::AdmittedStoreMigrationReceipt; pub use canonical_format_marker::CanonicalStoreFormatMarker; +pub use canonical_migration_intent::CanonicalStoreMigrationIntent; pub use empty_disposition_set_digest::EmptyDispositionSetDigest; pub use format_definition_digest::StoreFormatDefinitionDigest; pub use format_marker_decode_error::StoreFormatMarkerDecodeError; diff --git a/src/adapters/store_migration/canonical_migration_intent.rs b/src/adapters/store_migration/canonical_migration_intent.rs new file mode 100644 index 0000000..f6e2303 --- /dev/null +++ b/src/adapters/store_migration/canonical_migration_intent.rs @@ -0,0 +1,67 @@ +//! This boundary module owns canonical owned migration-intent bytes. + +use super::{ + ImmutablePoolInventoryDigest, StoreIdentifier, StoreMigrationIntentDigest, + StoreRootDeviceIdentity, StoreRootFileIdentity, StoreRootMountIdentity, + migration_intent_encoder, migration_intent_format, +}; +use crate::CatalogSnapshot; + +/// Owned canonical version-2 store-migration intent. +/// +/// Construction preserves admitted catalog coordinates and serializes the +/// supplied inventory and physical-root coordinates. It does not prove that +/// the inventory or physical root remains current. +#[must_use] +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct CanonicalStoreMigrationIntent { + encoded: [u8; migration_intent_format::ENCODED_LENGTH], + digest: StoreMigrationIntentDigest, + store_identifier: StoreIdentifier, +} + +impl CanonicalStoreMigrationIntent { + /// Constructs one canonical intent from typed migration coordinates. + pub fn from_snapshot( + snapshot: &CatalogSnapshot<'_, '_, '_>, + inventory_digest: ImmutablePoolInventoryDigest, + root_device_identity: StoreRootDeviceIdentity, + root_mount_identity: StoreRootMountIdentity, + root_file_identity: StoreRootFileIdentity, + ) -> Self { + migration_intent_encoder::encode( + snapshot, + inventory_digest, + root_device_identity, + root_mount_identity, + root_file_identity, + ) + } + + /// Returns the exact canonical intent bytes. + pub const fn encoded(&self) -> &[u8] { + &self.encoded + } + + /// Returns the domain-separated identity of all intent bytes. + pub const fn digest(&self) -> StoreMigrationIntentDigest { + self.digest + } + + /// Returns the deterministic logical store identity. + pub const fn store_identifier(&self) -> StoreIdentifier { + self.store_identifier + } + + pub(super) const fn admitted( + encoded: [u8; migration_intent_format::ENCODED_LENGTH], + digest: StoreMigrationIntentDigest, + store_identifier: StoreIdentifier, + ) -> Self { + Self { + encoded, + digest, + store_identifier, + } + } +} diff --git a/src/adapters/store_migration/migration_intent_decoder.rs b/src/adapters/store_migration/migration_intent_decoder.rs index 7dabc5c..7eeb34e 100644 --- a/src/adapters/store_migration/migration_intent_decoder.rs +++ b/src/adapters/store_migration/migration_intent_decoder.rs @@ -1,25 +1,17 @@ //! This boundary module owns store-migration intent decoding order. use super::admitted_migration_intent::StoreMigrationIntentFields; +use super::migration_intent_format::{self as format, StoreIdentifierFields}; use super::migration_record_bytes::{ read_array, read_u16, read_u32, read_u64, require_length, wrong_length, }; use super::{ AdmittedStoreMigrationIntent, ImmutablePoolInventoryDigest, StoreFormatDefinitionDigest, - StoreIdentifier, StoreMigrationIntentDecodeError, StoreMigrationIntentDigest, - StoreRootDeviceIdentity, StoreRootFileIdentity, StoreRootMountIdentity, + StoreIdentifier, StoreMigrationIntentDecodeError, StoreRootDeviceIdentity, + StoreRootFileIdentity, StoreRootMountIdentity, }; use crate::{CatalogDigest, CatalogGeneration, CatalogLength}; -const CHECKSUM_OFFSET: usize = 224; -const MAGIC: [u8; 16] = *b"KEEP:MIG:INT2\0\0\0"; -const VERSION: u16 = 2; -const RECORD_LENGTH: u16 = 256; -const CHECKSUM_DOMAIN: &[u8] = b"keep.store-migration-intent-checksum/v2\0"; -const DIGEST_DOMAIN: &[u8] = b"keep.store-migration-intent/v2\0"; -const STORE_IDENTIFIER_DOMAIN: &[u8] = b"keep.store-identifier/v2\0"; -const ZERO_DIGEST: [u8; 32] = [0; 32]; - pub(super) fn decode( encoded: &[u8], ) -> Result, StoreMigrationIntentDecodeError> { @@ -47,26 +39,26 @@ pub(super) fn decode( Ok(AdmittedStoreMigrationIntent::admitted( encoded, fields, - digest(encoded), + format::digest(encoded), )) } fn validate_fixed_fields(encoded: &[u8]) -> Result<(), StoreMigrationIntentDecodeError> { let magic = read_array(encoded, 0)?; - if magic != MAGIC { + if magic != format::MAGIC { return Err(StoreMigrationIntentDecodeError::InvalidMagic { observed: magic }); } let version = read_u16(encoded, 16)?; - if version != VERSION { + if version != format::VERSION { return Err(StoreMigrationIntentDecodeError::UnsupportedVersion { - expected: VERSION, + expected: format::VERSION, observed: version, }); } let record_length = read_u16(encoded, 18)?; - if record_length != RECORD_LENGTH { + if record_length != format::RECORD_LENGTH { return Err(StoreMigrationIntentDecodeError::InvalidRecordLength { - expected: RECORD_LENGTH, + expected: format::RECORD_LENGTH, observed: record_length, }); } @@ -79,10 +71,10 @@ fn validate_fixed_fields(encoded: &[u8]) -> Result<(), StoreMigrationIntentDecod fn verify_checksum(encoded: &[u8]) -> Result<(), StoreMigrationIntentDecodeError> { let preimage = encoded - .get(..CHECKSUM_OFFSET) + .get(..format::CHECKSUM_OFFSET) .ok_or_else(|| wrong_length(encoded))?; - let observed = read_array(encoded, CHECKSUM_OFFSET)?; - let expected = hash(CHECKSUM_DOMAIN, &[preimage]); + let observed = read_array(encoded, format::CHECKSUM_OFFSET)?; + let expected = format::checksum(preimage); if observed == expected { Ok(()) } else { @@ -111,13 +103,13 @@ fn read_predecessor( observed: [u8; 32], ) -> Result, StoreMigrationIntentDecodeError> { if generation.get() == 1 { - return if observed == ZERO_DIGEST { + return if observed == format::ZERO_DIGEST { Ok(None) } else { Err(StoreMigrationIntentDecodeError::NonZeroInitialPredecessor { observed }) }; } - if observed == ZERO_DIGEST { + if observed == format::ZERO_DIGEST { return Err( StoreMigrationIntentDecodeError::MissingSuccessorPredecessor { generation: generation.get(), @@ -144,38 +136,21 @@ fn read_definition_digest( fn verify_store_identifier( fields: &StoreMigrationIntentFields, ) -> Result<(), StoreMigrationIntentDecodeError> { - let predecessor = fields - .predecessor_catalog_digest - .as_ref() - .map_or(&ZERO_DIGEST, CatalogDigest::as_bytes); - let expected = hash( - STORE_IDENTIFIER_DOMAIN, - &[ - &fields.catalog_generation.get().to_be_bytes(), - &fields.catalog_length.get().to_be_bytes(), - fields.catalog_digest.as_bytes(), - predecessor, - fields.inventory_digest.as_bytes(), - fields.target_definition_digest.as_bytes(), - ], - ); + let expected = format::store_identifier(&StoreIdentifierFields { + catalog_generation: fields.catalog_generation, + catalog_length: fields.catalog_length, + catalog_digest: fields.catalog_digest, + predecessor_catalog_digest: fields.predecessor_catalog_digest, + inventory_digest: fields.inventory_digest, + target_definition_digest: fields.target_definition_digest, + }); let observed = *fields.store_identifier.as_bytes(); - if observed == expected { + if observed == *expected.as_bytes() { Ok(()) } else { - Err(StoreMigrationIntentDecodeError::StoreIdentifierMismatch { expected, observed }) - } -} - -fn digest(encoded: &[u8]) -> StoreMigrationIntentDigest { - StoreMigrationIntentDigest::from_hash(hash(DIGEST_DOMAIN, &[encoded])) -} - -fn hash(domain: &[u8], fields: &[&[u8]]) -> [u8; 32] { - let mut hasher = blake3::Hasher::new(); - hasher.update(domain); - for field in fields { - hasher.update(field); + Err(StoreMigrationIntentDecodeError::StoreIdentifierMismatch { + expected: *expected.as_bytes(), + observed, + }) } - *hasher.finalize().as_bytes() } diff --git a/src/adapters/store_migration/migration_intent_encoder.rs b/src/adapters/store_migration/migration_intent_encoder.rs new file mode 100644 index 0000000..a461655 --- /dev/null +++ b/src/adapters/store_migration/migration_intent_encoder.rs @@ -0,0 +1,84 @@ +//! This boundary module owns canonical migration-intent encoding. + +use super::migration_intent_format::StoreIdentifierFields; +use super::{ + CanonicalStoreMigrationIntent, ImmutablePoolInventoryDigest, StoreFormatDefinitionDigest, + StoreIdentifier, StoreRootDeviceIdentity, StoreRootFileIdentity, StoreRootMountIdentity, + migration_intent_format as format, +}; +use crate::CatalogSnapshot; + +#[derive(Clone, Copy)] +struct RootIdentities { + device: StoreRootDeviceIdentity, + mount: StoreRootMountIdentity, + file: StoreRootFileIdentity, +} + +pub(super) fn encode( + snapshot: &CatalogSnapshot<'_, '_, '_>, + inventory_digest: ImmutablePoolInventoryDigest, + root_device_identity: StoreRootDeviceIdentity, + root_mount_identity: StoreRootMountIdentity, + root_file_identity: StoreRootFileIdentity, +) -> CanonicalStoreMigrationIntent { + let fields = StoreIdentifierFields { + catalog_generation: snapshot.generation(), + catalog_length: snapshot.catalog_length(), + catalog_digest: snapshot.catalog_digest(), + predecessor_catalog_digest: snapshot.previous_catalog_digest(), + inventory_digest, + target_definition_digest: StoreFormatDefinitionDigest::VERSION_TWO, + }; + let roots = RootIdentities { + device: root_device_identity, + mount: root_mount_identity, + file: root_file_identity, + }; + let store_identifier = format::store_identifier(&fields); + let mut encoded = [0_u8; format::ENCODED_LENGTH]; + let (preimage, checksum_slot) = encoded.split_at_mut(format::CHECKSUM_OFFSET); + write_preimage(preimage, &fields, roots, store_identifier); + checksum_slot.copy_from_slice(&format::checksum(preimage)); + let digest = format::digest(&encoded); + CanonicalStoreMigrationIntent::admitted(encoded, digest, store_identifier) +} + +fn write_preimage( + output: &mut [u8], + fields: &StoreIdentifierFields, + roots: RootIdentities, + store_identifier: StoreIdentifier, +) { + let (magic, output) = output.split_at_mut(16); + magic.copy_from_slice(&format::MAGIC); + let (version, output) = output.split_at_mut(2); + version.copy_from_slice(&format::VERSION.to_be_bytes()); + let (record_length, output) = output.split_at_mut(2); + record_length.copy_from_slice(&format::RECORD_LENGTH.to_be_bytes()); + let (flags, output) = output.split_at_mut(4); + flags.copy_from_slice(&0_u32.to_be_bytes()); + let (generation, output) = output.split_at_mut(8); + generation.copy_from_slice(&fields.catalog_generation.get().to_be_bytes()); + let (catalog_length, output) = output.split_at_mut(8); + catalog_length.copy_from_slice(&fields.catalog_length.get().to_be_bytes()); + let (catalog_digest, output) = output.split_at_mut(32); + catalog_digest.copy_from_slice(fields.catalog_digest.as_bytes()); + let predecessor = fields + .predecessor_catalog_digest + .as_ref() + .map_or(&format::ZERO_DIGEST, crate::CatalogDigest::as_bytes); + let (predecessor_digest, output) = output.split_at_mut(32); + predecessor_digest.copy_from_slice(predecessor); + let (inventory_digest, output) = output.split_at_mut(32); + inventory_digest.copy_from_slice(fields.inventory_digest.as_bytes()); + let (device_identity, output) = output.split_at_mut(8); + device_identity.copy_from_slice(&roots.device.get().to_be_bytes()); + let (mount_identity, output) = output.split_at_mut(8); + mount_identity.copy_from_slice(&roots.mount.get().to_be_bytes()); + let (file_identity, output) = output.split_at_mut(8); + file_identity.copy_from_slice(&roots.file.get().to_be_bytes()); + let (definition_digest, store_identifier_slot) = output.split_at_mut(32); + definition_digest.copy_from_slice(fields.target_definition_digest.as_bytes()); + store_identifier_slot.copy_from_slice(store_identifier.as_bytes()); +} diff --git a/src/adapters/store_migration/migration_intent_format.rs b/src/adapters/store_migration/migration_intent_format.rs new file mode 100644 index 0000000..0077bcb --- /dev/null +++ b/src/adapters/store_migration/migration_intent_format.rs @@ -0,0 +1,61 @@ +//! This boundary module owns shared migration-intent format identity. + +use super::{ + ImmutablePoolInventoryDigest, StoreFormatDefinitionDigest, StoreIdentifier, + StoreMigrationIntentDigest, +}; +use crate::{CatalogDigest, CatalogGeneration, CatalogLength}; + +pub(super) const CHECKSUM_OFFSET: usize = 224; +pub(super) const ENCODED_LENGTH: usize = 256; +pub(super) const MAGIC: [u8; 16] = *b"KEEP:MIG:INT2\0\0\0"; +pub(super) const RECORD_LENGTH: u16 = 256; +pub(super) const VERSION: u16 = 2; +const CHECKSUM_DOMAIN: &[u8] = b"keep.store-migration-intent-checksum/v2\0"; +const DIGEST_DOMAIN: &[u8] = b"keep.store-migration-intent/v2\0"; +const STORE_IDENTIFIER_DOMAIN: &[u8] = b"keep.store-identifier/v2\0"; +pub(super) const ZERO_DIGEST: [u8; 32] = [0; 32]; + +pub(super) struct StoreIdentifierFields { + pub(super) catalog_generation: CatalogGeneration, + pub(super) catalog_length: CatalogLength, + pub(super) catalog_digest: CatalogDigest, + pub(super) predecessor_catalog_digest: Option, + pub(super) inventory_digest: ImmutablePoolInventoryDigest, + pub(super) target_definition_digest: StoreFormatDefinitionDigest, +} + +pub(super) fn checksum(preimage: &[u8]) -> [u8; 32] { + hash(CHECKSUM_DOMAIN, &[preimage]) +} + +pub(super) fn digest(encoded: &[u8]) -> StoreMigrationIntentDigest { + StoreMigrationIntentDigest::from_hash(hash(DIGEST_DOMAIN, &[encoded])) +} + +pub(super) fn store_identifier(fields: &StoreIdentifierFields) -> StoreIdentifier { + let predecessor = fields + .predecessor_catalog_digest + .as_ref() + .map_or(&ZERO_DIGEST, CatalogDigest::as_bytes); + StoreIdentifier::from_hash(hash( + STORE_IDENTIFIER_DOMAIN, + &[ + &fields.catalog_generation.get().to_be_bytes(), + &fields.catalog_length.get().to_be_bytes(), + fields.catalog_digest.as_bytes(), + predecessor, + fields.inventory_digest.as_bytes(), + fields.target_definition_digest.as_bytes(), + ], + )) +} + +fn hash(domain: &[u8], fields: &[&[u8]]) -> [u8; 32] { + let mut hasher = blake3::Hasher::new(); + hasher.update(domain); + for field in fields { + hasher.update(field); + } + *hasher.finalize().as_bytes() +} diff --git a/src/lib.rs b/src/lib.rs index bacb7a1..0f49f61 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -58,16 +58,17 @@ pub use adapters::{ AdmittedCatalog, AdmittedRecoveryStageBytes, AdmittedSegment, AdmittedSegmentRecord, AdmittedStoreFormatMarker, AdmittedStoreMigrationIntent, AdmittedStoreMigrationReceipt, BlobIdBinaryParseError, BlobIdTextParseError, CanonicalCatalog, CanonicalLayoutRecord, - CanonicalPublicationHead, CanonicalStoreFormatMarker, CatalogAdmissionError, - CatalogAllocationPhase, CatalogDecodeError, CatalogEncodeError, CatalogEntryDecodeError, - CatalogPublicationError, CatalogPublicationExpectation, CatalogPublicationOutcome, - CatalogPublicationPhase, CatalogPublicationReadiness, CatalogPublicationReceipt, - CatalogPublicationStorage, CatalogRestartArtifact, CatalogRestartByteLimit, - CatalogRestartByteLimitError, CatalogRestartError, CatalogRestartPhase, CatalogRestartPolicy, - CatalogSnapshot, CatalogSnapshotError, CatalogSuccessor, CatalogTransitionError, - ChecksummedCatalog, ChecksummedPublicationHead, ChecksummedSegmentRecord, ClosedSegment, - EmptyDispositionSetDigest, FilesystemCatalogPublicationError, FilesystemCatalogPublisher, - FilesystemCatalogSnapshot, FilesystemPlatformAdmission, FilesystemPlatformAdmissionError, + CanonicalPublicationHead, CanonicalStoreFormatMarker, CanonicalStoreMigrationIntent, + CatalogAdmissionError, CatalogAllocationPhase, CatalogDecodeError, CatalogEncodeError, + CatalogEntryDecodeError, CatalogPublicationError, CatalogPublicationExpectation, + CatalogPublicationOutcome, CatalogPublicationPhase, CatalogPublicationReadiness, + CatalogPublicationReceipt, CatalogPublicationStorage, CatalogRestartArtifact, + CatalogRestartByteLimit, CatalogRestartByteLimitError, CatalogRestartError, + CatalogRestartPhase, CatalogRestartPolicy, CatalogSnapshot, CatalogSnapshotError, + CatalogSuccessor, CatalogTransitionError, ChecksummedCatalog, ChecksummedPublicationHead, + ChecksummedSegmentRecord, ClosedSegment, EmptyDispositionSetDigest, + FilesystemCatalogPublicationError, FilesystemCatalogPublisher, FilesystemCatalogSnapshot, + FilesystemPlatformAdmission, FilesystemPlatformAdmissionError, FilesystemRecoveryInventoryReader, FilesystemRecoveryNextHeadFinalizationOpenError, FilesystemRecoveryNextHeadFinalizer, FilesystemRecoverySegmentResumeOpenError, FilesystemRecoverySegmentResumer, FilesystemRecoverySegmentStage, diff --git a/tests/store_migration_intent_encoding.rs b/tests/store_migration_intent_encoding.rs new file mode 100644 index 0000000..2158dbc --- /dev/null +++ b/tests/store_migration_intent_encoding.rs @@ -0,0 +1,95 @@ +//! Canonical version-2 store-migration intent encoding laws. + +#[path = "store_migration_intent/fixture.rs"] +mod fixture; +mod support; + +use std::error::Error; + +use keep::{ + AdmittedSegment, AdmittedStoreMigrationIntent, CanonicalStoreMigrationIntent, + ChecksummedCatalog, ChecksummedPublicationHead, LayoutEntryLimit, SegmentReadPolicy, + SegmentRecordLimit, +}; +use support::decode_hex; + +const SEGMENT: &str = include_str!("../conformance/segment-store/v1/one-zero-segment.hex"); +const CATALOG: &str = include_str!("../conformance/segment-store/v1/one-zero-catalog.hex"); +const HEAD: &str = include_str!("../conformance/segment-store/v1/one-zero-head.hex"); +const CATALOG_TWO: &str = + include_str!("../conformance/segment-store/v1/one-zero-catalog-generation-two.hex"); +const HEAD_TWO: &str = + include_str!("../conformance/segment-store/v1/one-zero-head-generation-two.hex"); +const PREDECESSOR_OFFSET: usize = 72; +const PREDECESSOR_END: usize = 104; + +#[test] +fn admitted_coordinates_reproduce_the_frozen_intent() -> Result<(), Box> { + let expected = fixture::fixture_bytes()?; + let admitted = AdmittedStoreMigrationIntent::decode(&expected)?; + assert_eq!( + admitted.inventory_digest().as_bytes(), + &fixture::INVENTORY_DIGEST + ); + let canonical = canonical_intent(CATALOG, HEAD, &admitted)?; + + assert_eq!(canonical.encoded(), expected); + assert_eq!(canonical.digest(), admitted.digest()); + assert_eq!(canonical.store_identifier(), admitted.store_identifier()); + assert_eq!(canonical.digest().as_bytes(), &fixture::INTENT_DIGEST); + assert_eq!( + canonical.store_identifier().as_bytes(), + &fixture::STORE_IDENTIFIER + ); + Ok(()) +} + +#[test] +fn successor_intent_encodes_the_exact_predecessor() -> Result<(), Box> { + let source_bytes = fixture::fixture_bytes()?; + let source = AdmittedStoreMigrationIntent::decode(&source_bytes)?; + let canonical = canonical_intent(CATALOG_TWO, HEAD_TWO, &source)?; + let admitted = AdmittedStoreMigrationIntent::decode(canonical.encoded())?; + + assert_eq!(admitted.catalog_generation().get(), 2); + assert_eq!( + canonical.encoded().get(PREDECESSOR_OFFSET..PREDECESSOR_END), + Some(fixture::CATALOG_DIGEST.as_slice()) + ); + assert_eq!( + admitted + .predecessor_catalog_digest() + .ok_or("successor intent omitted its predecessor")? + .as_bytes(), + &fixture::CATALOG_DIGEST + ); + Ok(()) +} + +fn canonical_intent( + catalog_hex: &str, + head_hex: &str, + source: &AdmittedStoreMigrationIntent<'_>, +) -> Result> { + let segment_bytes = protocol_fixture(SEGMENT)?; + let catalog_bytes = protocol_fixture(catalog_hex)?; + let head_bytes = protocol_fixture(head_hex)?; + let segment = AdmittedSegment::decode(&segment_bytes, maximum_policy())?; + let catalog = ChecksummedCatalog::decode(&catalog_bytes)?.admit(&[segment])?; + let snapshot = ChecksummedPublicationHead::decode(&head_bytes)?.admit(catalog)?; + Ok(CanonicalStoreMigrationIntent::from_snapshot( + &snapshot, + source.inventory_digest(), + source.root_device_identity(), + source.root_mount_identity(), + source.root_file_identity(), + )) +} + +const fn maximum_policy() -> SegmentReadPolicy { + SegmentReadPolicy::new(SegmentRecordLimit::MAXIMUM, LayoutEntryLimit::MAXIMUM) +} + +fn protocol_fixture(hex: &str) -> Result, Box> { + decode_hex(hex.strip_suffix('\n').ok_or("fixture must end in one LF")?).map_err(Into::into) +} From ea293f8bb02e64bb14f77c63a23492dd071d4ffc Mon Sep 17 00:00:00 2001 From: James Ross Date: Thu, 30 Jul 2026 03:52:15 -0700 Subject: [PATCH 037/111] Add: Construct store migration receipts --- CHANGELOG.md | 2 +- docs/formats/segment-store-v2/recovery.md | 2 +- docs/formats/segment-store-v2/requirements.md | 2 +- src/adapters/store_migration.rs | 4 ++ .../canonical_migration_intent.rs | 11 ++++ .../canonical_migration_receipt.rs | 35 +++++++++++++ .../migration_receipt_decoder.rs | 30 ++++------- .../migration_receipt_encoder.rs | 52 +++++++++++++++++++ .../migration_receipt_format.rs | 15 ++++++ .../migration_receipt_initial_state.rs | 27 +++++++--- src/lib.rs | 16 +++--- tests/store_migration_receipt_encoding.rs | 46 ++++++++++++++++ 12 files changed, 204 insertions(+), 38 deletions(-) create mode 100644 src/adapters/store_migration/canonical_migration_receipt.rs create mode 100644 src/adapters/store_migration/migration_receipt_encoder.rs create mode 100644 src/adapters/store_migration/migration_receipt_format.rs create mode 100644 tests/store_migration_receipt_encoding.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index 94ceadd..9c00eb6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,7 +10,7 @@ after its public API and format compatibility policies are established. ### Changed -- Version-2 marker, canonical migration-intent construction, and record admission bind +- Version-2 marker, canonical intent/receipt construction, and record admission bind exact catalog, predecessor, root, definition, store, empty-state, and checksum, digest, and synchronization-mask coordinates; migration fuzzing drives all three decoders, streamed inventory is bounded, and `StoreMigrationPhase` diff --git a/docs/formats/segment-store-v2/recovery.md b/docs/formats/segment-store-v2/recovery.md index b652509..f446cf3 100644 --- a/docs/formats/segment-store-v2/recovery.md +++ b/docs/formats/segment-store-v2/recovery.md @@ -61,7 +61,7 @@ corpus `definition.tsv` bytes. The format-marker digest is BLAKE3-256 of `keep.store-format-marker/v2\0` followed by all 96 marker bytes. `CanonicalStoreFormatMarker` produces the registered marker; `AdmittedStoreFormatMarker` admits its framing, checksum, definition, and namespace bound. -`CanonicalStoreMigrationIntent` reproduces intent bytes from typed coordinates; `AdmittedStoreMigrationIntent` admits integrity and identity; `AdmittedStoreMigrationReceipt` binds the intent, marker, empty states, and synchronization bits. +`CanonicalStoreMigrationIntent` reproduces typed intent bytes; `CanonicalStoreMigrationReceipt` binds canonical intent, marker, empty states, and complete synchronization; admitted record types verify both. These record boundaries do not prove the named live inventory, physical root, store version, or execution of filesystem migration. diff --git a/docs/formats/segment-store-v2/requirements.md b/docs/formats/segment-store-v2/requirements.md index 072dc6c..e68ae63 100644 --- a/docs/formats/segment-store-v2/requirements.md +++ b/docs/formats/segment-store-v2/requirements.md @@ -29,7 +29,7 @@ case is not evidence. | ID | Requirement | Evidence | Status | | --- | --- | --- | --- | | `KEEP-MIGRATION-001` | Exact version-1 stores remain admitted until a durable migration artifact exists | compatibility fixtures | Planned in #19 | -| `KEEP-MIGRATION-002` | Format marker, intent, and receipt have complete fixed byte tables, named domains, bounds, checksums, deterministic store identity, and exact initial-state digests | exact admission in `tests/store_format_marker.rs`, `tests/store_migration_intent.rs`, and `tests/store_migration_receipt.rs`; canonical intent construction in `tests/store_migration_intent_encoding.rs`; seeded `migration_format` fuzz target | Implemented | +| `KEEP-MIGRATION-002` | Format marker, intent, and receipt have complete fixed byte tables, named domains, bounds, checksums, deterministic store identity, and exact initial-state digests | exact admission in `tests/store_format_marker.rs`, `tests/store_migration_intent.rs`, and `tests/store_migration_receipt.rs`; canonical construction in `tests/store_migration_intent_encoding.rs` and `tests/store_migration_receipt_encoding.rs`; seeded `migration_format` fuzz target | Implemented | | `KEEP-MIGRATION-003` | Migration revalidates version-1 head, catalog, pools, root identity, and writer authority before mutation | bounded canonical pool-inventory evidence in `tests/store_migration_inventory.rs`; capability-relative integration tests remain | In progress in #19 | | `KEEP-MIGRATION-004` | Every partial migration prefix continues idempotently under writer authority | state-machine and recovery tests | Planned in #19 | | `KEEP-MIGRATION-005` | Unknown, out-of-order, substituted, corrupt, conflicting, or changed evidence is unrecoverable ambiguity | corruption and mutation matrix | Planned in #19 | diff --git a/src/adapters/store_migration.rs b/src/adapters/store_migration.rs index ee2ab13..2515c97 100644 --- a/src/adapters/store_migration.rs +++ b/src/adapters/store_migration.rs @@ -5,6 +5,7 @@ mod admitted_migration_intent; mod admitted_migration_receipt; mod canonical_format_marker; mod canonical_migration_intent; +mod canonical_migration_receipt; mod empty_disposition_set_digest; mod format_definition_digest; mod format_marker_decode_error; @@ -30,6 +31,8 @@ mod migration_phase; mod migration_receipt_decode_error; mod migration_receipt_decode_error_display; mod migration_receipt_decoder; +mod migration_receipt_encoder; +mod migration_receipt_format; mod migration_receipt_initial_state; mod migration_record_bytes; mod migration_synchronization_mask; @@ -41,6 +44,7 @@ pub use admitted_migration_intent::AdmittedStoreMigrationIntent; pub use admitted_migration_receipt::AdmittedStoreMigrationReceipt; pub use canonical_format_marker::CanonicalStoreFormatMarker; pub use canonical_migration_intent::CanonicalStoreMigrationIntent; +pub use canonical_migration_receipt::CanonicalStoreMigrationReceipt; pub use empty_disposition_set_digest::EmptyDispositionSetDigest; pub use format_definition_digest::StoreFormatDefinitionDigest; pub use format_marker_decode_error::StoreFormatMarkerDecodeError; diff --git a/src/adapters/store_migration/canonical_migration_intent.rs b/src/adapters/store_migration/canonical_migration_intent.rs index f6e2303..9c70969 100644 --- a/src/adapters/store_migration/canonical_migration_intent.rs +++ b/src/adapters/store_migration/canonical_migration_intent.rs @@ -21,6 +21,17 @@ pub struct CanonicalStoreMigrationIntent { } impl CanonicalStoreMigrationIntent { + /// Owns the exact bytes and identities of an admitted intent. + pub const fn from_admitted(intent: &super::AdmittedStoreMigrationIntent<'_>) -> Self { + let mut encoded = [0_u8; migration_intent_format::ENCODED_LENGTH]; + encoded.copy_from_slice(intent.encoded()); + Self { + encoded, + digest: intent.digest(), + store_identifier: intent.store_identifier(), + } + } + /// Constructs one canonical intent from typed migration coordinates. pub fn from_snapshot( snapshot: &CatalogSnapshot<'_, '_, '_>, diff --git a/src/adapters/store_migration/canonical_migration_receipt.rs b/src/adapters/store_migration/canonical_migration_receipt.rs new file mode 100644 index 0000000..7c40d86 --- /dev/null +++ b/src/adapters/store_migration/canonical_migration_receipt.rs @@ -0,0 +1,35 @@ +//! This boundary module owns canonical owned migration-receipt bytes. + +use super::{ + CanonicalStoreFormatMarker, CanonicalStoreMigrationIntent, migration_receipt_encoder, + migration_receipt_format, +}; + +/// Owned canonical version-2 store-migration completion receipt. +/// +/// Construction binds canonical artifacts and the registered complete initial +/// state. It does not prove that the named filesystem transitions occurred. +#[must_use] +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct CanonicalStoreMigrationReceipt { + encoded: [u8; migration_receipt_format::ENCODED_LENGTH], +} + +impl CanonicalStoreMigrationReceipt { + /// Constructs the one complete receipt for `intent` and `marker`. + pub fn from_canonical( + intent: &CanonicalStoreMigrationIntent, + marker: &CanonicalStoreFormatMarker, + ) -> Self { + migration_receipt_encoder::encode(intent, marker) + } + + /// Returns the exact canonical receipt bytes. + pub const fn encoded(&self) -> &[u8] { + &self.encoded + } + + pub(super) const fn admitted(encoded: [u8; migration_receipt_format::ENCODED_LENGTH]) -> Self { + Self { encoded } + } +} diff --git a/src/adapters/store_migration/migration_receipt_decoder.rs b/src/adapters/store_migration/migration_receipt_decoder.rs index 308308d..cd92d71 100644 --- a/src/adapters/store_migration/migration_receipt_decoder.rs +++ b/src/adapters/store_migration/migration_receipt_decoder.rs @@ -10,14 +10,9 @@ use super::migration_record_bytes::{ use super::{ AdmittedStoreFormatMarker, AdmittedStoreMigrationIntent, AdmittedStoreMigrationReceipt, MigrationSynchronizationMask, StoreMigrationReceiptDecodeError, + migration_receipt_format as format, }; -const CHECKSUM_OFFSET: usize = 224; -const MAGIC: [u8; 16] = *b"KEEP:MIG:REC2\0\0\0"; -const VERSION: u16 = 2; -const RECORD_LENGTH: u16 = 256; -const CHECKSUM_DOMAIN: &[u8] = b"keep.store-migration-receipt-checksum/v2\0"; - pub(super) fn decode<'encoded>( encoded: &'encoded [u8], intent: &AdmittedStoreMigrationIntent<'_>, @@ -49,20 +44,20 @@ pub(super) fn decode<'encoded>( fn validate_fixed_fields(encoded: &[u8]) -> Result<(), StoreMigrationReceiptDecodeError> { let magic = read_array(encoded, 0)?; - if magic != MAGIC { + if magic != format::MAGIC { return Err(StoreMigrationReceiptDecodeError::InvalidMagic { observed: magic }); } let version = read_u16(encoded, 16)?; - if version != VERSION { + if version != format::VERSION { return Err(StoreMigrationReceiptDecodeError::UnsupportedVersion { - expected: VERSION, + expected: format::VERSION, observed: version, }); } let record_length = read_u16(encoded, 18)?; - if record_length != RECORD_LENGTH { + if record_length != format::RECORD_LENGTH { return Err(StoreMigrationReceiptDecodeError::InvalidRecordLength { - expected: RECORD_LENGTH, + expected: format::RECORD_LENGTH, observed: record_length, }); } @@ -75,10 +70,10 @@ fn validate_fixed_fields(encoded: &[u8]) -> Result<(), StoreMigrationReceiptDeco fn verify_checksum(encoded: &[u8]) -> Result<(), StoreMigrationReceiptDecodeError> { let preimage = encoded - .get(..CHECKSUM_OFFSET) + .get(..format::CHECKSUM_OFFSET) .ok_or_else(|| wrong_length(encoded))?; - let observed = read_array(encoded, CHECKSUM_OFFSET)?; - let expected = hash(CHECKSUM_DOMAIN, preimage); + let observed = read_array(encoded, format::CHECKSUM_OFFSET)?; + let expected = format::checksum(preimage); if observed == expected { Ok(()) } else { @@ -148,10 +143,3 @@ fn read_synchronization_mask( } Ok(MigrationSynchronizationMask::complete()) } - -fn hash(domain: &[u8], bytes: &[u8]) -> [u8; 32] { - let mut hasher = blake3::Hasher::new(); - hasher.update(domain); - hasher.update(bytes); - *hasher.finalize().as_bytes() -} diff --git a/src/adapters/store_migration/migration_receipt_encoder.rs b/src/adapters/store_migration/migration_receipt_encoder.rs new file mode 100644 index 0000000..9020298 --- /dev/null +++ b/src/adapters/store_migration/migration_receipt_encoder.rs @@ -0,0 +1,52 @@ +//! This boundary module owns canonical migration-receipt encoding. + +use super::migration_receipt_initial_state::{ + empty_disposition_digest, initial_gc_digest, initial_retention_digest, +}; +use super::{ + CanonicalStoreFormatMarker, CanonicalStoreMigrationIntent, CanonicalStoreMigrationReceipt, + MigrationSynchronizationMask, migration_receipt_format as format, +}; + +pub(super) fn encode( + intent: &CanonicalStoreMigrationIntent, + marker: &CanonicalStoreFormatMarker, +) -> CanonicalStoreMigrationReceipt { + let mut encoded = [0_u8; format::ENCODED_LENGTH]; + let (preimage, checksum_slot) = encoded.split_at_mut(format::CHECKSUM_OFFSET); + write_preimage(preimage, intent, marker); + checksum_slot.copy_from_slice(&format::checksum(preimage)); + CanonicalStoreMigrationReceipt::admitted(encoded) +} + +fn write_preimage( + output: &mut [u8], + intent: &CanonicalStoreMigrationIntent, + marker: &CanonicalStoreFormatMarker, +) { + let (magic, output) = output.split_at_mut(16); + magic.copy_from_slice(&format::MAGIC); + let (version, output) = output.split_at_mut(2); + version.copy_from_slice(&format::VERSION.to_be_bytes()); + let (record_length, output) = output.split_at_mut(2); + record_length.copy_from_slice(&format::RECORD_LENGTH.to_be_bytes()); + let (flags, output) = output.split_at_mut(4); + flags.copy_from_slice(&0_u32.to_be_bytes()); + let (intent_digest, output) = output.split_at_mut(32); + intent_digest.copy_from_slice(intent.digest().as_bytes()); + let (store_identifier, output) = output.split_at_mut(32); + store_identifier.copy_from_slice(intent.store_identifier().as_bytes()); + let (marker_digest, output) = output.split_at_mut(32); + marker_digest.copy_from_slice(marker.digest().as_bytes()); + let (retention_digest, output) = output.split_at_mut(32); + retention_digest.copy_from_slice(initial_retention_digest().as_bytes()); + let (gc_digest, output) = output.split_at_mut(32); + gc_digest.copy_from_slice(initial_gc_digest().as_bytes()); + let (disposition_digest, synchronization_mask) = output.split_at_mut(32); + disposition_digest.copy_from_slice(empty_disposition_digest().as_bytes()); + synchronization_mask.copy_from_slice( + &MigrationSynchronizationMask::complete() + .bits() + .to_be_bytes(), + ); +} diff --git a/src/adapters/store_migration/migration_receipt_format.rs b/src/adapters/store_migration/migration_receipt_format.rs new file mode 100644 index 0000000..392ff3d --- /dev/null +++ b/src/adapters/store_migration/migration_receipt_format.rs @@ -0,0 +1,15 @@ +//! This boundary module owns shared migration-receipt framing and integrity. + +pub(super) const CHECKSUM_OFFSET: usize = 224; +pub(super) const ENCODED_LENGTH: usize = 256; +pub(super) const MAGIC: [u8; 16] = *b"KEEP:MIG:REC2\0\0\0"; +pub(super) const RECORD_LENGTH: u16 = 256; +pub(super) const VERSION: u16 = 2; +const CHECKSUM_DOMAIN: &[u8] = b"keep.store-migration-receipt-checksum/v2\0"; + +pub(super) fn checksum(preimage: &[u8]) -> [u8; 32] { + let mut hasher = blake3::Hasher::new(); + hasher.update(CHECKSUM_DOMAIN); + hasher.update(preimage); + *hasher.finalize().as_bytes() +} diff --git a/src/adapters/store_migration/migration_receipt_initial_state.rs b/src/adapters/store_migration/migration_receipt_initial_state.rs index 01a43d5..d89af3e 100644 --- a/src/adapters/store_migration/migration_receipt_initial_state.rs +++ b/src/adapters/store_migration/migration_receipt_initial_state.rs @@ -13,10 +13,11 @@ const EMPTY_DISPOSITION_DOMAIN: &[u8] = b"keep.empty-disposition-set/v2\0"; pub(super) fn read_initial_retention_digest( encoded: &[u8], ) -> Result { - let expected = digest(INITIAL_RETENTION_DOMAIN); + let admitted = initial_retention_digest(); + let expected = *admitted.as_bytes(); let observed = read_array(encoded, 120)?; if observed == expected { - Ok(InitialRetentionStateDigest::from_hash(expected)) + Ok(admitted) } else { Err( StoreMigrationReceiptDecodeError::InitialRetentionStateDigestMismatch { @@ -30,10 +31,11 @@ pub(super) fn read_initial_retention_digest( pub(super) fn read_initial_gc_digest( encoded: &[u8], ) -> Result { - let expected = digest(INITIAL_GC_DOMAIN); + let admitted = initial_gc_digest(); + let expected = *admitted.as_bytes(); let observed = read_array(encoded, 152)?; if observed == expected { - Ok(InitialGcStateDigest::from_hash(expected)) + Ok(admitted) } else { Err(StoreMigrationReceiptDecodeError::InitialGcStateDigestMismatch { expected, observed }) } @@ -42,10 +44,11 @@ pub(super) fn read_initial_gc_digest( pub(super) fn read_empty_disposition_digest( encoded: &[u8], ) -> Result { - let expected = digest(EMPTY_DISPOSITION_DOMAIN); + let admitted = empty_disposition_digest(); + let expected = *admitted.as_bytes(); let observed = read_array(encoded, 184)?; if observed == expected { - Ok(EmptyDispositionSetDigest::from_hash(expected)) + Ok(admitted) } else { Err( StoreMigrationReceiptDecodeError::EmptyDispositionSetDigestMismatch { @@ -56,6 +59,18 @@ pub(super) fn read_empty_disposition_digest( } } +pub(super) fn initial_retention_digest() -> InitialRetentionStateDigest { + InitialRetentionStateDigest::from_hash(digest(INITIAL_RETENTION_DOMAIN)) +} + +pub(super) fn initial_gc_digest() -> InitialGcStateDigest { + InitialGcStateDigest::from_hash(digest(INITIAL_GC_DOMAIN)) +} + +pub(super) fn empty_disposition_digest() -> EmptyDispositionSetDigest { + EmptyDispositionSetDigest::from_hash(digest(EMPTY_DISPOSITION_DOMAIN)) +} + fn digest(domain: &[u8]) -> [u8; 32] { *blake3::hash(domain).as_bytes() } diff --git a/src/lib.rs b/src/lib.rs index 0f49f61..e6a551a 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -59,14 +59,14 @@ pub use adapters::{ AdmittedStoreFormatMarker, AdmittedStoreMigrationIntent, AdmittedStoreMigrationReceipt, BlobIdBinaryParseError, BlobIdTextParseError, CanonicalCatalog, CanonicalLayoutRecord, CanonicalPublicationHead, CanonicalStoreFormatMarker, CanonicalStoreMigrationIntent, - CatalogAdmissionError, CatalogAllocationPhase, CatalogDecodeError, CatalogEncodeError, - CatalogEntryDecodeError, CatalogPublicationError, CatalogPublicationExpectation, - CatalogPublicationOutcome, CatalogPublicationPhase, CatalogPublicationReadiness, - CatalogPublicationReceipt, CatalogPublicationStorage, CatalogRestartArtifact, - CatalogRestartByteLimit, CatalogRestartByteLimitError, CatalogRestartError, - CatalogRestartPhase, CatalogRestartPolicy, CatalogSnapshot, CatalogSnapshotError, - CatalogSuccessor, CatalogTransitionError, ChecksummedCatalog, ChecksummedPublicationHead, - ChecksummedSegmentRecord, ClosedSegment, EmptyDispositionSetDigest, + CanonicalStoreMigrationReceipt, CatalogAdmissionError, CatalogAllocationPhase, + CatalogDecodeError, CatalogEncodeError, CatalogEntryDecodeError, CatalogPublicationError, + CatalogPublicationExpectation, CatalogPublicationOutcome, CatalogPublicationPhase, + CatalogPublicationReadiness, CatalogPublicationReceipt, CatalogPublicationStorage, + CatalogRestartArtifact, CatalogRestartByteLimit, CatalogRestartByteLimitError, + CatalogRestartError, CatalogRestartPhase, CatalogRestartPolicy, CatalogSnapshot, + CatalogSnapshotError, CatalogSuccessor, CatalogTransitionError, ChecksummedCatalog, + ChecksummedPublicationHead, ChecksummedSegmentRecord, ClosedSegment, EmptyDispositionSetDigest, FilesystemCatalogPublicationError, FilesystemCatalogPublisher, FilesystemCatalogSnapshot, FilesystemPlatformAdmission, FilesystemPlatformAdmissionError, FilesystemRecoveryInventoryReader, FilesystemRecoveryNextHeadFinalizationOpenError, diff --git a/tests/store_migration_receipt_encoding.rs b/tests/store_migration_receipt_encoding.rs new file mode 100644 index 0000000..c5f8793 --- /dev/null +++ b/tests/store_migration_receipt_encoding.rs @@ -0,0 +1,46 @@ +//! Canonical version-2 store-migration receipt encoding laws. + +#[path = "store_migration_receipt/fixture.rs"] +mod fixture; +mod support; + +use std::error::Error; + +use keep::{ + AdmittedStoreFormatMarker, AdmittedStoreMigrationIntent, AdmittedStoreMigrationReceipt, + CanonicalStoreFormatMarker, CanonicalStoreMigrationIntent, CanonicalStoreMigrationReceipt, +}; + +#[test] +fn canonical_completion_receipt_reproduces_every_frozen_field() -> Result<(), Box> { + let expected = fixture::receipt_bytes()?; + let intent_bytes = fixture::intent_bytes()?; + let admitted_intent = AdmittedStoreMigrationIntent::decode(&intent_bytes)?; + let intent = CanonicalStoreMigrationIntent::from_admitted(&admitted_intent); + let marker = CanonicalStoreFormatMarker::version_two(); + assert_eq!(marker.encoded(), fixture::marker_bytes()?); + + let canonical = CanonicalStoreMigrationReceipt::from_canonical(&intent, &marker); + assert_eq!(canonical.encoded(), expected); + + let admitted_marker = AdmittedStoreFormatMarker::decode(marker.encoded())?; + let admitted = AdmittedStoreMigrationReceipt::decode( + canonical.encoded(), + &admitted_intent, + &admitted_marker, + )?; + assert_eq!( + admitted.initial_retention_state_digest().as_bytes(), + &fixture::INITIAL_RETENTION_DIGEST + ); + assert_eq!( + admitted.initial_gc_state_digest().as_bytes(), + &fixture::INITIAL_GC_DIGEST + ); + assert_eq!( + admitted.empty_disposition_set_digest().as_bytes(), + &fixture::DISPOSITION_DIGEST + ); + assert_eq!(admitted.synchronization_mask().bits(), 0x03ff); + Ok(()) +} From 5a38fa75f7178f9451e0a6a8c7204a473fbdcebb Mon Sep 17 00:00:00 2001 From: James Ross Date: Thu, 30 Jul 2026 04:03:18 -0700 Subject: [PATCH 038/111] Add: Retain canonical migration coordinates --- CHANGELOG.md | 2 +- docs/formats/segment-store-v2/recovery.md | 2 +- .../canonical_migration_intent.rs | 75 ++++++++++++++++--- .../migration_intent_encoder.rs | 18 ++++- tests/store_migration_intent_encoding.rs | 36 ++++++++- 5 files changed, 120 insertions(+), 13 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9c00eb6..783c217 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,7 +10,7 @@ after its public API and format compatibility policies are established. ### Changed -- Version-2 marker, canonical intent/receipt construction, and record admission bind +- Version-2 marker, typed canonical intent/receipt construction, and record admission bind exact catalog, predecessor, root, definition, store, empty-state, and checksum, digest, and synchronization-mask coordinates; migration fuzzing drives all three decoders, streamed inventory is bounded, and `StoreMigrationPhase` diff --git a/docs/formats/segment-store-v2/recovery.md b/docs/formats/segment-store-v2/recovery.md index f446cf3..a5e4444 100644 --- a/docs/formats/segment-store-v2/recovery.md +++ b/docs/formats/segment-store-v2/recovery.md @@ -61,7 +61,7 @@ corpus `definition.tsv` bytes. The format-marker digest is BLAKE3-256 of `keep.store-format-marker/v2\0` followed by all 96 marker bytes. `CanonicalStoreFormatMarker` produces the registered marker; `AdmittedStoreFormatMarker` admits its framing, checksum, definition, and namespace bound. -`CanonicalStoreMigrationIntent` reproduces typed intent bytes; `CanonicalStoreMigrationReceipt` binds canonical intent, marker, empty states, and complete synchronization; admitted record types verify both. +`CanonicalStoreMigrationIntent` retains and reproduces typed intent coordinates; `CanonicalStoreMigrationReceipt` binds intent, marker, empty states, and complete synchronization; admitted record types verify both. These record boundaries do not prove the named live inventory, physical root, store version, or execution of filesystem migration. diff --git a/src/adapters/store_migration/canonical_migration_intent.rs b/src/adapters/store_migration/canonical_migration_intent.rs index 9c70969..28a95ca 100644 --- a/src/adapters/store_migration/canonical_migration_intent.rs +++ b/src/adapters/store_migration/canonical_migration_intent.rs @@ -1,11 +1,12 @@ //! This boundary module owns canonical owned migration-intent bytes. +use super::admitted_migration_intent::StoreMigrationIntentFields; use super::{ - ImmutablePoolInventoryDigest, StoreIdentifier, StoreMigrationIntentDigest, - StoreRootDeviceIdentity, StoreRootFileIdentity, StoreRootMountIdentity, - migration_intent_encoder, migration_intent_format, + ImmutablePoolInventoryDigest, StoreFormatDefinitionDigest, StoreIdentifier, + StoreMigrationIntentDigest, StoreRootDeviceIdentity, StoreRootFileIdentity, + StoreRootMountIdentity, migration_intent_encoder, migration_intent_format, }; -use crate::CatalogSnapshot; +use crate::{CatalogDigest, CatalogGeneration, CatalogLength, CatalogSnapshot}; /// Owned canonical version-2 store-migration intent. /// @@ -16,8 +17,8 @@ use crate::CatalogSnapshot; #[derive(Clone, Debug, Eq, PartialEq)] pub struct CanonicalStoreMigrationIntent { encoded: [u8; migration_intent_format::ENCODED_LENGTH], + fields: StoreMigrationIntentFields, digest: StoreMigrationIntentDigest, - store_identifier: StoreIdentifier, } impl CanonicalStoreMigrationIntent { @@ -27,8 +28,19 @@ impl CanonicalStoreMigrationIntent { encoded.copy_from_slice(intent.encoded()); Self { encoded, + fields: StoreMigrationIntentFields { + catalog_generation: intent.catalog_generation(), + catalog_length: intent.catalog_length(), + catalog_digest: intent.catalog_digest(), + predecessor_catalog_digest: intent.predecessor_catalog_digest(), + inventory_digest: intent.inventory_digest(), + root_device_identity: intent.root_device_identity(), + root_mount_identity: intent.root_mount_identity(), + root_file_identity: intent.root_file_identity(), + target_definition_digest: intent.target_definition_digest(), + store_identifier: intent.store_identifier(), + }, digest: intent.digest(), - store_identifier: intent.store_identifier(), } } @@ -59,20 +71,65 @@ impl CanonicalStoreMigrationIntent { self.digest } + /// Returns the positive catalog generation named by the intent. + pub const fn catalog_generation(&self) -> CatalogGeneration { + self.fields.catalog_generation + } + + /// Returns the exact catalog byte length named by the intent. + pub const fn catalog_length(&self) -> CatalogLength { + self.fields.catalog_length + } + + /// Returns the catalog digest named by the intent. + pub const fn catalog_digest(&self) -> CatalogDigest { + self.fields.catalog_digest + } + + /// Returns the generation-relative predecessor digest. + pub const fn predecessor_catalog_digest(&self) -> Option { + self.fields.predecessor_catalog_digest + } + + /// Returns the immutable-pool inventory digest named by the intent. + pub const fn inventory_digest(&self) -> ImmutablePoolInventoryDigest { + self.fields.inventory_digest + } + + /// Returns the serialized root device coordinate. + pub const fn root_device_identity(&self) -> StoreRootDeviceIdentity { + self.fields.root_device_identity + } + + /// Returns the serialized root mount coordinate. + pub const fn root_mount_identity(&self) -> StoreRootMountIdentity { + self.fields.root_mount_identity + } + + /// Returns the serialized root file coordinate. + pub const fn root_file_identity(&self) -> StoreRootFileIdentity { + self.fields.root_file_identity + } + + /// Returns the registered target format-definition digest. + pub const fn target_definition_digest(&self) -> StoreFormatDefinitionDigest { + self.fields.target_definition_digest + } + /// Returns the deterministic logical store identity. pub const fn store_identifier(&self) -> StoreIdentifier { - self.store_identifier + self.fields.store_identifier } pub(super) const fn admitted( encoded: [u8; migration_intent_format::ENCODED_LENGTH], + fields: StoreMigrationIntentFields, digest: StoreMigrationIntentDigest, - store_identifier: StoreIdentifier, ) -> Self { Self { encoded, + fields, digest, - store_identifier, } } } diff --git a/src/adapters/store_migration/migration_intent_encoder.rs b/src/adapters/store_migration/migration_intent_encoder.rs index a461655..faa7572 100644 --- a/src/adapters/store_migration/migration_intent_encoder.rs +++ b/src/adapters/store_migration/migration_intent_encoder.rs @@ -1,5 +1,6 @@ //! This boundary module owns canonical migration-intent encoding. +use super::admitted_migration_intent::StoreMigrationIntentFields; use super::migration_intent_format::StoreIdentifierFields; use super::{ CanonicalStoreMigrationIntent, ImmutablePoolInventoryDigest, StoreFormatDefinitionDigest, @@ -41,7 +42,22 @@ pub(super) fn encode( write_preimage(preimage, &fields, roots, store_identifier); checksum_slot.copy_from_slice(&format::checksum(preimage)); let digest = format::digest(&encoded); - CanonicalStoreMigrationIntent::admitted(encoded, digest, store_identifier) + CanonicalStoreMigrationIntent::admitted( + encoded, + StoreMigrationIntentFields { + catalog_generation: fields.catalog_generation, + catalog_length: fields.catalog_length, + catalog_digest: fields.catalog_digest, + predecessor_catalog_digest: fields.predecessor_catalog_digest, + inventory_digest: fields.inventory_digest, + root_device_identity: roots.device, + root_mount_identity: roots.mount, + root_file_identity: roots.file, + target_definition_digest: fields.target_definition_digest, + store_identifier, + }, + digest, + ) } fn write_preimage( diff --git a/tests/store_migration_intent_encoding.rs b/tests/store_migration_intent_encoding.rs index 2158dbc..34a85a9 100644 --- a/tests/store_migration_intent_encoding.rs +++ b/tests/store_migration_intent_encoding.rs @@ -35,7 +35,7 @@ fn admitted_coordinates_reproduce_the_frozen_intent() -> Result<(), Box Result<(), Box, +) { + assert_eq!( + canonical.catalog_generation(), + admitted.catalog_generation() + ); + assert_eq!(canonical.catalog_length(), admitted.catalog_length()); + assert_eq!(canonical.catalog_digest(), admitted.catalog_digest()); + assert_eq!( + canonical.predecessor_catalog_digest(), + admitted.predecessor_catalog_digest() + ); + assert_eq!(canonical.inventory_digest(), admitted.inventory_digest()); + assert_eq!( + canonical.root_device_identity(), + admitted.root_device_identity() + ); + assert_eq!( + canonical.root_mount_identity(), + admitted.root_mount_identity() + ); + assert_eq!( + canonical.root_file_identity(), + admitted.root_file_identity() + ); + assert_eq!( + canonical.target_definition_digest(), + admitted.target_definition_digest() + ); + assert_eq!(canonical.store_identifier(), admitted.store_identifier()); +} + #[test] fn successor_intent_encodes_the_exact_predecessor() -> Result<(), Box> { let source_bytes = fixture::fixture_bytes()?; From 9b3c711eb933ec9eefcb264026820c92c0a4c3c0 Mon Sep 17 00:00:00 2001 From: James Ross Date: Thu, 30 Jul 2026 04:19:08 -0700 Subject: [PATCH 039/111] Add: Define store migration storage port --- CHANGELOG.md | 6 +- docs/formats/segment-store-v2/recovery.md | 6 +- docs/formats/segment-store-v2/requirements.md | 4 +- src/adapters/store_migration.rs | 2 + .../store_migration/migration_storage.rs | 173 ++++++++++++++++++ src/lib.rs | 14 +- tests/store_migration_storage.rs | 87 +++++++++ .../recording_storage.rs | 146 +++++++++++++++ 8 files changed, 423 insertions(+), 15 deletions(-) create mode 100644 src/adapters/store_migration/migration_storage.rs create mode 100644 tests/store_migration_storage.rs create mode 100644 tests/store_migration_storage/recording_storage.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index 783c217..24ee878 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,9 +14,9 @@ after its public API and format compatibility policies are established. exact catalog, predecessor, root, definition, store, empty-state, and checksum, digest, and synchronization-mask coordinates; migration fuzzing drives all three decoders, streamed inventory is bounded, and `StoreMigrationPhase` - freezes 21 transitions. Retention preflight combines - expected-generation planning with deterministic closure verification; - authority-revalidated 17-phase orchestration returns its receipt after durable cleanup. + freezes 21 transitions behind explicit blocking storage capabilities. + Retention preflight combines expected-generation planning with deterministic + closure verification; authority-revalidated 17-phase orchestration returns its receipt after durable cleanup. - Repository crash-matrix execution now terminates isolated writer process groups at all 105 canonical before/during/after coordinates, retains open writer and stage authority until termination, executes production diff --git a/docs/formats/segment-store-v2/recovery.md b/docs/formats/segment-store-v2/recovery.md index a5e4444..6cefd88 100644 --- a/docs/formats/segment-store-v2/recovery.md +++ b/docs/formats/segment-store-v2/recovery.md @@ -61,9 +61,9 @@ corpus `definition.tsv` bytes. The format-marker digest is BLAKE3-256 of `keep.store-format-marker/v2\0` followed by all 96 marker bytes. `CanonicalStoreFormatMarker` produces the registered marker; `AdmittedStoreFormatMarker` admits its framing, checksum, definition, and namespace bound. -`CanonicalStoreMigrationIntent` retains and reproduces typed intent coordinates; `CanonicalStoreMigrationReceipt` binds intent, marker, empty states, and complete synchronization; admitted record types verify both. -These record boundaries do not prove the named live inventory, physical root, -store version, or execution of filesystem migration. +`CanonicalStoreMigrationIntent` retains typed intent coordinates; `CanonicalStoreMigrationReceipt` binds completion; admitted record types verify both. +`StoreMigrationStorage` names current-state verification and all 21 blocking durability capabilities but does not prove the live inventory, physical root, +store version, execution, or recovery of filesystem migration. ## Reader fence diff --git a/docs/formats/segment-store-v2/requirements.md b/docs/formats/segment-store-v2/requirements.md index e68ae63..e9bf0de 100644 --- a/docs/formats/segment-store-v2/requirements.md +++ b/docs/formats/segment-store-v2/requirements.md @@ -30,11 +30,11 @@ case is not evidence. | --- | --- | --- | --- | | `KEEP-MIGRATION-001` | Exact version-1 stores remain admitted until a durable migration artifact exists | compatibility fixtures | Planned in #19 | | `KEEP-MIGRATION-002` | Format marker, intent, and receipt have complete fixed byte tables, named domains, bounds, checksums, deterministic store identity, and exact initial-state digests | exact admission in `tests/store_format_marker.rs`, `tests/store_migration_intent.rs`, and `tests/store_migration_receipt.rs`; canonical construction in `tests/store_migration_intent_encoding.rs` and `tests/store_migration_receipt_encoding.rs`; seeded `migration_format` fuzz target | Implemented | -| `KEEP-MIGRATION-003` | Migration revalidates version-1 head, catalog, pools, root identity, and writer authority before mutation | bounded canonical pool-inventory evidence in `tests/store_migration_inventory.rs`; capability-relative integration tests remain | In progress in #19 | +| `KEEP-MIGRATION-003` | Migration revalidates version-1 head, catalog, pools, root identity, and writer authority before mutation | bounded canonical pool inventory in `tests/store_migration_inventory.rs`; mandatory verification capability in `tests/store_migration_storage.rs`; filesystem integration remains | In progress in #19 | | `KEEP-MIGRATION-004` | Every partial migration prefix continues idempotently under writer authority | state-machine and recovery tests | Planned in #19 | | `KEEP-MIGRATION-005` | Unknown, out-of-order, substituted, corrupt, conflicting, or changed evidence is unrecoverable ambiguity | corruption and mutation matrix | Planned in #19 | | `KEEP-MIGRATION-006` | Migration never rewrites or deletes admitted version-1 immutable bytes | byte-for-byte before/after witness | Planned in #19 | -| `KEEP-MIGRATION-007` | Process death around every intent stage, canonical link, namespace prefix, marker stage, receipt stage, cleanup, and synchronization boundary reaches a documented lawful state | ordered `StoreMigrationPhase` vocabulary in `tests/store_migration_phase.rs`; `KEEP-CRASH-053..=073` process-death matrix remains | In progress in #19 | +| `KEEP-MIGRATION-007` | Process death around every intent stage, canonical link, namespace prefix, marker stage, receipt stage, cleanup, and synchronization boundary reaches a documented lawful state | ordered `StoreMigrationPhase` vocabulary and matching storage capabilities in `tests/store_migration_phase.rs` and `tests/store_migration_storage.rs`; `KEEP-CRASH-053..=073` process-death matrix remains | In progress in #19 | | `KEEP-MIGRATION-008` | Version-1 admission refuses every version-2 or partial-migration artifact after migration begins | compatibility and fuzz tests | Planned in #19 | diff --git a/src/adapters/store_migration.rs b/src/adapters/store_migration.rs index 2515c97..e73c255 100644 --- a/src/adapters/store_migration.rs +++ b/src/adapters/store_migration.rs @@ -35,6 +35,7 @@ mod migration_receipt_encoder; mod migration_receipt_format; mod migration_receipt_initial_state; mod migration_record_bytes; +mod migration_storage; mod migration_synchronization_mask; mod store_identifier; mod store_root_identity; @@ -61,6 +62,7 @@ pub use migration_inventory_error::StoreMigrationInventoryError; pub use migration_inventory_hasher::StoreMigrationInventoryHasher; pub use migration_phase::StoreMigrationPhase; pub use migration_receipt_decode_error::StoreMigrationReceiptDecodeError; +pub use migration_storage::StoreMigrationStorage; pub use migration_synchronization_mask::MigrationSynchronizationMask; pub use store_identifier::StoreIdentifier; pub use store_root_identity::{ diff --git a/src/adapters/store_migration/migration_storage.rs b/src/adapters/store_migration/migration_storage.rs new file mode 100644 index 0000000..fed5ec8 --- /dev/null +++ b/src/adapters/store_migration/migration_storage.rs @@ -0,0 +1,173 @@ +//! This boundary module owns blocking store-migration durability capabilities. + +use std::io; + +use super::{ + CanonicalStoreFormatMarker, CanonicalStoreMigrationIntent, CanonicalStoreMigrationReceipt, +}; + +/// Blocking storage capabilities for one writer-locked version-2 migration. +/// +/// An implementation must retain exclusive writer authority and one pinned +/// store root for the complete operation. After `verify_current`, each method +/// corresponds to one [`StoreMigrationPhase`](super::StoreMigrationPhase) and +/// must not report success before its durability and verification obligations +/// are complete. +pub trait StoreMigrationStorage { + /// Revalidates the exact version-1 authority bound by `intent`. + /// + /// This must verify the catalog coordinates, inventory, physical root, + /// version-1 format, and absence of migration or version-2 artifacts. + /// + /// # Errors + /// + /// Returns the exact current-state or recovery-required refusal. + fn verify_current(&mut self, intent: &CanonicalStoreMigrationIntent) -> io::Result<()>; + + /// Exclusively creates and completely writes `migration.intent.next`. + /// + /// # Errors + /// + /// Returns the exact creation, write, or flush failure. + fn write_intent_stage(&mut self, intent: &CanonicalStoreMigrationIntent) -> io::Result<()>; + + /// Synchronizes the complete intent stage. + /// + /// # Errors + /// + /// Returns the exact file-synchronization failure. + fn synchronize_intent_stage(&mut self) -> io::Result<()>; + + /// Links and exactly verifies canonical `migration.intent`. + /// + /// # Errors + /// + /// Returns the exact link, reopen, or verification failure. + fn link_intent(&mut self, intent: &CanonicalStoreMigrationIntent) -> io::Result<()>; + + /// Synchronizes the store root after the intent link. + /// + /// # Errors + /// + /// Returns the exact directory-synchronization failure. + fn synchronize_root_after_intent(&mut self) -> io::Result<()>; + + /// Removes only the retained intent stage. + /// + /// # Errors + /// + /// Returns the exact removal failure. + fn remove_intent_stage(&mut self) -> io::Result<()>; + + /// Synchronizes the store root after intent-stage cleanup. + /// + /// # Errors + /// + /// Returns the exact directory-synchronization failure. + fn synchronize_root_after_intent_cleanup(&mut self) -> io::Result<()>; + + /// Creates or exactly admits persistent `reader.lock`. + /// + /// # Errors + /// + /// Returns the exact creation, open, or verification failure. + fn admit_reader_fence(&mut self) -> io::Result<()>; + + /// Creates or exactly admits the complete version-2 directory prefix. + /// + /// # Errors + /// + /// Returns the exact namespace creation or admission failure. + fn admit_namespace_prefix(&mut self) -> io::Result<()>; + + /// Synchronizes created namespaces and the store root. + /// + /// # Errors + /// + /// Returns the exact directory-synchronization failure. + fn synchronize_root_after_namespace(&mut self) -> io::Result<()>; + + /// Exclusively creates and completely writes `FORMAT.next`. + /// + /// # Errors + /// + /// Returns the exact creation, write, or flush failure. + fn write_marker_stage(&mut self, marker: &CanonicalStoreFormatMarker) -> io::Result<()>; + + /// Synchronizes the complete marker stage. + /// + /// # Errors + /// + /// Returns the exact file-synchronization failure. + fn synchronize_marker_stage(&mut self) -> io::Result<()>; + + /// Links and exactly verifies canonical `FORMAT`. + /// + /// # Errors + /// + /// Returns the exact link, reopen, or verification failure. + fn link_marker(&mut self, marker: &CanonicalStoreFormatMarker) -> io::Result<()>; + + /// Synchronizes the store root after the marker link. + /// + /// # Errors + /// + /// Returns the exact directory-synchronization failure. + fn synchronize_root_after_marker(&mut self) -> io::Result<()>; + + /// Removes only the retained marker stage. + /// + /// # Errors + /// + /// Returns the exact removal failure. + fn remove_marker_stage(&mut self) -> io::Result<()>; + + /// Synchronizes the store root after marker-stage cleanup. + /// + /// # Errors + /// + /// Returns the exact directory-synchronization failure. + fn synchronize_root_after_marker_cleanup(&mut self) -> io::Result<()>; + + /// Exclusively creates and completely writes `migration.receipt.next`. + /// + /// # Errors + /// + /// Returns the exact creation, write, or flush failure. + fn write_receipt_stage(&mut self, receipt: &CanonicalStoreMigrationReceipt) -> io::Result<()>; + + /// Synchronizes the complete receipt stage. + /// + /// # Errors + /// + /// Returns the exact file-synchronization failure. + fn synchronize_receipt_stage(&mut self) -> io::Result<()>; + + /// Links and exactly verifies canonical `migration.receipt`. + /// + /// # Errors + /// + /// Returns the exact link, reopen, or verification failure. + fn link_receipt(&mut self, receipt: &CanonicalStoreMigrationReceipt) -> io::Result<()>; + + /// Synchronizes the store root after the receipt link. + /// + /// # Errors + /// + /// Returns the exact directory-synchronization failure. + fn synchronize_root_after_receipt(&mut self) -> io::Result<()>; + + /// Removes only the retained receipt stage. + /// + /// # Errors + /// + /// Returns the exact removal failure. + fn remove_receipt_stage(&mut self) -> io::Result<()>; + + /// Synchronizes the store root after receipt-stage cleanup. + /// + /// # Errors + /// + /// Returns the exact directory-synchronization failure. + fn synchronize_root_after_receipt_cleanup(&mut self) -> io::Result<()>; +} diff --git a/src/lib.rs b/src/lib.rs index e6a551a..5446185 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -114,13 +114,13 @@ pub use adapters::{ StoreMigrationIntentDigest, StoreMigrationInventoryEntry, StoreMigrationInventoryEntryCount, StoreMigrationInventoryEntryCountError, StoreMigrationInventoryError, StoreMigrationInventoryHasher, StoreMigrationPhase, StoreMigrationReceiptDecodeError, - StoreRootDeviceIdentity, StoreRootFileIdentity, StoreRootMountIdentity, WriterLockAcquireError, - WriterLockAcquirePhase, admit_recovery_stage_bytes, assess_recovery_stage, - classify_recovery_catalog_stage, classify_recovery_names, classify_recovery_next_head_stage, - classify_recovery_segment_stage, execute_recovery_next_head_finalization, - execute_recovery_segment_resume, execute_recovery_stage_completion, - execute_recovery_stage_discard, fingerprint_recovery_stage, initialize_store, - plan_recovery_next_head_finalization, plan_recovery_segment_resume, + StoreMigrationStorage, StoreRootDeviceIdentity, StoreRootFileIdentity, StoreRootMountIdentity, + WriterLockAcquireError, WriterLockAcquirePhase, admit_recovery_stage_bytes, + assess_recovery_stage, classify_recovery_catalog_stage, classify_recovery_names, + classify_recovery_next_head_stage, classify_recovery_segment_stage, + execute_recovery_next_head_finalization, execute_recovery_segment_resume, + execute_recovery_stage_completion, execute_recovery_stage_discard, fingerprint_recovery_stage, + initialize_store, plan_recovery_next_head_finalization, plan_recovery_segment_resume, plan_recovery_stage_completion, plan_recovery_stage_discard, publish_catalog_generation, read_recovery_inventory, }; diff --git a/tests/store_migration_storage.rs b/tests/store_migration_storage.rs new file mode 100644 index 0000000..a1b3142 --- /dev/null +++ b/tests/store_migration_storage.rs @@ -0,0 +1,87 @@ +//! Version-2 store-migration storage capability laws. + +#[path = "store_migration_storage/recording_storage.rs"] +pub mod recording_storage; +mod support; + +use std::error::Error; +use std::io; + +use keep::{ + AdmittedStoreMigrationIntent, CanonicalStoreFormatMarker, CanonicalStoreMigrationIntent, + CanonicalStoreMigrationReceipt, StoreMigrationPhase, StoreMigrationStorage, +}; +use recording_storage::RecordingStorage; + +const INTENT: &str = include_str!("../conformance/segment-store/v2/migration-intent.hex"); + +#[test] +fn storage_port_names_every_migration_phase() -> Result<(), Box> { + let intent_bytes = support::decode_hex(INTENT.trim_end())?; + let admitted = AdmittedStoreMigrationIntent::decode(&intent_bytes)?; + let intent = CanonicalStoreMigrationIntent::from_admitted(&admitted); + let marker = CanonicalStoreFormatMarker::version_two(); + let receipt = CanonicalStoreMigrationReceipt::from_canonical(&intent, &marker); + let mut storage = RecordingStorage::default(); + + exercise_storage(&mut storage, &intent, &marker, &receipt)?; + + assert_eq!(storage.verification_count(), 1); + assert_eq!(storage.observed(), StoreMigrationPhase::ALL); + Ok(()) +} + +fn exercise_storage( + storage: &mut impl StoreMigrationStorage, + intent: &CanonicalStoreMigrationIntent, + marker: &CanonicalStoreFormatMarker, + receipt: &CanonicalStoreMigrationReceipt, +) -> io::Result<()> { + storage.verify_current(intent)?; + exercise_intent(storage, intent)?; + exercise_namespace(storage)?; + exercise_marker(storage, marker)?; + exercise_receipt(storage, receipt) +} + +fn exercise_intent( + storage: &mut impl StoreMigrationStorage, + intent: &CanonicalStoreMigrationIntent, +) -> io::Result<()> { + storage.write_intent_stage(intent)?; + storage.synchronize_intent_stage()?; + storage.link_intent(intent)?; + storage.synchronize_root_after_intent()?; + storage.remove_intent_stage()?; + storage.synchronize_root_after_intent_cleanup() +} + +fn exercise_namespace(storage: &mut impl StoreMigrationStorage) -> io::Result<()> { + storage.admit_reader_fence()?; + storage.admit_namespace_prefix()?; + storage.synchronize_root_after_namespace() +} + +fn exercise_marker( + storage: &mut impl StoreMigrationStorage, + marker: &CanonicalStoreFormatMarker, +) -> io::Result<()> { + storage.write_marker_stage(marker)?; + storage.synchronize_marker_stage()?; + storage.link_marker(marker)?; + storage.synchronize_root_after_marker()?; + storage.remove_marker_stage()?; + storage.synchronize_root_after_marker_cleanup() +} + +fn exercise_receipt( + storage: &mut impl StoreMigrationStorage, + receipt: &CanonicalStoreMigrationReceipt, +) -> io::Result<()> { + storage.write_receipt_stage(receipt)?; + storage.synchronize_receipt_stage()?; + storage.link_receipt(receipt)?; + storage.synchronize_root_after_receipt()?; + storage.remove_receipt_stage()?; + storage.synchronize_root_after_receipt_cleanup() +} diff --git a/tests/store_migration_storage/recording_storage.rs b/tests/store_migration_storage/recording_storage.rs new file mode 100644 index 0000000..277d1b3 --- /dev/null +++ b/tests/store_migration_storage/recording_storage.rs @@ -0,0 +1,146 @@ +//! This module owns the store-migration storage test double. + +use std::io; + +use keep::{ + CanonicalStoreFormatMarker, CanonicalStoreMigrationIntent, CanonicalStoreMigrationReceipt, + StoreMigrationPhase, StoreMigrationStorage, +}; + +#[derive(Default)] +/// Storage port that records every attempted migration phase. +pub struct RecordingStorage { + observed: Vec, + verification_count: usize, +} + +impl RecordingStorage { + /// Returns attempted migration phases in call order. + pub fn observed(&self) -> &[StoreMigrationPhase] { + &self.observed + } + + /// Returns the number of current-state verification attempts. + pub const fn verification_count(&self) -> usize { + self.verification_count + } + + fn record(&mut self, phase: StoreMigrationPhase) { + self.observed.push(phase); + } +} + +impl StoreMigrationStorage for RecordingStorage { + fn verify_current(&mut self, _intent: &CanonicalStoreMigrationIntent) -> io::Result<()> { + self.verification_count = self + .verification_count + .checked_add(1) + .ok_or_else(|| io::Error::other("verification count overflow"))?; + Ok(()) + } + + fn write_intent_stage(&mut self, _intent: &CanonicalStoreMigrationIntent) -> io::Result<()> { + self.record(StoreMigrationPhase::WriteIntentStage); + Ok(()) + } + + fn synchronize_intent_stage(&mut self) -> io::Result<()> { + self.record(StoreMigrationPhase::SynchronizeIntentStage); + Ok(()) + } + + fn link_intent(&mut self, _intent: &CanonicalStoreMigrationIntent) -> io::Result<()> { + self.record(StoreMigrationPhase::LinkIntent); + Ok(()) + } + + fn synchronize_root_after_intent(&mut self) -> io::Result<()> { + self.record(StoreMigrationPhase::SynchronizeRootAfterIntent); + Ok(()) + } + + fn remove_intent_stage(&mut self) -> io::Result<()> { + self.record(StoreMigrationPhase::RemoveIntentStage); + Ok(()) + } + + fn synchronize_root_after_intent_cleanup(&mut self) -> io::Result<()> { + self.record(StoreMigrationPhase::SynchronizeRootAfterIntentCleanup); + Ok(()) + } + + fn admit_reader_fence(&mut self) -> io::Result<()> { + self.record(StoreMigrationPhase::AdmitReaderFence); + Ok(()) + } + + fn admit_namespace_prefix(&mut self) -> io::Result<()> { + self.record(StoreMigrationPhase::AdmitNamespacePrefix); + Ok(()) + } + + fn synchronize_root_after_namespace(&mut self) -> io::Result<()> { + self.record(StoreMigrationPhase::SynchronizeRootAfterNamespace); + Ok(()) + } + + fn write_marker_stage(&mut self, _marker: &CanonicalStoreFormatMarker) -> io::Result<()> { + self.record(StoreMigrationPhase::WriteMarkerStage); + Ok(()) + } + + fn synchronize_marker_stage(&mut self) -> io::Result<()> { + self.record(StoreMigrationPhase::SynchronizeMarkerStage); + Ok(()) + } + + fn link_marker(&mut self, _marker: &CanonicalStoreFormatMarker) -> io::Result<()> { + self.record(StoreMigrationPhase::LinkMarker); + Ok(()) + } + + fn synchronize_root_after_marker(&mut self) -> io::Result<()> { + self.record(StoreMigrationPhase::SynchronizeRootAfterMarker); + Ok(()) + } + + fn remove_marker_stage(&mut self) -> io::Result<()> { + self.record(StoreMigrationPhase::RemoveMarkerStage); + Ok(()) + } + + fn synchronize_root_after_marker_cleanup(&mut self) -> io::Result<()> { + self.record(StoreMigrationPhase::SynchronizeRootAfterMarkerCleanup); + Ok(()) + } + + fn write_receipt_stage(&mut self, _receipt: &CanonicalStoreMigrationReceipt) -> io::Result<()> { + self.record(StoreMigrationPhase::WriteReceiptStage); + Ok(()) + } + + fn synchronize_receipt_stage(&mut self) -> io::Result<()> { + self.record(StoreMigrationPhase::SynchronizeReceiptStage); + Ok(()) + } + + fn link_receipt(&mut self, _receipt: &CanonicalStoreMigrationReceipt) -> io::Result<()> { + self.record(StoreMigrationPhase::LinkReceipt); + Ok(()) + } + + fn synchronize_root_after_receipt(&mut self) -> io::Result<()> { + self.record(StoreMigrationPhase::SynchronizeRootAfterReceipt); + Ok(()) + } + + fn remove_receipt_stage(&mut self) -> io::Result<()> { + self.record(StoreMigrationPhase::RemoveReceiptStage); + Ok(()) + } + + fn synchronize_root_after_receipt_cleanup(&mut self) -> io::Result<()> { + self.record(StoreMigrationPhase::SynchronizeRootAfterReceiptCleanup); + Ok(()) + } +} From 3fc96642894115c3fef4928be9159d8657c7180b Mon Sep 17 00:00:00 2001 From: James Ross Date: Thu, 30 Jul 2026 04:33:26 -0700 Subject: [PATCH 040/111] Add: Execute ordered store migration --- CHANGELOG.md | 2 +- docs/formats/segment-store-v2/recovery.md | 4 +- docs/formats/segment-store-v2/requirements.md | 4 +- src/adapters/store_migration.rs | 4 + .../store_migration/migration_error.rs | 45 ++++++ .../store_migration/migration_execution.rs | 136 ++++++++++++++++++ src/lib.rs | 24 ++-- tests/store_migration_execution.rs | 97 +++++++++++++ .../recording_storage.rs | 91 ++++++------ 9 files changed, 346 insertions(+), 61 deletions(-) create mode 100644 src/adapters/store_migration/migration_error.rs create mode 100644 src/adapters/store_migration/migration_execution.rs create mode 100644 tests/store_migration_execution.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index 24ee878..e907f1e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,7 +14,7 @@ after its public API and format compatibility policies are established. exact catalog, predecessor, root, definition, store, empty-state, and checksum, digest, and synchronization-mask coordinates; migration fuzzing drives all three decoders, streamed inventory is bounded, and `StoreMigrationPhase` - freezes 21 transitions behind explicit blocking storage capabilities. + freezes 21 transitions with explicit storage and verification-first execution. Retention preflight combines expected-generation planning with deterministic closure verification; authority-revalidated 17-phase orchestration returns its receipt after durable cleanup. - Repository crash-matrix execution now terminates isolated writer process diff --git a/docs/formats/segment-store-v2/recovery.md b/docs/formats/segment-store-v2/recovery.md index 6cefd88..0946883 100644 --- a/docs/formats/segment-store-v2/recovery.md +++ b/docs/formats/segment-store-v2/recovery.md @@ -62,8 +62,8 @@ corpus `definition.tsv` bytes. The format-marker digest is BLAKE3-256 of `CanonicalStoreFormatMarker` produces the registered marker; `AdmittedStoreFormatMarker` admits its framing, checksum, definition, and namespace bound. `CanonicalStoreMigrationIntent` retains typed intent coordinates; `CanonicalStoreMigrationReceipt` binds completion; admitted record types verify both. -`StoreMigrationStorage` names current-state verification and all 21 blocking durability capabilities but does not prove the live inventory, physical root, -store version, execution, or recovery of filesystem migration. +`StoreMigrationStorage` names all 21 durability capabilities; `execute_store_migration` verifies current authority first and returns only after final synchronization. +These boundaries do not prove a filesystem implementation or partial-prefix recovery. ## Reader fence diff --git a/docs/formats/segment-store-v2/requirements.md b/docs/formats/segment-store-v2/requirements.md index e9bf0de..551f526 100644 --- a/docs/formats/segment-store-v2/requirements.md +++ b/docs/formats/segment-store-v2/requirements.md @@ -30,11 +30,11 @@ case is not evidence. | --- | --- | --- | --- | | `KEEP-MIGRATION-001` | Exact version-1 stores remain admitted until a durable migration artifact exists | compatibility fixtures | Planned in #19 | | `KEEP-MIGRATION-002` | Format marker, intent, and receipt have complete fixed byte tables, named domains, bounds, checksums, deterministic store identity, and exact initial-state digests | exact admission in `tests/store_format_marker.rs`, `tests/store_migration_intent.rs`, and `tests/store_migration_receipt.rs`; canonical construction in `tests/store_migration_intent_encoding.rs` and `tests/store_migration_receipt_encoding.rs`; seeded `migration_format` fuzz target | Implemented | -| `KEEP-MIGRATION-003` | Migration revalidates version-1 head, catalog, pools, root identity, and writer authority before mutation | bounded canonical pool inventory in `tests/store_migration_inventory.rs`; mandatory verification capability in `tests/store_migration_storage.rs`; filesystem integration remains | In progress in #19 | +| `KEEP-MIGRATION-003` | Migration revalidates version-1 head, catalog, pools, root identity, and writer authority before mutation | bounded canonical pool inventory in `tests/store_migration_inventory.rs`; verification-first execution in `tests/store_migration_execution.rs`; filesystem integration remains | In progress in #19 | | `KEEP-MIGRATION-004` | Every partial migration prefix continues idempotently under writer authority | state-machine and recovery tests | Planned in #19 | | `KEEP-MIGRATION-005` | Unknown, out-of-order, substituted, corrupt, conflicting, or changed evidence is unrecoverable ambiguity | corruption and mutation matrix | Planned in #19 | | `KEEP-MIGRATION-006` | Migration never rewrites or deletes admitted version-1 immutable bytes | byte-for-byte before/after witness | Planned in #19 | -| `KEEP-MIGRATION-007` | Process death around every intent stage, canonical link, namespace prefix, marker stage, receipt stage, cleanup, and synchronization boundary reaches a documented lawful state | ordered `StoreMigrationPhase` vocabulary and matching storage capabilities in `tests/store_migration_phase.rs` and `tests/store_migration_storage.rs`; `KEEP-CRASH-053..=073` process-death matrix remains | In progress in #19 | +| `KEEP-MIGRATION-007` | Process death around every intent stage, canonical link, namespace prefix, marker stage, receipt stage, cleanup, and synchronization boundary reaches a documented lawful state | ordered phases and capabilities in `tests/store_migration_phase.rs` and `tests/store_migration_storage.rs`; exact phase-failure execution in `tests/store_migration_execution.rs`; `KEEP-CRASH-053..=073` process-death matrix remains | In progress in #19 | | `KEEP-MIGRATION-008` | Version-1 admission refuses every version-2 or partial-migration artifact after migration begins | compatibility and fuzz tests | Planned in #19 | diff --git a/src/adapters/store_migration.rs b/src/adapters/store_migration.rs index e73c255..df65667 100644 --- a/src/adapters/store_migration.rs +++ b/src/adapters/store_migration.rs @@ -16,6 +16,8 @@ mod format_marker_encoder; mod immutable_pool_inventory_digest; mod initial_gc_state_digest; mod initial_retention_state_digest; +mod migration_error; +mod migration_execution; mod migration_intent_decode_error; mod migration_intent_decode_error_display; mod migration_intent_decoder; @@ -53,6 +55,8 @@ pub use format_marker_digest::StoreFormatMarkerDigest; pub use immutable_pool_inventory_digest::ImmutablePoolInventoryDigest; pub use initial_gc_state_digest::InitialGcStateDigest; pub use initial_retention_state_digest::InitialRetentionStateDigest; +pub use migration_error::StoreMigrationError; +pub use migration_execution::execute_store_migration; pub use migration_intent_decode_error::StoreMigrationIntentDecodeError; pub use migration_intent_digest::StoreMigrationIntentDigest; pub use migration_inventory_entry::StoreMigrationInventoryEntry; diff --git a/src/adapters/store_migration/migration_error.rs b/src/adapters/store_migration/migration_error.rs new file mode 100644 index 0000000..9fc39e2 --- /dev/null +++ b/src/adapters/store_migration/migration_error.rs @@ -0,0 +1,45 @@ +//! This boundary module owns ordered store-migration execution failures. + +use std::error::Error; +use std::fmt; +use std::io; + +use super::StoreMigrationPhase; + +/// Failure before or during ordered version-2 store migration. +#[derive(Debug)] +pub enum StoreMigrationError { + /// Current version-1 authority could not be revalidated before mutation. + CurrentVerification { + /// Preserved storage refusal. + source: io::Error, + }, + /// One exact durability phase failed. + Storage { + /// Phase attempted when storage refused. + phase: StoreMigrationPhase, + /// Preserved storage refusal. + source: io::Error, + }, +} + +impl fmt::Display for StoreMigrationError { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::CurrentVerification { .. } => { + formatter.write_str("store-migration authority verification failed") + } + Self::Storage { phase, .. } => { + write!(formatter, "store-migration phase {phase} failed") + } + } + } +} + +impl Error for StoreMigrationError { + fn source(&self) -> Option<&(dyn Error + 'static)> { + match self { + Self::CurrentVerification { source } | Self::Storage { source, .. } => Some(source), + } + } +} diff --git a/src/adapters/store_migration/migration_execution.rs b/src/adapters/store_migration/migration_execution.rs new file mode 100644 index 0000000..065fdd3 --- /dev/null +++ b/src/adapters/store_migration/migration_execution.rs @@ -0,0 +1,136 @@ +//! This boundary module owns ordered version-2 store-migration execution. + +use std::io; + +use super::{ + CanonicalStoreFormatMarker, CanonicalStoreMigrationIntent, CanonicalStoreMigrationReceipt, + StoreMigrationError, StoreMigrationPhase, StoreMigrationStorage, +}; + +/// Executes one version-2 migration under revalidated version-1 authority. +/// +/// The returned receipt exists only after all canonical artifacts are visible, +/// all retained stages are removed, and final store-root cleanup is synchronized. +/// +/// # Errors +/// +/// Returns [`StoreMigrationError`] for current-state revalidation or the exact +/// failed durability phase. Failure returns no receipt. +pub fn execute_store_migration( + storage: &mut impl StoreMigrationStorage, + intent: &CanonicalStoreMigrationIntent, +) -> Result { + storage + .verify_current(intent) + .map_err(|source| StoreMigrationError::CurrentVerification { source })?; + let marker = CanonicalStoreFormatMarker::version_two(); + let receipt = CanonicalStoreMigrationReceipt::from_canonical(intent, &marker); + execute_intent(storage, intent)?; + execute_namespace(storage)?; + execute_marker(storage, &marker)?; + execute_receipt(storage, &receipt)?; + Ok(receipt) +} + +fn execute_intent( + storage: &mut impl StoreMigrationStorage, + intent: &CanonicalStoreMigrationIntent, +) -> Result<(), StoreMigrationError> { + require( + storage.write_intent_stage(intent), + StoreMigrationPhase::WriteIntentStage, + )?; + require( + storage.synchronize_intent_stage(), + StoreMigrationPhase::SynchronizeIntentStage, + )?; + require(storage.link_intent(intent), StoreMigrationPhase::LinkIntent)?; + require( + storage.synchronize_root_after_intent(), + StoreMigrationPhase::SynchronizeRootAfterIntent, + )?; + require( + storage.remove_intent_stage(), + StoreMigrationPhase::RemoveIntentStage, + )?; + require( + storage.synchronize_root_after_intent_cleanup(), + StoreMigrationPhase::SynchronizeRootAfterIntentCleanup, + ) +} + +fn execute_namespace(storage: &mut impl StoreMigrationStorage) -> Result<(), StoreMigrationError> { + require( + storage.admit_reader_fence(), + StoreMigrationPhase::AdmitReaderFence, + )?; + require( + storage.admit_namespace_prefix(), + StoreMigrationPhase::AdmitNamespacePrefix, + )?; + require( + storage.synchronize_root_after_namespace(), + StoreMigrationPhase::SynchronizeRootAfterNamespace, + ) +} + +fn execute_marker( + storage: &mut impl StoreMigrationStorage, + marker: &CanonicalStoreFormatMarker, +) -> Result<(), StoreMigrationError> { + require( + storage.write_marker_stage(marker), + StoreMigrationPhase::WriteMarkerStage, + )?; + require( + storage.synchronize_marker_stage(), + StoreMigrationPhase::SynchronizeMarkerStage, + )?; + require(storage.link_marker(marker), StoreMigrationPhase::LinkMarker)?; + require( + storage.synchronize_root_after_marker(), + StoreMigrationPhase::SynchronizeRootAfterMarker, + )?; + require( + storage.remove_marker_stage(), + StoreMigrationPhase::RemoveMarkerStage, + )?; + require( + storage.synchronize_root_after_marker_cleanup(), + StoreMigrationPhase::SynchronizeRootAfterMarkerCleanup, + ) +} + +fn execute_receipt( + storage: &mut impl StoreMigrationStorage, + receipt: &CanonicalStoreMigrationReceipt, +) -> Result<(), StoreMigrationError> { + require( + storage.write_receipt_stage(receipt), + StoreMigrationPhase::WriteReceiptStage, + )?; + require( + storage.synchronize_receipt_stage(), + StoreMigrationPhase::SynchronizeReceiptStage, + )?; + require( + storage.link_receipt(receipt), + StoreMigrationPhase::LinkReceipt, + )?; + require( + storage.synchronize_root_after_receipt(), + StoreMigrationPhase::SynchronizeRootAfterReceipt, + )?; + require( + storage.remove_receipt_stage(), + StoreMigrationPhase::RemoveReceiptStage, + )?; + require( + storage.synchronize_root_after_receipt_cleanup(), + StoreMigrationPhase::SynchronizeRootAfterReceiptCleanup, + ) +} + +fn require(result: io::Result, phase: StoreMigrationPhase) -> Result { + result.map_err(|source| StoreMigrationError::Storage { phase, source }) +} diff --git a/src/lib.rs b/src/lib.rs index 5446185..2c0f2ac 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -110,19 +110,19 @@ pub use adapters::{ SegmentWriteError, SegmentWritePhase, StagedSegment, StorageProfileIdParseError, StoreFormatDefinitionDigest, StoreFormatMarkerDecodeError, StoreFormatMarkerDigest, StoreIdentifier, StoreInitializationError, StoreInitializationPhase, - StoreInitializationReceipt, StoreInitializationStorage, StoreMigrationIntentDecodeError, - StoreMigrationIntentDigest, StoreMigrationInventoryEntry, StoreMigrationInventoryEntryCount, - StoreMigrationInventoryEntryCountError, StoreMigrationInventoryError, - StoreMigrationInventoryHasher, StoreMigrationPhase, StoreMigrationReceiptDecodeError, - StoreMigrationStorage, StoreRootDeviceIdentity, StoreRootFileIdentity, StoreRootMountIdentity, - WriterLockAcquireError, WriterLockAcquirePhase, admit_recovery_stage_bytes, - assess_recovery_stage, classify_recovery_catalog_stage, classify_recovery_names, - classify_recovery_next_head_stage, classify_recovery_segment_stage, + StoreInitializationReceipt, StoreInitializationStorage, StoreMigrationError, + StoreMigrationIntentDecodeError, StoreMigrationIntentDigest, StoreMigrationInventoryEntry, + StoreMigrationInventoryEntryCount, StoreMigrationInventoryEntryCountError, + StoreMigrationInventoryError, StoreMigrationInventoryHasher, StoreMigrationPhase, + StoreMigrationReceiptDecodeError, StoreMigrationStorage, StoreRootDeviceIdentity, + StoreRootFileIdentity, StoreRootMountIdentity, WriterLockAcquireError, WriterLockAcquirePhase, + admit_recovery_stage_bytes, assess_recovery_stage, classify_recovery_catalog_stage, + classify_recovery_names, classify_recovery_next_head_stage, classify_recovery_segment_stage, execute_recovery_next_head_finalization, execute_recovery_segment_resume, - execute_recovery_stage_completion, execute_recovery_stage_discard, fingerprint_recovery_stage, - initialize_store, plan_recovery_next_head_finalization, plan_recovery_segment_resume, - plan_recovery_stage_completion, plan_recovery_stage_discard, publish_catalog_generation, - read_recovery_inventory, + execute_recovery_stage_completion, execute_recovery_stage_discard, execute_store_migration, + fingerprint_recovery_stage, initialize_store, plan_recovery_next_head_finalization, + plan_recovery_segment_resume, plan_recovery_stage_completion, plan_recovery_stage_discard, + publish_catalog_generation, read_recovery_inventory, }; pub use adapters::{ AdmittedRetentionManifest, AdmittedRetentionRoot, CanonicalRetentionHead, diff --git a/tests/store_migration_execution.rs b/tests/store_migration_execution.rs new file mode 100644 index 0000000..1c7f015 --- /dev/null +++ b/tests/store_migration_execution.rs @@ -0,0 +1,97 @@ +//! Version-2 store-migration execution laws. + +#[path = "store_migration_storage/recording_storage.rs"] +pub mod recording_storage; +mod support; + +use std::error::Error; +use std::io; + +use keep::{ + AdmittedStoreMigrationIntent, CanonicalStoreFormatMarker, CanonicalStoreMigrationIntent, + CanonicalStoreMigrationReceipt, StoreMigrationError, StoreMigrationPhase, + execute_store_migration, +}; +use recording_storage::RecordingStorage; + +const INTENT: &str = include_str!("../conformance/segment-store/v2/migration-intent.hex"); + +#[test] +fn migration_executes_every_phase_before_returning_its_receipt() -> Result<(), Box> { + let (intent, marker) = artifacts()?; + let expected = CanonicalStoreMigrationReceipt::from_canonical(&intent, &marker); + let mut storage = RecordingStorage::default(); + + let receipt = execute_store_migration(&mut storage, &intent)?; + + assert_eq!(receipt, expected); + assert_eq!(storage.verification_count(), 1); + assert_eq!(storage.observed(), StoreMigrationPhase::ALL); + Ok(()) +} + +#[test] +fn current_verification_refuses_before_every_migration_phase() -> Result<(), Box> { + let (intent, _marker) = artifacts()?; + let mut storage = RecordingStorage::verification_failure(); + + let Err(error) = execute_store_migration(&mut storage, &intent) else { + return Err("current-state refusal unexpectedly admitted migration".into()); + }; + + match error { + StoreMigrationError::CurrentVerification { source } => { + assert_eq!(source.kind(), io::ErrorKind::PermissionDenied); + } + other @ StoreMigrationError::Storage { .. } => { + return Err(format!("unexpected migration error: {other}").into()); + } + } + assert_eq!(storage.verification_count(), 1); + assert!(storage.observed().is_empty()); + Ok(()) +} + +#[test] +fn every_phase_failure_stops_before_all_later_mutation() -> Result<(), Box> { + let (intent, _marker) = artifacts()?; + for (index, phase) in StoreMigrationPhase::ALL.into_iter().enumerate() { + let mut storage = RecordingStorage::failing_at(phase); + let Err(error) = execute_store_migration(&mut storage, &intent) else { + return Err("injected refusal unexpectedly admitted migration".into()); + }; + assert_storage_error(error, phase)?; + assert_eq!(storage.verification_count(), 1); + let expected = StoreMigrationPhase::ALL + .get(..=index) + .ok_or("migration phase prefix is out of bounds")?; + assert_eq!(storage.observed(), expected); + } + Ok(()) +} + +fn assert_storage_error( + error: StoreMigrationError, + expected_phase: StoreMigrationPhase, +) -> Result<(), Box> { + match error { + StoreMigrationError::Storage { phase, source } => { + assert_eq!(phase, expected_phase); + assert_eq!(source.kind(), io::ErrorKind::Other); + Ok(()) + } + other @ StoreMigrationError::CurrentVerification { .. } => { + Err(format!("unexpected migration error: {other}").into()) + } + } +} + +fn artifacts() -> Result<(CanonicalStoreMigrationIntent, CanonicalStoreFormatMarker), Box> +{ + let intent_bytes = support::decode_hex(INTENT.trim_end())?; + let admitted = AdmittedStoreMigrationIntent::decode(&intent_bytes)?; + Ok(( + CanonicalStoreMigrationIntent::from_admitted(&admitted), + CanonicalStoreFormatMarker::version_two(), + )) +} diff --git a/tests/store_migration_storage/recording_storage.rs b/tests/store_migration_storage/recording_storage.rs index 277d1b3..d3dbf25 100644 --- a/tests/store_migration_storage/recording_storage.rs +++ b/tests/store_migration_storage/recording_storage.rs @@ -12,9 +12,27 @@ use keep::{ pub struct RecordingStorage { observed: Vec, verification_count: usize, + fail_at: Option, + verification_failure: Option, } impl RecordingStorage { + /// Creates storage that refuses at one exact migration phase. + pub fn failing_at(phase: StoreMigrationPhase) -> Self { + Self { + fail_at: Some(phase), + ..Self::default() + } + } + + /// Creates storage that refuses current-state verification. + pub fn verification_failure() -> Self { + Self { + verification_failure: Some(io::ErrorKind::PermissionDenied), + ..Self::default() + } + } + /// Returns attempted migration phases in call order. pub fn observed(&self) -> &[StoreMigrationPhase] { &self.observed @@ -25,8 +43,13 @@ impl RecordingStorage { self.verification_count } - fn record(&mut self, phase: StoreMigrationPhase) { + fn record(&mut self, phase: StoreMigrationPhase) -> io::Result<()> { self.observed.push(phase); + if self.fail_at == Some(phase) { + Err(io::Error::other("injected store-migration failure")) + } else { + Ok(()) + } } } @@ -36,111 +59,91 @@ impl StoreMigrationStorage for RecordingStorage { .verification_count .checked_add(1) .ok_or_else(|| io::Error::other("verification count overflow"))?; - Ok(()) + self.verification_failure + .map_or(Ok(()), |kind| Err(kind.into())) } fn write_intent_stage(&mut self, _intent: &CanonicalStoreMigrationIntent) -> io::Result<()> { - self.record(StoreMigrationPhase::WriteIntentStage); - Ok(()) + self.record(StoreMigrationPhase::WriteIntentStage) } fn synchronize_intent_stage(&mut self) -> io::Result<()> { - self.record(StoreMigrationPhase::SynchronizeIntentStage); - Ok(()) + self.record(StoreMigrationPhase::SynchronizeIntentStage) } fn link_intent(&mut self, _intent: &CanonicalStoreMigrationIntent) -> io::Result<()> { - self.record(StoreMigrationPhase::LinkIntent); - Ok(()) + self.record(StoreMigrationPhase::LinkIntent) } fn synchronize_root_after_intent(&mut self) -> io::Result<()> { - self.record(StoreMigrationPhase::SynchronizeRootAfterIntent); - Ok(()) + self.record(StoreMigrationPhase::SynchronizeRootAfterIntent) } fn remove_intent_stage(&mut self) -> io::Result<()> { - self.record(StoreMigrationPhase::RemoveIntentStage); - Ok(()) + self.record(StoreMigrationPhase::RemoveIntentStage) } fn synchronize_root_after_intent_cleanup(&mut self) -> io::Result<()> { - self.record(StoreMigrationPhase::SynchronizeRootAfterIntentCleanup); - Ok(()) + self.record(StoreMigrationPhase::SynchronizeRootAfterIntentCleanup) } fn admit_reader_fence(&mut self) -> io::Result<()> { - self.record(StoreMigrationPhase::AdmitReaderFence); - Ok(()) + self.record(StoreMigrationPhase::AdmitReaderFence) } fn admit_namespace_prefix(&mut self) -> io::Result<()> { - self.record(StoreMigrationPhase::AdmitNamespacePrefix); - Ok(()) + self.record(StoreMigrationPhase::AdmitNamespacePrefix) } fn synchronize_root_after_namespace(&mut self) -> io::Result<()> { - self.record(StoreMigrationPhase::SynchronizeRootAfterNamespace); - Ok(()) + self.record(StoreMigrationPhase::SynchronizeRootAfterNamespace) } fn write_marker_stage(&mut self, _marker: &CanonicalStoreFormatMarker) -> io::Result<()> { - self.record(StoreMigrationPhase::WriteMarkerStage); - Ok(()) + self.record(StoreMigrationPhase::WriteMarkerStage) } fn synchronize_marker_stage(&mut self) -> io::Result<()> { - self.record(StoreMigrationPhase::SynchronizeMarkerStage); - Ok(()) + self.record(StoreMigrationPhase::SynchronizeMarkerStage) } fn link_marker(&mut self, _marker: &CanonicalStoreFormatMarker) -> io::Result<()> { - self.record(StoreMigrationPhase::LinkMarker); - Ok(()) + self.record(StoreMigrationPhase::LinkMarker) } fn synchronize_root_after_marker(&mut self) -> io::Result<()> { - self.record(StoreMigrationPhase::SynchronizeRootAfterMarker); - Ok(()) + self.record(StoreMigrationPhase::SynchronizeRootAfterMarker) } fn remove_marker_stage(&mut self) -> io::Result<()> { - self.record(StoreMigrationPhase::RemoveMarkerStage); - Ok(()) + self.record(StoreMigrationPhase::RemoveMarkerStage) } fn synchronize_root_after_marker_cleanup(&mut self) -> io::Result<()> { - self.record(StoreMigrationPhase::SynchronizeRootAfterMarkerCleanup); - Ok(()) + self.record(StoreMigrationPhase::SynchronizeRootAfterMarkerCleanup) } fn write_receipt_stage(&mut self, _receipt: &CanonicalStoreMigrationReceipt) -> io::Result<()> { - self.record(StoreMigrationPhase::WriteReceiptStage); - Ok(()) + self.record(StoreMigrationPhase::WriteReceiptStage) } fn synchronize_receipt_stage(&mut self) -> io::Result<()> { - self.record(StoreMigrationPhase::SynchronizeReceiptStage); - Ok(()) + self.record(StoreMigrationPhase::SynchronizeReceiptStage) } fn link_receipt(&mut self, _receipt: &CanonicalStoreMigrationReceipt) -> io::Result<()> { - self.record(StoreMigrationPhase::LinkReceipt); - Ok(()) + self.record(StoreMigrationPhase::LinkReceipt) } fn synchronize_root_after_receipt(&mut self) -> io::Result<()> { - self.record(StoreMigrationPhase::SynchronizeRootAfterReceipt); - Ok(()) + self.record(StoreMigrationPhase::SynchronizeRootAfterReceipt) } fn remove_receipt_stage(&mut self) -> io::Result<()> { - self.record(StoreMigrationPhase::RemoveReceiptStage); - Ok(()) + self.record(StoreMigrationPhase::RemoveReceiptStage) } fn synchronize_root_after_receipt_cleanup(&mut self) -> io::Result<()> { - self.record(StoreMigrationPhase::SynchronizeRootAfterReceiptCleanup); - Ok(()) + self.record(StoreMigrationPhase::SynchronizeRootAfterReceiptCleanup) } } From 117a92acffb9e02dba7eebaa0a258803ec978f11 Mon Sep 17 00:00:00 2001 From: James Ross Date: Thu, 30 Jul 2026 05:52:48 -0700 Subject: [PATCH 041/111] Add: Inventory filesystem migration pools --- CHANGELOG.md | 6 +- .../segment-store-v2/migration-inventory.md | 10 +- docs/formats/segment-store-v2/recovery.md | 2 +- docs/formats/segment-store-v2/requirements.md | 2 +- src/adapters/store_migration.rs | 35 ++++ .../filesystem_inventory_catalog_errors.rs | 107 ++++++++++ .../filesystem_inventory_catalogs.rs | 178 +++++++++++++++++ ...system_inventory_catalogs_refusal_tests.rs | 184 ++++++++++++++++++ ...esystem_inventory_catalogs_test_fixture.rs | 109 +++++++++++ .../filesystem_inventory_catalogs_tests.rs | 35 ++++ .../filesystem_inventory_directory.rs | 96 +++++++++ .../filesystem_inventory_error.rs | 155 +++++++++++++++ .../filesystem_inventory_error_display.rs | 147 ++++++++++++++ .../filesystem_inventory_file.rs | 128 ++++++++++++ .../filesystem_inventory_file_tests.rs | 67 +++++++ .../filesystem_inventory_names.rs | 61 ++++++ .../filesystem_inventory_names_tests.rs | 41 ++++ .../filesystem_inventory_reader.rs | 168 ++++++++++++++++ .../filesystem_inventory_reader_tests.rs | 80 ++++++++ .../filesystem_inventory_segments.rs | 183 +++++++++++++++++ ...system_inventory_segments_refusal_tests.rs | 156 +++++++++++++++ ...esystem_inventory_segments_test_fixture.rs | 79 ++++++++ .../filesystem_inventory_segments_tests.rs | 36 ++++ .../migration_catalog_admission.rs | 106 ++++++++++ .../store_migration/migration_catalog_plan.rs | 51 +++++ .../migration_catalog_records.rs | 93 +++++++++ .../migration_inventory_entry.rs | 11 ++ src/lib.rs | 10 +- 28 files changed, 2326 insertions(+), 10 deletions(-) create mode 100644 src/adapters/store_migration/filesystem_inventory_catalog_errors.rs create mode 100644 src/adapters/store_migration/filesystem_inventory_catalogs.rs create mode 100644 src/adapters/store_migration/filesystem_inventory_catalogs_refusal_tests.rs create mode 100644 src/adapters/store_migration/filesystem_inventory_catalogs_test_fixture.rs create mode 100644 src/adapters/store_migration/filesystem_inventory_catalogs_tests.rs create mode 100644 src/adapters/store_migration/filesystem_inventory_directory.rs create mode 100644 src/adapters/store_migration/filesystem_inventory_error.rs create mode 100644 src/adapters/store_migration/filesystem_inventory_error_display.rs create mode 100644 src/adapters/store_migration/filesystem_inventory_file.rs create mode 100644 src/adapters/store_migration/filesystem_inventory_file_tests.rs create mode 100644 src/adapters/store_migration/filesystem_inventory_names.rs create mode 100644 src/adapters/store_migration/filesystem_inventory_names_tests.rs create mode 100644 src/adapters/store_migration/filesystem_inventory_reader.rs create mode 100644 src/adapters/store_migration/filesystem_inventory_reader_tests.rs create mode 100644 src/adapters/store_migration/filesystem_inventory_segments.rs create mode 100644 src/adapters/store_migration/filesystem_inventory_segments_refusal_tests.rs create mode 100644 src/adapters/store_migration/filesystem_inventory_segments_test_fixture.rs create mode 100644 src/adapters/store_migration/filesystem_inventory_segments_tests.rs create mode 100644 src/adapters/store_migration/migration_catalog_admission.rs create mode 100644 src/adapters/store_migration/migration_catalog_plan.rs create mode 100644 src/adapters/store_migration/migration_catalog_records.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index e907f1e..adf1891 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,8 +13,10 @@ after its public API and format compatibility policies are established. - Version-2 marker, typed canonical intent/receipt construction, and record admission bind exact catalog, predecessor, root, definition, store, empty-state, and checksum, digest, and synchronization-mask coordinates; migration fuzzing drives all - three decoders, streamed inventory is bounded, and `StoreMigrationPhase` - freezes 21 transitions with explicit storage and verification-first execution. + three decoders, streamed inventory is bounded, writer-locked filesystem + inventory completely admits every immutable-pool artifact, and + `StoreMigrationPhase` freezes 21 transitions with explicit storage and + verification-first execution. Retention preflight combines expected-generation planning with deterministic closure verification; authority-revalidated 17-phase orchestration returns its receipt after durable cleanup. - Repository crash-matrix execution now terminates isolated writer process diff --git a/docs/formats/segment-store-v2/migration-inventory.md b/docs/formats/segment-store-v2/migration-inventory.md index f970abb..09c233c 100644 --- a/docs/formats/segment-store-v2/migration-inventory.md +++ b/docs/formats/segment-store-v2/migration-inventory.md @@ -51,5 +51,11 @@ the version-2 corpus `StoreMigrationInventoryEntry` derives canonical bytes only from admitted artifacts. `StoreMigrationInventoryHasher` requires the bounded entry count before streaming, retains only the preceding entry, refuses duplicate or -out-of-order evidence, and reproduces the frozen digest. Capability-relative -filesystem inventory and mutation revalidation remain unimplemented. +out-of-order evidence, and reproduces the frozen digest. + +`FilesystemStoreMigrationInventoryReader` retains exclusive writer authority +and pinned capabilities for both immutable pools. It inventories every regular +entry, including artifacts not reachable from the current publication head, +and reproduces the frozen digest without retaining every artifact body at +once. Migration-session integration that revalidates this inventory +immediately before the first namespace mutation remains in progress. diff --git a/docs/formats/segment-store-v2/recovery.md b/docs/formats/segment-store-v2/recovery.md index 0946883..b526f9b 100644 --- a/docs/formats/segment-store-v2/recovery.md +++ b/docs/formats/segment-store-v2/recovery.md @@ -63,7 +63,7 @@ corpus `definition.tsv` bytes. The format-marker digest is BLAKE3-256 of `CanonicalStoreFormatMarker` produces the registered marker; `AdmittedStoreFormatMarker` admits its framing, checksum, definition, and namespace bound. `CanonicalStoreMigrationIntent` retains typed intent coordinates; `CanonicalStoreMigrationReceipt` binds completion; admitted record types verify both. `StoreMigrationStorage` names all 21 durability capabilities; `execute_store_migration` verifies current authority first and returns only after final synchronization. -These boundaries do not prove a filesystem implementation or partial-prefix recovery. +`FilesystemStoreMigrationInventoryReader` inventories every version-1 immutable artifact under retained writer authority and pinned pool capabilities; migration-session integration and partial-prefix recovery remain unimplemented. ## Reader fence diff --git a/docs/formats/segment-store-v2/requirements.md b/docs/formats/segment-store-v2/requirements.md index 551f526..3ea5b2c 100644 --- a/docs/formats/segment-store-v2/requirements.md +++ b/docs/formats/segment-store-v2/requirements.md @@ -30,7 +30,7 @@ case is not evidence. | --- | --- | --- | --- | | `KEEP-MIGRATION-001` | Exact version-1 stores remain admitted until a durable migration artifact exists | compatibility fixtures | Planned in #19 | | `KEEP-MIGRATION-002` | Format marker, intent, and receipt have complete fixed byte tables, named domains, bounds, checksums, deterministic store identity, and exact initial-state digests | exact admission in `tests/store_format_marker.rs`, `tests/store_migration_intent.rs`, and `tests/store_migration_receipt.rs`; canonical construction in `tests/store_migration_intent_encoding.rs` and `tests/store_migration_receipt_encoding.rs`; seeded `migration_format` fuzz target | Implemented | -| `KEEP-MIGRATION-003` | Migration revalidates version-1 head, catalog, pools, root identity, and writer authority before mutation | bounded canonical pool inventory in `tests/store_migration_inventory.rs`; verification-first execution in `tests/store_migration_execution.rs`; filesystem integration remains | In progress in #19 | +| `KEEP-MIGRATION-003` | Migration revalidates version-1 head, catalog, pools, root identity, and writer authority before mutation | bounded canonical pool inventory in `tests/store_migration_inventory.rs`; writer-locked filesystem pool admission in `filesystem_inventory_*_tests`; verification-first execution in `tests/store_migration_execution.rs`; mutation-time integration remains | In progress in #19 | | `KEEP-MIGRATION-004` | Every partial migration prefix continues idempotently under writer authority | state-machine and recovery tests | Planned in #19 | | `KEEP-MIGRATION-005` | Unknown, out-of-order, substituted, corrupt, conflicting, or changed evidence is unrecoverable ambiguity | corruption and mutation matrix | Planned in #19 | | `KEEP-MIGRATION-006` | Migration never rewrites or deletes admitted version-1 immutable bytes | byte-for-byte before/after witness | Planned in #19 | diff --git a/src/adapters/store_migration.rs b/src/adapters/store_migration.rs index df65667..93cdfc9 100644 --- a/src/adapters/store_migration.rs +++ b/src/adapters/store_migration.rs @@ -7,6 +7,33 @@ mod canonical_format_marker; mod canonical_migration_intent; mod canonical_migration_receipt; mod empty_disposition_set_digest; +mod filesystem_inventory_catalog_errors; +mod filesystem_inventory_catalogs; +#[cfg(test)] +mod filesystem_inventory_catalogs_refusal_tests; +#[cfg(test)] +mod filesystem_inventory_catalogs_test_fixture; +#[cfg(test)] +mod filesystem_inventory_catalogs_tests; +mod filesystem_inventory_directory; +mod filesystem_inventory_error; +mod filesystem_inventory_error_display; +mod filesystem_inventory_file; +#[cfg(test)] +mod filesystem_inventory_file_tests; +mod filesystem_inventory_names; +#[cfg(test)] +mod filesystem_inventory_names_tests; +mod filesystem_inventory_reader; +#[cfg(test)] +mod filesystem_inventory_reader_tests; +mod filesystem_inventory_segments; +#[cfg(test)] +mod filesystem_inventory_segments_refusal_tests; +#[cfg(test)] +mod filesystem_inventory_segments_test_fixture; +#[cfg(test)] +mod filesystem_inventory_segments_tests; mod format_definition_digest; mod format_marker_decode_error; mod format_marker_decode_error_display; @@ -16,6 +43,9 @@ mod format_marker_encoder; mod immutable_pool_inventory_digest; mod initial_gc_state_digest; mod initial_retention_state_digest; +mod migration_catalog_admission; +mod migration_catalog_plan; +mod migration_catalog_records; mod migration_error; mod migration_execution; mod migration_intent_decode_error; @@ -49,6 +79,11 @@ pub use canonical_format_marker::CanonicalStoreFormatMarker; pub use canonical_migration_intent::CanonicalStoreMigrationIntent; pub use canonical_migration_receipt::CanonicalStoreMigrationReceipt; pub use empty_disposition_set_digest::EmptyDispositionSetDigest; +pub use filesystem_inventory_error::{ + FilesystemMigrationInventoryError, FilesystemMigrationInventoryOperation, + MigrationInventoryNamespace, MigrationInventoryPool, +}; +pub use filesystem_inventory_reader::FilesystemStoreMigrationInventoryReader; pub use format_definition_digest::StoreFormatDefinitionDigest; pub use format_marker_decode_error::StoreFormatMarkerDecodeError; pub use format_marker_digest::StoreFormatMarkerDigest; diff --git a/src/adapters/store_migration/filesystem_inventory_catalog_errors.rs b/src/adapters/store_migration/filesystem_inventory_catalog_errors.rs new file mode 100644 index 0000000..7a579e2 --- /dev/null +++ b/src/adapters/store_migration/filesystem_inventory_catalog_errors.rs @@ -0,0 +1,107 @@ +//! This module owns filesystem migration catalog-inventory error translation. + +use super::filesystem_inventory_error::{ + FilesystemMigrationInventoryError, MigrationInventoryPool, +}; +use super::filesystem_inventory_file::FilesystemInventoryFileError; +use super::migration_catalog_admission::MigrationCatalogAdmissionError; +use crate::adapters::{ + CatalogAdmissionError, CatalogRestartError, RecoveryEntryName, SegmentDigest, +}; + +const POOL: MigrationInventoryPool = MigrationInventoryPool::Catalogs; + +pub(super) fn admission( + name: &RecoveryEntryName, + source: MigrationCatalogAdmissionError, +) -> FilesystemMigrationInventoryError { + match source { + MigrationCatalogAdmissionError::Catalog(source) => catalog_admission(name, *source), + MigrationCatalogAdmissionError::SegmentSource { digest, source } => { + referenced_segment(digest, source) + } + MigrationCatalogAdmissionError::SegmentCoordinate { expected, observed } => { + FilesystemMigrationInventoryError::ReferencedSegment { + digest: expected, + source: Box::new(CatalogRestartError::SegmentCoordinate { expected, observed }), + } + } + } +} + +fn catalog_admission( + name: &RecoveryEntryName, + source: CatalogAdmissionError, +) -> FilesystemMigrationInventoryError { + match source { + CatalogAdmissionError::MissingSegment { digest } => { + FilesystemMigrationInventoryError::ReferencedSegment { + digest, + source: Box::new(CatalogRestartError::CatalogAdmission { + source: Box::new(CatalogAdmissionError::MissingSegment { digest }), + }), + } + } + CatalogAdmissionError::Segment { digest, source } => { + FilesystemMigrationInventoryError::ReferencedSegment { + digest, + source: Box::new(CatalogRestartError::Segment { + expected: digest, + source, + }), + } + } + source => artifact( + name, + CatalogRestartError::CatalogAdmission { + source: Box::new(source), + }, + ), + } +} + +pub(super) fn catalog_file( + name: &RecoveryEntryName, + source: FilesystemInventoryFileError, +) -> FilesystemMigrationInventoryError { + match source { + FilesystemInventoryFileError::Artifact(source) => { + FilesystemMigrationInventoryError::Artifact { + pool: POOL, + name: name.clone(), + source, + } + } + FilesystemInventoryFileError::Changed => { + FilesystemMigrationInventoryError::ArtifactChanged { + pool: POOL, + name: name.clone(), + } + } + } +} + +fn referenced_segment( + digest: SegmentDigest, + source: FilesystemInventoryFileError, +) -> FilesystemMigrationInventoryError { + match source { + FilesystemInventoryFileError::Artifact(source) => { + FilesystemMigrationInventoryError::ReferencedSegment { digest, source } + } + FilesystemInventoryFileError::Changed => { + FilesystemMigrationInventoryError::ReferencedSegmentChanged { digest } + } + } +} + +pub(super) fn artifact( + name: &RecoveryEntryName, + source: CatalogRestartError, +) -> FilesystemMigrationInventoryError { + FilesystemMigrationInventoryError::Artifact { + pool: POOL, + name: name.clone(), + source: Box::new(source), + } +} diff --git a/src/adapters/store_migration/filesystem_inventory_catalogs.rs b/src/adapters/store_migration/filesystem_inventory_catalogs.rs new file mode 100644 index 0000000..147a608 --- /dev/null +++ b/src/adapters/store_migration/filesystem_inventory_catalogs.rs @@ -0,0 +1,178 @@ +//! This module owns complete filesystem migration catalog-pool admission. + +use std::collections::TryReserveError; + +use cap_std::fs::Dir; + +use super::StoreMigrationInventoryEntry; +use super::filesystem_inventory_catalog_errors; +use super::filesystem_inventory_error::{ + FilesystemMigrationInventoryError, MigrationInventoryPool, +}; +use super::filesystem_inventory_file::{ + self, FilesystemInventoryFileError, FilesystemInventoryFilePolicy, +}; +use super::filesystem_inventory_names; +use super::filesystem_inventory_segments::FilesystemMigrationSegmentInventory; +use super::migration_catalog_admission::{self, MigrationSegmentLoadError}; +use crate::CatalogLength; +use crate::adapters::{ + CatalogRestartArtifact, CatalogRestartError, CatalogRestartPhase, ChecksummedCatalog, + RecoveryEntryName, SegmentDigest, SegmentReadPolicy, physical_pool_name, recovery_pool_name, +}; + +const POOL: MigrationInventoryPool = MigrationInventoryPool::Catalogs; + +pub(super) struct FilesystemMigrationCatalogInventory { + entries: Vec, + names: Vec, + remaining: u32, +} + +impl FilesystemMigrationCatalogInventory { + pub(super) fn entries(&self) -> &[StoreMigrationInventoryEntry] { + &self.entries + } + + pub(super) const fn len(&self) -> usize { + self.entries.len() + } + + pub(super) fn verify_names( + &self, + directory: &Dir, + ) -> Result<(), FilesystemMigrationInventoryError> { + filesystem_inventory_names::verify(directory, POOL, self.remaining, &self.names) + } +} + +pub(super) fn read( + catalogs: &Dir, + segments: &Dir, + admitted_segments: &FilesystemMigrationSegmentInventory, + remaining: u32, + policy: SegmentReadPolicy, +) -> Result { + let names = filesystem_inventory_names::read(catalogs, POOL, remaining)?; + let capacity = names.len(); + let entry_count = u64::try_from(capacity) + .map_err(|_source| FilesystemMigrationInventoryError::EntryCountHostWidth { pool: POOL })?; + let mut entries = reserve(capacity, entry_count)?; + for name in &names { + entries.push(admit(catalogs, segments, admitted_segments, name, policy)?); + } + entries.sort_unstable(); + Ok(FilesystemMigrationCatalogInventory { + entries, + names, + remaining, + }) +} + +fn reserve( + capacity: usize, + entry_count: u64, +) -> Result, FilesystemMigrationInventoryError> { + let mut values = Vec::new(); + values + .try_reserve_exact(capacity) + .map_err(|source| allocation(entry_count, source))?; + Ok(values) +} + +const fn allocation( + entry_count: u64, + source: TryReserveError, +) -> FilesystemMigrationInventoryError { + FilesystemMigrationInventoryError::Allocation { + pool: POOL, + entry_count, + source, + } +} + +fn admit( + catalogs: &Dir, + segments: &Dir, + admitted_segments: &FilesystemMigrationSegmentInventory, + name: &RecoveryEntryName, + policy: SegmentReadPolicy, +) -> Result { + let (generation, digest) = recovery_pool_name::catalog(name).map_err(|source| { + FilesystemMigrationInventoryError::Name { + pool: POOL, + name: name.clone(), + source, + } + })?; + let encoded = read_catalog(catalogs, name, generation, digest)?; + let catalog = ChecksummedCatalog::decode(&encoded).map_err(|source| { + filesystem_inventory_catalog_errors::artifact(name, CatalogRestartError::Catalog { source }) + })?; + require_coordinate(name, generation, digest, catalog)?; + let admitted = migration_catalog_admission::admit(catalog, policy, |required| { + load_segment(segments, admitted_segments, required) + }) + .map_err(|source| filesystem_inventory_catalog_errors::admission(name, source))?; + Ok(StoreMigrationInventoryEntry::from_migration_catalog( + &admitted, + )) +} + +fn read_catalog( + directory: &Dir, + name: &RecoveryEntryName, + generation: crate::CatalogGeneration, + digest: crate::CatalogDigest, +) -> Result, FilesystemMigrationInventoryError> { + let canonical_name = physical_pool_name::catalog(generation, digest); + let policy = FilesystemInventoryFilePolicy::new( + CatalogRestartArtifact::Catalog, + CatalogRestartPhase::OpenCatalog, + CatalogRestartPhase::ReadCatalog, + CatalogLength::MAXIMUM.get(), + ); + filesystem_inventory_file::read(directory, &canonical_name, policy) + .map_err(|source| filesystem_inventory_catalog_errors::catalog_file(name, source)) +} + +fn require_coordinate( + name: &RecoveryEntryName, + generation: crate::CatalogGeneration, + digest: crate::CatalogDigest, + catalog: ChecksummedCatalog<'_>, +) -> Result<(), FilesystemMigrationInventoryError> { + if catalog.generation() == generation && catalog.digest() == digest { + return Ok(()); + } + Err(filesystem_inventory_catalog_errors::artifact( + name, + CatalogRestartError::CatalogCoordinate { + expected_generation: generation, + observed_generation: catalog.generation(), + expected_length: catalog.length(), + observed_length: catalog.length(), + expected_digest: digest, + observed_digest: catalog.digest(), + }, + )) +} + +fn load_segment( + directory: &Dir, + admitted: &FilesystemMigrationSegmentInventory, + digest: SegmentDigest, +) -> Result, MigrationSegmentLoadError> { + if !admitted.contains(digest) { + return Err(MigrationSegmentLoadError::Missing); + } + let name = physical_pool_name::segment(digest); + let policy = FilesystemInventoryFilePolicy::new( + CatalogRestartArtifact::Segment { digest }, + CatalogRestartPhase::OpenSegment, + CatalogRestartPhase::ReadSegment, + crate::adapters::segment_header::MAXIMUM_SEGMENT_LENGTH, + ); + filesystem_inventory_file::read(directory, &name, policy) + .map_err(MigrationSegmentLoadError::Source) +} diff --git a/src/adapters/store_migration/filesystem_inventory_catalogs_refusal_tests.rs b/src/adapters/store_migration/filesystem_inventory_catalogs_refusal_tests.rs new file mode 100644 index 0000000..8676d40 --- /dev/null +++ b/src/adapters/store_migration/filesystem_inventory_catalogs_refusal_tests.rs @@ -0,0 +1,184 @@ +//! Filesystem migration catalog-pool refusal and orphan laws. + +use std::error::Error; +use std::fs; +use std::io; + +use super::filesystem_inventory_catalogs; +use super::filesystem_inventory_catalogs_test_fixture::{ + CatalogPoolFixture, empty_segment_bytes, maximum_policy, +}; +use super::filesystem_inventory_error::FilesystemMigrationInventoryError; +use super::filesystem_inventory_segments; +use crate::adapters::{ + AdmittedSegment, CatalogAdmissionError, CatalogRestartError, physical_pool_name, +}; + +#[test] +fn unrelated_orphan_segment_remains_in_exact_pool_inventory() -> Result<(), Box> { + let fixture = CatalogPoolFixture::create("migration-catalog-orphan-segment")?; + let orphan_bytes = empty_segment_bytes()?; + let orphan = AdmittedSegment::decode(&orphan_bytes, maximum_policy())?; + fs::write( + fixture + .segments_path() + .join(physical_pool_name::segment(orphan.digest())), + &orphan_bytes, + )?; + let segments_directory = fixture.open_segments()?; + let catalogs_directory = fixture.open_catalogs()?; + let segments = filesystem_inventory_segments::read(&segments_directory, 2, maximum_policy())?; + + let catalogs = filesystem_inventory_catalogs::read( + &catalogs_directory, + &segments_directory, + &segments, + 1, + maximum_policy(), + )?; + assert_eq!(catalogs.entries().len(), 1); + drop(catalogs_directory); + drop(segments_directory); + fixture.remove()?; + Ok(()) +} + +#[test] +fn catalog_missing_its_segment_refuses_inventory() -> Result<(), Box> { + let fixture = CatalogPoolFixture::create("migration-catalog-missing-segment")?; + let missing = AdmittedSegment::decode(fixture.segment_bytes(), maximum_policy())?.digest(); + remove_fixture_segment(&fixture)?; + let segments_directory = fixture.open_segments()?; + let catalogs_directory = fixture.open_catalogs()?; + let segments = filesystem_inventory_segments::read(&segments_directory, 0, maximum_policy())?; + + let error = require_error(filesystem_inventory_catalogs::read( + &catalogs_directory, + &segments_directory, + &segments, + 1, + maximum_policy(), + ))?; + let FilesystemMigrationInventoryError::ReferencedSegment { digest, source } = error else { + return Err(io::Error::other("missing segment returned wrong refusal").into()); + }; + assert_eq!(digest, missing); + assert!(matches!( + source.as_ref(), + CatalogRestartError::CatalogAdmission { + source + } if matches!( + source.as_ref(), + CatalogAdmissionError::MissingSegment { digest } if *digest == missing + ) + )); + drop(catalogs_directory); + drop(segments_directory); + fixture.remove()?; + Ok(()) +} + +#[test] +fn segment_corruption_after_pool_admission_refuses_catalog() -> Result<(), Box> { + let fixture = CatalogPoolFixture::create("migration-catalog-segment-corruption")?; + let segments_directory = fixture.open_segments()?; + let catalogs_directory = fixture.open_catalogs()?; + let segments = filesystem_inventory_segments::read(&segments_directory, 1, maximum_policy())?; + let segment = AdmittedSegment::decode(fixture.segment_bytes(), maximum_policy())?; + fs::write( + fixture + .segments_path() + .join(physical_pool_name::segment(segment.digest())), + b"corrupt", + )?; + + let error = require_error(filesystem_inventory_catalogs::read( + &catalogs_directory, + &segments_directory, + &segments, + 1, + maximum_policy(), + ))?; + let (digest, source) = match error { + FilesystemMigrationInventoryError::ReferencedSegment { digest, source } => (digest, source), + other => { + return Err(io::Error::other(format!( + "corrupt referenced segment returned wrong refusal: {other:?}" + )) + .into()); + } + }; + assert_eq!(digest, segment.digest()); + assert!(matches!( + source.as_ref(), + CatalogRestartError::Segment { + expected, + .. + } if *expected == segment.digest() + )); + drop(catalogs_directory); + drop(segments_directory); + fixture.remove()?; + Ok(()) +} + +#[test] +fn valid_segment_substitution_names_the_referenced_coordinate() -> Result<(), Box> { + let fixture = CatalogPoolFixture::create("migration-catalog-segment-substitution")?; + let segments_directory = fixture.open_segments()?; + let catalogs_directory = fixture.open_catalogs()?; + let segments = filesystem_inventory_segments::read(&segments_directory, 1, maximum_policy())?; + let expected = AdmittedSegment::decode(fixture.segment_bytes(), maximum_policy())?; + let replacement_bytes = empty_segment_bytes()?; + let replacement = AdmittedSegment::decode(&replacement_bytes, maximum_policy())?; + assert_ne!(expected.digest(), replacement.digest()); + fs::write( + fixture + .segments_path() + .join(physical_pool_name::segment(expected.digest())), + &replacement_bytes, + )?; + + let error = require_error(filesystem_inventory_catalogs::read( + &catalogs_directory, + &segments_directory, + &segments, + 1, + maximum_policy(), + ))?; + let FilesystemMigrationInventoryError::ReferencedSegment { digest, source } = error else { + return Err(io::Error::other("segment substitution returned wrong refusal").into()); + }; + assert_eq!(digest, expected.digest()); + assert!(matches!( + source.as_ref(), + CatalogRestartError::SegmentCoordinate { + expected: expected_digest, + observed + } if *expected_digest == expected.digest() && *observed == replacement.digest() + )); + drop(catalogs_directory); + drop(segments_directory); + fixture.remove()?; + Ok(()) +} + +fn remove_fixture_segment(fixture: &CatalogPoolFixture) -> Result<(), Box> { + let segment = AdmittedSegment::decode(fixture.segment_bytes(), maximum_policy())?; + fs::remove_file( + fixture + .segments_path() + .join(physical_pool_name::segment(segment.digest())), + )?; + Ok(()) +} + +fn require_error( + result: Result, +) -> Result { + result.map_or_else(Ok, |_value| { + Err(io::Error::other( + "filesystem migration catalog inventory unexpectedly succeeded", + )) + }) +} diff --git a/src/adapters/store_migration/filesystem_inventory_catalogs_test_fixture.rs b/src/adapters/store_migration/filesystem_inventory_catalogs_test_fixture.rs new file mode 100644 index 0000000..c9a45c3 --- /dev/null +++ b/src/adapters/store_migration/filesystem_inventory_catalogs_test_fixture.rs @@ -0,0 +1,109 @@ +//! Deterministic filesystem migration catalog-pool fixture. + +use std::error::Error; +use std::fs; +use std::path::{Path, PathBuf}; + +use cap_std::ambient_authority; +use cap_std::fs::Dir; + +use crate::LayoutEntryLimit; +use crate::adapters::filesystem_test_sandbox::TestDirectory; +use crate::adapters::test_support::decode_hex; +use crate::adapters::{ + AdmittedSegment, ChecksummedCatalog, SegmentReadPolicy, SegmentRecordLimit, physical_pool_name, +}; + +const SEGMENT_HEX: &str = + include_str!("../../../conformance/segment-store/v1/one-zero-segment.hex"); +const CATALOG_HEX: &str = + include_str!("../../../conformance/segment-store/v1/one-zero-catalog.hex"); +const EMPTY_SEGMENT_HEX: &str = + include_str!("../../../conformance/segment-store/v1/empty-segment.hex"); + +pub(super) struct CatalogPoolFixture { + sandbox: TestDirectory, + segment_bytes: Vec, + catalog_bytes: Vec, +} + +impl CatalogPoolFixture { + pub(super) fn create(name: &str) -> Result> { + let sandbox = TestDirectory::create(name)?; + fs::create_dir(sandbox.path().join("segments"))?; + fs::create_dir(sandbox.path().join("catalogs"))?; + let segment_bytes = decode_hex(SEGMENT_HEX.trim())?; + let catalog_bytes = decode_hex(CATALOG_HEX.trim())?; + let segment = AdmittedSegment::decode(&segment_bytes, maximum_policy())?; + let catalog = ChecksummedCatalog::decode(&catalog_bytes)?; + fs::write( + sandbox + .path() + .join("segments") + .join(physical_pool_name::segment(segment.digest())), + &segment_bytes, + )?; + fs::write( + sandbox + .path() + .join("catalogs") + .join(physical_pool_name::catalog( + catalog.generation(), + catalog.digest(), + )), + &catalog_bytes, + )?; + Ok(Self { + sandbox, + segment_bytes, + catalog_bytes, + }) + } + + pub(super) fn path(&self) -> &Path { + self.sandbox.path() + } + + pub(super) fn segments_path(&self) -> PathBuf { + self.path().join("segments") + } + + pub(super) fn catalogs_path(&self) -> PathBuf { + self.path().join("catalogs") + } + + pub(super) fn open_segments(&self) -> Result> { + Ok(Dir::open_ambient_dir( + self.segments_path(), + ambient_authority(), + )?) + } + + pub(super) fn open_catalogs(&self) -> Result> { + Ok(Dir::open_ambient_dir( + self.catalogs_path(), + ambient_authority(), + )?) + } + + pub(super) fn segment_bytes(&self) -> &[u8] { + &self.segment_bytes + } + + pub(super) fn catalog_bytes(&self) -> &[u8] { + &self.catalog_bytes + } + + pub(super) fn remove(self) -> Result<(), Box> { + self.sandbox.remove()?; + Ok(()) + } +} + +pub(super) const fn maximum_policy() -> SegmentReadPolicy { + SegmentReadPolicy::new(SegmentRecordLimit::MAXIMUM, LayoutEntryLimit::MAXIMUM) +} + +pub(super) fn empty_segment_bytes() -> Result, Box> { + Ok(decode_hex(EMPTY_SEGMENT_HEX.trim())?) +} diff --git a/src/adapters/store_migration/filesystem_inventory_catalogs_tests.rs b/src/adapters/store_migration/filesystem_inventory_catalogs_tests.rs new file mode 100644 index 0000000..08aff5b --- /dev/null +++ b/src/adapters/store_migration/filesystem_inventory_catalogs_tests.rs @@ -0,0 +1,35 @@ +//! Filesystem migration catalog-pool admission laws. + +use std::error::Error; + +use super::StoreMigrationInventoryEntry; +use super::filesystem_inventory_catalogs; +use super::filesystem_inventory_catalogs_test_fixture::{CatalogPoolFixture, maximum_policy}; +use super::filesystem_inventory_segments; +use crate::adapters::{AdmittedSegment, ChecksummedCatalog}; + +#[test] +fn every_catalog_binds_exact_pool_segment_records() -> Result<(), Box> { + let fixture = CatalogPoolFixture::create("migration-catalog-inventory")?; + let segments_directory = fixture.open_segments()?; + let catalogs_directory = fixture.open_catalogs()?; + let segments = filesystem_inventory_segments::read(&segments_directory, 1, maximum_policy())?; + let catalogs = filesystem_inventory_catalogs::read( + &catalogs_directory, + &segments_directory, + &segments, + 1, + maximum_policy(), + )?; + let segment = AdmittedSegment::decode(fixture.segment_bytes(), maximum_policy())?; + let admitted = ChecksummedCatalog::decode(fixture.catalog_bytes())?.admit(&[segment])?; + + assert_eq!( + catalogs.entries(), + &[StoreMigrationInventoryEntry::from_catalog(&admitted)] + ); + drop(catalogs_directory); + drop(segments_directory); + fixture.remove()?; + Ok(()) +} diff --git a/src/adapters/store_migration/filesystem_inventory_directory.rs b/src/adapters/store_migration/filesystem_inventory_directory.rs new file mode 100644 index 0000000..c797f72 --- /dev/null +++ b/src/adapters/store_migration/filesystem_inventory_directory.rs @@ -0,0 +1,96 @@ +//! This module owns pinned migration pool-directory identity. + +use cap_fs_ext::MetadataExt; +use cap_std::fs::{Dir, Metadata}; + +use super::filesystem_inventory_error::{ + FilesystemMigrationInventoryError, FilesystemMigrationInventoryOperation, + MigrationInventoryNamespace, MigrationInventoryPool, +}; +use crate::adapters::sync_capable_directory; + +pub(super) struct PinnedMigrationPoolDirectory { + pool: MigrationInventoryPool, + name: &'static str, + identity: DirectoryIdentity, + directory: Dir, +} + +impl PinnedMigrationPoolDirectory { + pub(super) fn open( + root: &Dir, + pool: MigrationInventoryPool, + name: &'static str, + ) -> Result { + let directory = sync_capable_directory::open(root, name).map_err(|source| { + FilesystemMigrationInventoryError::Io { + namespace: MigrationInventoryNamespace::from(pool), + operation: FilesystemMigrationInventoryOperation::OpenPool, + source, + } + })?; + let identity = DirectoryIdentity::read(&directory).map_err(|source| { + FilesystemMigrationInventoryError::Io { + namespace: MigrationInventoryNamespace::from(pool), + operation: FilesystemMigrationInventoryOperation::OpenPool, + source, + } + })?; + Ok(Self { + pool, + name, + identity, + directory, + }) + } + + pub(super) fn verify(&self, root: &Dir) -> Result<(), FilesystemMigrationInventoryError> { + let handle = DirectoryIdentity::read(&self.directory).map_err(|source| { + FilesystemMigrationInventoryError::Io { + namespace: MigrationInventoryNamespace::from(self.pool), + operation: FilesystemMigrationInventoryOperation::VerifyPool, + source, + } + })?; + let metadata = root.symlink_metadata(self.name).map_err(|source| { + FilesystemMigrationInventoryError::Io { + namespace: MigrationInventoryNamespace::from(self.pool), + operation: FilesystemMigrationInventoryOperation::VerifyPool, + source, + } + })?; + let current = DirectoryIdentity::from(&metadata); + if metadata.is_dir() && handle == self.identity && current == self.identity { + Ok(()) + } else { + Err(FilesystemMigrationInventoryError::NamespaceChanged { pool: self.pool }) + } + } + + pub(super) const fn directory(&self) -> &Dir { + &self.directory + } +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +struct DirectoryIdentity { + device: u64, + inode: u64, +} + +impl DirectoryIdentity { + fn read(directory: &Dir) -> std::io::Result { + directory + .dir_metadata() + .map(|metadata| Self::from(&metadata)) + } +} + +impl From<&Metadata> for DirectoryIdentity { + fn from(metadata: &Metadata) -> Self { + Self { + device: metadata.dev(), + inode: metadata.ino(), + } + } +} diff --git a/src/adapters/store_migration/filesystem_inventory_error.rs b/src/adapters/store_migration/filesystem_inventory_error.rs new file mode 100644 index 0000000..df903b2 --- /dev/null +++ b/src/adapters/store_migration/filesystem_inventory_error.rs @@ -0,0 +1,155 @@ +//! This boundary module owns filesystem migration-inventory failures. + +use std::collections::TryReserveError; +use std::io; + +use super::super::{CatalogRestartError, RecoveryEntryName, RecoveryPoolNameError, SegmentDigest}; + +/// Immutable version-1 pool selected during migration inventory. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum MigrationInventoryPool { + /// The `segments` immutable pool. + Segments, + /// The `catalogs` immutable pool. + Catalogs, +} + +/// Pinned filesystem namespace observed during migration inventory. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum MigrationInventoryNamespace { + /// The writer-authorized store root. + Root, + /// The `segments` immutable pool. + Segments, + /// The `catalogs` immutable pool. + Catalogs, +} + +impl From for MigrationInventoryNamespace { + fn from(pool: MigrationInventoryPool) -> Self { + match pool { + MigrationInventoryPool::Segments => Self::Segments, + MigrationInventoryPool::Catalogs => Self::Catalogs, + } + } +} + +/// Capability-relative directory operation attempted during inventory. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum FilesystemMigrationInventoryOperation { + /// Clone the writer-authorized root capability. + CloneRoot, + /// Open one immutable-pool directory without following links. + OpenPool, + /// Revalidate one pinned immutable-pool directory. + VerifyPool, + /// Count entries under the pinned directory capability. + CountEntries, + /// Read exact raw entry names under the pinned directory capability. + ReadEntryNames, +} + +/// Failure to derive exact migration inventory from immutable filesystem pools. +#[derive(Debug)] +pub enum FilesystemMigrationInventoryError { + /// One capability-relative directory operation failed. + Io { + /// Namespace being observed. + namespace: MigrationInventoryNamespace, + /// Exact failed operation. + operation: FilesystemMigrationInventoryOperation, + /// Preserved filesystem source. + source: io::Error, + }, + /// A pinned immutable-pool directory changed identity. + NamespaceChanged { + /// Pool whose canonical directory entry changed. + pool: MigrationInventoryPool, + }, + /// Exact raw pool membership changed during artifact admission. + EntriesChanged { + /// Pool whose entry-name set changed. + pool: MigrationInventoryPool, + }, + /// A pool exceeded the remaining inventory entry budget. + EntryLimitExceeded { + /// Pool being observed. + pool: MigrationInventoryPool, + /// Remaining entry budget. + maximum: u32, + /// Smallest count observed before enumeration stopped. + observed_at_least: u64, + }, + /// The directory entry count changed between bounded passes. + EntryCountChanged { + /// Pool being observed. + pool: MigrationInventoryPool, + /// Count established by the first pass. + expected: u64, + /// Count established by the second pass. + observed: u64, + }, + /// A host entry count did not fit the protocol representation. + EntryCountHostWidth { + /// Pool being observed. + pool: MigrationInventoryPool, + }, + /// Memory could not retain the bounded semantic inventory. + Allocation { + /// Pool being observed. + pool: MigrationInventoryPool, + /// Exact number of entries requested. + entry_count: u64, + /// Preserved allocation source. + source: TryReserveError, + }, + /// One immutable-pool name was not canonical. + Name { + /// Pool containing the entry. + pool: MigrationInventoryPool, + /// Exact raw name that was refused. + name: RecoveryEntryName, + /// Preserved canonical-name refusal. + source: RecoveryPoolNameError, + }, + /// One named immutable artifact could not be completely admitted. + Artifact { + /// Pool containing the artifact. + pool: MigrationInventoryPool, + /// Exact raw canonical name. + name: RecoveryEntryName, + /// Preserved artifact refusal. + source: Box, + }, + /// A canonical artifact changed identity while it was being admitted. + ArtifactChanged { + /// Pool containing the artifact. + pool: MigrationInventoryPool, + /// Exact raw canonical name. + name: RecoveryEntryName, + }, + /// A catalog-referenced segment could not be completely admitted. + ReferencedSegment { + /// Exact segment coordinate required by the catalog. + digest: SegmentDigest, + /// Preserved segment artifact refusal. + source: Box, + }, + /// A catalog-referenced segment changed identity during admission. + ReferencedSegmentChanged { + /// Exact segment coordinate required by the catalog. + digest: SegmentDigest, + }, + /// Combined pool count arithmetic could not be represented. + EntryCountArithmetic, + /// Combined pool count violated the canonical inventory bound. + EntryCount { + /// Preserved canonical count refusal. + source: super::StoreMigrationInventoryEntryCountError, + }, + /// Canonical entry streaming refused the observed inventory. + Canonical { + /// Preserved canonical inventory refusal. + source: Box, + }, +} diff --git a/src/adapters/store_migration/filesystem_inventory_error_display.rs b/src/adapters/store_migration/filesystem_inventory_error_display.rs new file mode 100644 index 0000000..aa955cf --- /dev/null +++ b/src/adapters/store_migration/filesystem_inventory_error_display.rs @@ -0,0 +1,147 @@ +//! This module owns filesystem migration-inventory error presentation. + +use std::error::Error; +use std::fmt; + +use super::filesystem_inventory_error::{ + FilesystemMigrationInventoryError, FilesystemMigrationInventoryOperation, + MigrationInventoryNamespace, MigrationInventoryPool, +}; + +impl fmt::Display for MigrationInventoryPool { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter.write_str(match self { + Self::Segments => "segments", + Self::Catalogs => "catalogs", + }) + } +} + +impl fmt::Display for FilesystemMigrationInventoryOperation { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter.write_str(match self { + Self::CloneRoot => "clone writer-authorized root", + Self::OpenPool => "open pool capability", + Self::VerifyPool => "verify pool capability", + Self::CountEntries => "count entries", + Self::ReadEntryNames => "read entry names", + }) + } +} + +impl fmt::Display for MigrationInventoryNamespace { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter.write_str(match self { + Self::Root => "root", + Self::Segments => "segments", + Self::Catalogs => "catalogs", + }) + } +} + +impl fmt::Display for FilesystemMigrationInventoryError { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::Io { + namespace, + operation, + .. + } => write!( + formatter, + "migration inventory failed to {operation} in {namespace}" + ), + Self::NamespaceChanged { pool } => { + write!( + formatter, + "migration inventory namespace changed for {pool}" + ) + } + Self::EntriesChanged { pool } => { + write!(formatter, "migration inventory entries changed for {pool}") + } + Self::EntryLimitExceeded { + pool, + maximum, + observed_at_least, + } => write!( + formatter, + "migration inventory observed at least {observed_at_least} entries in {pool}, \ + exceeding remaining limit {maximum}" + ), + Self::EntryCountChanged { + pool, + expected, + observed, + } => write!( + formatter, + "migration inventory entry count changed in {pool}: expected {expected}, \ + observed {observed}" + ), + Self::EntryCountHostWidth { pool } => { + write!( + formatter, + "migration inventory count does not fit for {pool}" + ) + } + Self::Allocation { + pool, entry_count, .. + } => write!( + formatter, + "migration inventory could not retain {entry_count} entries for {pool}" + ), + Self::Name { pool, name, .. } => write!( + formatter, + "migration inventory found noncanonical name {:?} in {pool}", + name.as_bytes() + ), + Self::Artifact { pool, name, .. } => write!( + formatter, + "migration inventory could not admit artifact {:?} in {pool}", + name.as_bytes() + ), + Self::ArtifactChanged { pool, name } => write!( + formatter, + "migration inventory artifact {:?} changed identity in {pool}", + name.as_bytes() + ), + Self::ReferencedSegment { digest, .. } => write!( + formatter, + "migration inventory could not admit catalog segment {digest:?}" + ), + Self::ReferencedSegmentChanged { digest } => write!( + formatter, + "migration inventory catalog segment {digest:?} changed identity" + ), + Self::EntryCountArithmetic => { + formatter.write_str("migration inventory entry count overflowed") + } + Self::EntryCount { .. } => { + formatter.write_str("migration inventory entry count was refused") + } + Self::Canonical { .. } => { + formatter.write_str("migration inventory canonical streaming failed") + } + } + } +} + +impl Error for FilesystemMigrationInventoryError { + fn source(&self) -> Option<&(dyn Error + 'static)> { + match self { + Self::Io { source, .. } => Some(source), + Self::Allocation { source, .. } => Some(source), + Self::Name { source, .. } => Some(source), + Self::Artifact { source, .. } | Self::ReferencedSegment { source, .. } => Some(source), + Self::EntryCount { source } => Some(source), + Self::Canonical { source } => Some(source), + Self::EntryLimitExceeded { .. } + | Self::EntryCountChanged { .. } + | Self::EntryCountHostWidth { .. } + | Self::ArtifactChanged { .. } + | Self::ReferencedSegmentChanged { .. } + | Self::NamespaceChanged { .. } + | Self::EntriesChanged { .. } + | Self::EntryCountArithmetic => None, + } + } +} diff --git a/src/adapters/store_migration/filesystem_inventory_file.rs b/src/adapters/store_migration/filesystem_inventory_file.rs new file mode 100644 index 0000000..7008016 --- /dev/null +++ b/src/adapters/store_migration/filesystem_inventory_file.rs @@ -0,0 +1,128 @@ +//! This module owns identity-stable migration artifact reads. + +use cap_fs_ext::MetadataExt; +use cap_std::fs::{Dir, File, Metadata}; + +use crate::adapters::{ + CatalogRestartArtifact, CatalogRestartError, CatalogRestartPhase, catalog_restart_io, +}; + +pub(super) enum FilesystemInventoryFileError { + Artifact(Box), + Changed, +} + +#[derive(Clone, Copy)] +pub(super) struct FilesystemInventoryFilePolicy { + artifact: CatalogRestartArtifact, + open_phase: CatalogRestartPhase, + read_phase: CatalogRestartPhase, + maximum_length: u64, +} + +impl FilesystemInventoryFilePolicy { + pub(super) const fn new( + artifact: CatalogRestartArtifact, + open_phase: CatalogRestartPhase, + read_phase: CatalogRestartPhase, + maximum_length: u64, + ) -> Self { + Self { + artifact, + open_phase, + read_phase, + maximum_length, + } + } +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +struct FileIdentity { + device: u64, + inode: u64, + length: u64, +} + +impl FileIdentity { + fn read(file: &File, phase: CatalogRestartPhase) -> Result { + file.metadata() + .map(|metadata| Self::from(&metadata)) + .map_err(|source| CatalogRestartError::io(phase, source)) + } +} + +impl From<&Metadata> for FileIdentity { + fn from(metadata: &Metadata) -> Self { + Self { + device: metadata.dev(), + inode: metadata.ino(), + length: metadata.len(), + } + } +} + +pub(super) fn read( + directory: &Dir, + name: &str, + policy: FilesystemInventoryFilePolicy, +) -> Result, FilesystemInventoryFileError> { + read_with(directory, name, policy, || {}) +} + +pub(super) fn read_with( + directory: &Dir, + name: &str, + policy: FilesystemInventoryFilePolicy, + before_verify: F, +) -> Result, FilesystemInventoryFileError> +where + F: FnOnce(), +{ + let (file, length) = + catalog_restart_io::open_regular(directory, name, policy.artifact, policy.open_phase) + .map_err(artifact_error)?; + if length > policy.maximum_length { + return Err(FilesystemInventoryFileError::Artifact(Box::new( + CatalogRestartError::Length { + artifact: policy.artifact, + minimum: 0, + maximum: policy.maximum_length, + observed: length, + }, + ))); + } + let identity = FileIdentity::read(&file, policy.read_phase).map_err(artifact_error)?; + let retained = file + .try_clone() + .map_err(|source| CatalogRestartError::io(policy.read_phase, source)) + .map_err(artifact_error)?; + let encoded = catalog_restart_io::read_exact(file, policy.artifact, policy.read_phase, length) + .map_err(artifact_error)?; + before_verify(); + verify(directory, name, &retained, identity, policy.read_phase)?; + Ok(encoded) +} + +fn verify( + directory: &Dir, + name: &str, + file: &File, + identity: FileIdentity, + phase: CatalogRestartPhase, +) -> Result<(), FilesystemInventoryFileError> { + let handle = FileIdentity::read(file, phase).map_err(artifact_error)?; + let metadata = directory + .symlink_metadata(name) + .map_err(|source| CatalogRestartError::io(phase, source)) + .map_err(artifact_error)?; + let current = FileIdentity::from(&metadata); + if metadata.is_file() && handle == identity && current == identity { + Ok(()) + } else { + Err(FilesystemInventoryFileError::Changed) + } +} + +fn artifact_error(source: CatalogRestartError) -> FilesystemInventoryFileError { + FilesystemInventoryFileError::Artifact(Box::new(source)) +} diff --git a/src/adapters/store_migration/filesystem_inventory_file_tests.rs b/src/adapters/store_migration/filesystem_inventory_file_tests.rs new file mode 100644 index 0000000..5735950 --- /dev/null +++ b/src/adapters/store_migration/filesystem_inventory_file_tests.rs @@ -0,0 +1,67 @@ +//! Identity-stable migration artifact read laws. + +use std::error::Error; +use std::fs; +use std::io; + +use cap_std::ambient_authority; +use cap_std::fs::Dir; + +use super::filesystem_inventory_file::{ + self, FilesystemInventoryFileError, FilesystemInventoryFilePolicy, +}; +use crate::adapters::filesystem_test_sandbox::TestDirectory; +use crate::adapters::{CatalogRestartArtifact, CatalogRestartPhase}; + +#[test] +fn replacement_after_read_refuses_the_opened_artifact() -> Result<(), Box> { + let sandbox = TestDirectory::create("migration-artifact-replacement")?; + let name = "artifact"; + fs::write(sandbox.path().join(name), b"old bytes")?; + let directory = Dir::open_ambient_dir(sandbox.path(), ambient_authority())?; + + let result = filesystem_inventory_file::read_with(&directory, name, policy(9), || { + replace(sandbox.path(), name); + }); + assert!(matches!(result, Err(FilesystemInventoryFileError::Changed))); + drop(directory); + sandbox.remove()?; + Ok(()) +} + +fn replace(root: &std::path::Path, name: &str) { + let renamed = fs::rename(root.join(name), root.join("replaced")); + let written = fs::write(root.join(name), b"old bytes"); + assert!(renamed.is_ok(), "test replacement rename failed"); + assert!(written.is_ok(), "test replacement write failed"); +} + +#[test] +fn regular_file_read_returns_exact_bytes() -> Result<(), Box> { + let sandbox = TestDirectory::create("migration-artifact-read")?; + let name = "artifact"; + fs::write(sandbox.path().join(name), b"exact")?; + let directory = Dir::open_ambient_dir(sandbox.path(), ambient_authority())?; + + let bytes = filesystem_inventory_file::read(&directory, name, policy(5)).map_err(file_error)?; + assert_eq!(bytes, b"exact"); + drop(directory); + sandbox.remove()?; + Ok(()) +} + +fn file_error(error: FilesystemInventoryFileError) -> io::Error { + match error { + FilesystemInventoryFileError::Artifact(source) => io::Error::other(source), + FilesystemInventoryFileError::Changed => io::Error::other("artifact changed"), + } +} + +const fn policy(maximum_length: u64) -> FilesystemInventoryFilePolicy { + FilesystemInventoryFilePolicy::new( + CatalogRestartArtifact::Catalog, + CatalogRestartPhase::OpenCatalog, + CatalogRestartPhase::ReadCatalog, + maximum_length, + ) +} diff --git a/src/adapters/store_migration/filesystem_inventory_names.rs b/src/adapters/store_migration/filesystem_inventory_names.rs new file mode 100644 index 0000000..ce76dcd --- /dev/null +++ b/src/adapters/store_migration/filesystem_inventory_names.rs @@ -0,0 +1,61 @@ +//! This module owns deterministic bounded migration pool-name scans. + +use cap_std::fs::Dir; + +use super::filesystem_inventory_error::{ + FilesystemMigrationInventoryError, FilesystemMigrationInventoryOperation, + MigrationInventoryNamespace, MigrationInventoryPool, +}; +use crate::adapters::{RecoveryEntryName, filesystem_recovery_inventory_scan}; + +pub(super) fn read( + directory: &Dir, + pool: MigrationInventoryPool, + remaining: u32, +) -> Result, FilesystemMigrationInventoryError> { + let expected = + filesystem_recovery_inventory_scan::count_entries(directory, u64::from(remaining)) + .map_err(|source| FilesystemMigrationInventoryError::Io { + namespace: MigrationInventoryNamespace::from(pool), + operation: FilesystemMigrationInventoryOperation::CountEntries, + source, + })?; + if expected > u64::from(remaining) { + return Err(FilesystemMigrationInventoryError::EntryLimitExceeded { + pool, + maximum: remaining, + observed_at_least: expected, + }); + } + let mut names = filesystem_recovery_inventory_scan::read_entry_names(directory, expected) + .map_err(|source| FilesystemMigrationInventoryError::Io { + namespace: MigrationInventoryNamespace::from(pool), + operation: FilesystemMigrationInventoryOperation::ReadEntryNames, + source, + })?; + let observed = u64::try_from(names.len()) + .map_err(|_source| FilesystemMigrationInventoryError::EntryCountHostWidth { pool })?; + if observed != expected { + return Err(FilesystemMigrationInventoryError::EntryCountChanged { + pool, + expected, + observed, + }); + } + names.sort_unstable(); + Ok(names) +} + +pub(super) fn verify( + directory: &Dir, + pool: MigrationInventoryPool, + remaining: u32, + expected: &[RecoveryEntryName], +) -> Result<(), FilesystemMigrationInventoryError> { + let observed = read(directory, pool, remaining)?; + if observed == expected { + Ok(()) + } else { + Err(FilesystemMigrationInventoryError::EntriesChanged { pool }) + } +} diff --git a/src/adapters/store_migration/filesystem_inventory_names_tests.rs b/src/adapters/store_migration/filesystem_inventory_names_tests.rs new file mode 100644 index 0000000..729b0cc --- /dev/null +++ b/src/adapters/store_migration/filesystem_inventory_names_tests.rs @@ -0,0 +1,41 @@ +//! Migration pool-name revalidation laws. + +use std::error::Error; +use std::fs; + +use cap_std::ambient_authority; +use cap_std::fs::Dir; + +use super::filesystem_inventory_error::{ + FilesystemMigrationInventoryError, MigrationInventoryPool, +}; +use super::filesystem_inventory_names; +use crate::adapters::filesystem_test_sandbox::TestDirectory; + +#[test] +fn membership_change_after_scan_refuses_revalidation() -> Result<(), Box> { + let sandbox = TestDirectory::create("migration-name-revalidation")?; + fs::write(sandbox.path().join("first"), b"one")?; + let directory = Dir::open_ambient_dir(sandbox.path(), ambient_authority())?; + let expected = + filesystem_inventory_names::read(&directory, MigrationInventoryPool::Segments, 2)?; + fs::write(sandbox.path().join("second"), b"two")?; + + let error = filesystem_inventory_names::verify( + &directory, + MigrationInventoryPool::Segments, + 2, + &expected, + ) + .err() + .ok_or("changed membership unexpectedly passed")?; + assert!(matches!( + error, + FilesystemMigrationInventoryError::EntriesChanged { + pool: MigrationInventoryPool::Segments + } + )); + drop(directory); + sandbox.remove()?; + Ok(()) +} diff --git a/src/adapters/store_migration/filesystem_inventory_reader.rs b/src/adapters/store_migration/filesystem_inventory_reader.rs new file mode 100644 index 0000000..06b00ea --- /dev/null +++ b/src/adapters/store_migration/filesystem_inventory_reader.rs @@ -0,0 +1,168 @@ +//! This module owns writer-locked filesystem migration pool inventory. + +use cap_std::fs::Dir; + +use super::filesystem_inventory_catalogs; +use super::filesystem_inventory_catalogs::FilesystemMigrationCatalogInventory; +use super::filesystem_inventory_directory::PinnedMigrationPoolDirectory; +use super::filesystem_inventory_error::{ + FilesystemMigrationInventoryError, FilesystemMigrationInventoryOperation, + MigrationInventoryNamespace, MigrationInventoryPool, +}; +use super::filesystem_inventory_segments; +use super::filesystem_inventory_segments::FilesystemMigrationSegmentInventory; +use super::{ + ImmutablePoolInventoryDigest, StoreMigrationInventoryEntryCount, StoreMigrationInventoryHasher, +}; +use crate::adapters::{FilesystemPlatformAdmission, FilesystemWriterLock, SegmentReadPolicy}; + +const SEGMENTS_NAME: &str = "segments"; +const CATALOGS_NAME: &str = "catalogs"; + +/// Writer-authorized reader for one exact version-1 immutable-pool inventory. +/// +/// The reader retains the writer lock and pinned root, segment-pool, and +/// catalog-pool capabilities. It performs no protocol mutation. +#[must_use] +pub struct FilesystemStoreMigrationInventoryReader { + root: Dir, + segments: PinnedMigrationPoolDirectory, + catalogs: PinnedMigrationPoolDirectory, + policy: SegmentReadPolicy, + _lock: FilesystemWriterLock, +} + +impl FilesystemStoreMigrationInventoryReader { + /// Pins both immutable pools under admitted exclusive writer authority. + /// + /// The synchronous call performs bounded capability-relative filesystem + /// I/O and allocates no content-sized memory. + /// + /// # Errors + /// + /// Returns [`FilesystemMigrationInventoryError`] when the root capability + /// cannot be cloned or either canonical pool is missing, linked, replaced, + /// or not a directory. + pub fn open( + admission: FilesystemPlatformAdmission, + policy: SegmentReadPolicy, + ) -> Result { + let lock = admission.into_lock(); + let root = + lock.clone_directory() + .map_err(|source| FilesystemMigrationInventoryError::Io { + namespace: MigrationInventoryNamespace::Root, + operation: FilesystemMigrationInventoryOperation::CloneRoot, + source, + })?; + let segments = PinnedMigrationPoolDirectory::open( + &root, + MigrationInventoryPool::Segments, + SEGMENTS_NAME, + )?; + let catalogs = PinnedMigrationPoolDirectory::open( + &root, + MigrationInventoryPool::Catalogs, + CATALOGS_NAME, + )?; + Ok(Self { + root, + segments, + catalogs, + policy, + _lock: lock, + }) + } + + /// Derives the exact bounded canonical inventory digest. + /// + /// Every segment and catalog pool entry is named canonically, opened + /// without following links, read under the fixed format bound, verified + /// against its physical coordinate, and completely admitted. Catalog + /// record bindings reopen only referenced admitted segment coordinates, so + /// peak content allocation is bounded by one catalog and one segment. + /// + /// The synchronous call may block on filesystem I/O and performs no + /// protocol mutation. + /// + /// # Errors + /// + /// Returns [`FilesystemMigrationInventoryError`] for namespace drift, + /// count drift or overflow, noncanonical names, linked or changed + /// artifacts, malformed content, catalog binding failure, allocation + /// refusal, or canonical digest-stream refusal. + pub fn read(&self) -> Result { + self.verify_directories()?; + let (segments, catalogs) = self.read_pools()?; + let digest = hash_inventory(&segments, &catalogs)?; + segments.verify_names(self.segments.directory())?; + catalogs.verify_names(self.catalogs.directory())?; + self.verify_directories()?; + Ok(digest) + } + + fn read_pools( + &self, + ) -> Result< + ( + FilesystemMigrationSegmentInventory, + FilesystemMigrationCatalogInventory, + ), + FilesystemMigrationInventoryError, + > { + let maximum = StoreMigrationInventoryEntryCount::MAXIMUM; + let segments = + filesystem_inventory_segments::read(self.segments.directory(), maximum, self.policy)?; + let segment_count = host_count(segments.len(), MigrationInventoryPool::Segments)?; + let remaining = maximum + .checked_sub(segment_count) + .ok_or(FilesystemMigrationInventoryError::EntryCountArithmetic)?; + let catalogs = filesystem_inventory_catalogs::read( + self.catalogs.directory(), + self.segments.directory(), + &segments, + remaining, + self.policy, + )?; + Ok((segments, catalogs)) + } + + fn verify_directories(&self) -> Result<(), FilesystemMigrationInventoryError> { + self.segments.verify(&self.root)?; + self.catalogs.verify(&self.root) + } +} + +fn hash_inventory( + segments: &FilesystemMigrationSegmentInventory, + catalogs: &FilesystemMigrationCatalogInventory, +) -> Result { + let segment_count = host_count(segments.len(), MigrationInventoryPool::Segments)?; + let catalog_count = host_count(catalogs.len(), MigrationInventoryPool::Catalogs)?; + let total = segment_count + .checked_add(catalog_count) + .ok_or(FilesystemMigrationInventoryError::EntryCountArithmetic)?; + let count = StoreMigrationInventoryEntryCount::new(total) + .map_err(|source| FilesystemMigrationInventoryError::EntryCount { source })?; + let mut hasher = StoreMigrationInventoryHasher::new(count); + for entry in segments.entries().iter().chain(catalogs.entries()) { + hasher.push(*entry).map_err(canonical_error)?; + } + hasher.finish().map_err(canonical_error) +} + +fn canonical_error( + source: super::StoreMigrationInventoryError, +) -> FilesystemMigrationInventoryError { + FilesystemMigrationInventoryError::Canonical { + source: Box::new(source), + } +} + +fn host_count( + count: usize, + pool: MigrationInventoryPool, +) -> Result { + u32::try_from(count) + .map_err(|_source| FilesystemMigrationInventoryError::EntryCountHostWidth { pool }) +} diff --git a/src/adapters/store_migration/filesystem_inventory_reader_tests.rs b/src/adapters/store_migration/filesystem_inventory_reader_tests.rs new file mode 100644 index 0000000..5087431 --- /dev/null +++ b/src/adapters/store_migration/filesystem_inventory_reader_tests.rs @@ -0,0 +1,80 @@ +//! Writer-locked filesystem migration inventory laws. + +use std::error::Error; +use std::fs; + +use super::filesystem_inventory_reader::FilesystemStoreMigrationInventoryReader; +use super::{FilesystemMigrationInventoryError, MigrationInventoryPool}; +use crate::adapters::filesystem_test_sandbox::TestDirectory; +use crate::adapters::test_support::decode_hex; +use crate::adapters::{ + AdmittedSegment, ChecksummedCatalog, FilesystemPlatformAdmission, physical_pool_name, +}; + +const SEGMENT_HEX: &str = + include_str!("../../../conformance/segment-store/v1/one-zero-segment.hex"); +const CATALOG_HEX: &str = + include_str!("../../../conformance/segment-store/v1/one-zero-catalog.hex"); +const INVENTORY_DIGEST: &str = "40bf5d49c34847ac9cf46a256f343cee80cd980d1405d2dd02ceff8f58d674f9"; + +#[test] +fn writer_locked_pools_reproduce_the_frozen_inventory_digest() -> Result<(), Box> { + let sandbox = TestDirectory::create("migration-inventory-reader")?; + let admission = FilesystemPlatformAdmission::initialize_unchecked_for_tests(sandbox.path())?; + let segment_bytes = decode_hex(SEGMENT_HEX.trim())?; + let catalog_bytes = decode_hex(CATALOG_HEX.trim())?; + let policy = super::filesystem_inventory_catalogs_test_fixture::maximum_policy(); + let segment = AdmittedSegment::decode(&segment_bytes, policy)?; + let catalog = ChecksummedCatalog::decode(&catalog_bytes)?; + fs::write( + sandbox + .path() + .join("segments") + .join(physical_pool_name::segment(segment.digest())), + &segment_bytes, + )?; + fs::write( + sandbox + .path() + .join("catalogs") + .join(physical_pool_name::catalog( + catalog.generation(), + catalog.digest(), + )), + &catalog_bytes, + )?; + + let reader = FilesystemStoreMigrationInventoryReader::open(admission, policy)?; + let digest = reader.read()?; + assert_eq!(digest.as_bytes().as_slice(), decode_hex(INVENTORY_DIGEST)?); + drop(reader); + sandbox.remove()?; + Ok(()) +} + +#[test] +fn replaced_pool_directory_refuses_before_artifact_reads() -> Result<(), Box> { + let sandbox = TestDirectory::create("migration-inventory-directory-replacement")?; + let admission = FilesystemPlatformAdmission::initialize_unchecked_for_tests(sandbox.path())?; + let policy = super::filesystem_inventory_catalogs_test_fixture::maximum_policy(); + let reader = FilesystemStoreMigrationInventoryReader::open(admission, policy)?; + fs::rename( + sandbox.path().join("segments"), + sandbox.path().join("segments-replaced"), + )?; + fs::create_dir(sandbox.path().join("segments"))?; + + let error = reader + .read() + .err() + .ok_or("replaced segment pool unexpectedly passed")?; + assert!(matches!( + error, + FilesystemMigrationInventoryError::NamespaceChanged { + pool: MigrationInventoryPool::Segments + } + )); + drop(reader); + sandbox.remove()?; + Ok(()) +} diff --git a/src/adapters/store_migration/filesystem_inventory_segments.rs b/src/adapters/store_migration/filesystem_inventory_segments.rs new file mode 100644 index 0000000..1964c22 --- /dev/null +++ b/src/adapters/store_migration/filesystem_inventory_segments.rs @@ -0,0 +1,183 @@ +//! This module owns complete filesystem migration segment-pool admission. + +use std::collections::TryReserveError; + +use cap_std::fs::Dir; + +use super::StoreMigrationInventoryEntry; +use super::filesystem_inventory_error::{ + FilesystemMigrationInventoryError, MigrationInventoryPool, +}; +use super::filesystem_inventory_file::{ + self, FilesystemInventoryFileError, FilesystemInventoryFilePolicy, +}; +use super::filesystem_inventory_names; +use crate::adapters::segment_header::MAXIMUM_SEGMENT_LENGTH; +use crate::adapters::{ + AdmittedSegment, CatalogRestartArtifact, CatalogRestartError, CatalogRestartPhase, + RecoveryEntryName, SegmentDigest, SegmentReadPolicy, physical_pool_name, recovery_pool_name, +}; + +const POOL: MigrationInventoryPool = MigrationInventoryPool::Segments; +pub(super) struct FilesystemMigrationSegmentInventory { + entries: Vec, + digests: Vec, + names: Vec, + remaining: u32, +} + +impl FilesystemMigrationSegmentInventory { + pub(super) fn entries(&self) -> &[StoreMigrationInventoryEntry] { + &self.entries + } + + pub(super) fn contains(&self, digest: SegmentDigest) -> bool { + self.digests.binary_search(&digest).is_ok() + } + + pub(super) const fn len(&self) -> usize { + self.entries.len() + } + + pub(super) fn verify_names( + &self, + directory: &Dir, + ) -> Result<(), FilesystemMigrationInventoryError> { + filesystem_inventory_names::verify(directory, POOL, self.remaining, &self.names) + } +} + +pub(super) fn read( + directory: &Dir, + remaining: u32, + policy: SegmentReadPolicy, +) -> Result { + let names = filesystem_inventory_names::read(directory, POOL, remaining)?; + let capacity = names.len(); + let entry_count = u64::try_from(capacity) + .map_err(|_source| FilesystemMigrationInventoryError::EntryCountHostWidth { pool: POOL })?; + let mut entries = reserve(capacity, entry_count)?; + let mut digests = reserve(capacity, entry_count)?; + for name in &names { + let (entry, digest) = admit(directory, name, policy)?; + entries.push(entry); + digests.push(digest); + } + entries.sort_unstable(); + digests.sort_unstable(); + Ok(FilesystemMigrationSegmentInventory { + entries, + digests, + names, + remaining, + }) +} + +fn reserve( + capacity: usize, + entry_count: u64, +) -> Result, FilesystemMigrationInventoryError> { + let mut values = Vec::new(); + values + .try_reserve_exact(capacity) + .map_err(|source| allocation(entry_count, source))?; + Ok(values) +} + +const fn allocation( + entry_count: u64, + source: TryReserveError, +) -> FilesystemMigrationInventoryError { + FilesystemMigrationInventoryError::Allocation { + pool: POOL, + entry_count, + source, + } +} + +fn admit( + directory: &Dir, + name: &RecoveryEntryName, + policy: SegmentReadPolicy, +) -> Result<(StoreMigrationInventoryEntry, SegmentDigest), FilesystemMigrationInventoryError> { + let expected = parse_name(name)?; + let encoded = read_encoded(directory, name, expected)?; + let segment = AdmittedSegment::decode(&encoded, policy).map_err(|source| { + artifact_error( + name, + CatalogRestartError::Segment { + expected, + source: Box::new(source), + }, + ) + })?; + if segment.digest() != expected { + return Err(artifact_error( + name, + CatalogRestartError::SegmentCoordinate { + expected, + observed: segment.digest(), + }, + )); + } + Ok(( + StoreMigrationInventoryEntry::from_segment(&segment), + segment.digest(), + )) +} + +fn parse_name( + name: &RecoveryEntryName, +) -> Result { + recovery_pool_name::segment(name).map_err(|source| FilesystemMigrationInventoryError::Name { + pool: POOL, + name: name.clone(), + source, + }) +} + +fn read_encoded( + directory: &Dir, + name: &RecoveryEntryName, + expected: SegmentDigest, +) -> Result, FilesystemMigrationInventoryError> { + let canonical_name = physical_pool_name::segment(expected); + let artifact = CatalogRestartArtifact::Segment { digest: expected }; + filesystem_inventory_file::read( + directory, + &canonical_name, + FilesystemInventoryFilePolicy::new( + artifact, + CatalogRestartPhase::OpenSegment, + CatalogRestartPhase::ReadSegment, + MAXIMUM_SEGMENT_LENGTH, + ), + ) + .map_err(|source| file_error(name, source)) +} + +fn artifact_error( + name: &RecoveryEntryName, + source: CatalogRestartError, +) -> FilesystemMigrationInventoryError { + FilesystemMigrationInventoryError::Artifact { + pool: POOL, + name: name.clone(), + source: Box::new(source), + } +} + +fn file_error( + name: &RecoveryEntryName, + source: FilesystemInventoryFileError, +) -> FilesystemMigrationInventoryError { + match source { + FilesystemInventoryFileError::Artifact(source) => artifact_error(name, *source), + FilesystemInventoryFileError::Changed => { + FilesystemMigrationInventoryError::ArtifactChanged { + pool: POOL, + name: name.clone(), + } + } + } +} diff --git a/src/adapters/store_migration/filesystem_inventory_segments_refusal_tests.rs b/src/adapters/store_migration/filesystem_inventory_segments_refusal_tests.rs new file mode 100644 index 0000000..1339844 --- /dev/null +++ b/src/adapters/store_migration/filesystem_inventory_segments_refusal_tests.rs @@ -0,0 +1,156 @@ +//! Filesystem migration segment-pool refusal laws. + +use std::error::Error; +use std::fs; +use std::io; + +use super::filesystem_inventory_error::{ + FilesystemMigrationInventoryError, MigrationInventoryPool, +}; +use super::filesystem_inventory_segments; +use super::filesystem_inventory_segments_test_fixture::{ + SegmentPoolFixture, maximum_policy, one_zero_bytes, +}; +use crate::adapters::{ + AdmittedSegment, CatalogRestartError, CatalogRestartPhase, RecoveryPoolNameError, + physical_pool_name, +}; + +#[test] +fn noncanonical_segment_name_refuses_inventory() -> Result<(), Box> { + let fixture = SegmentPoolFixture::create("migration-segment-name-refusal")?; + let bytes = one_zero_bytes()?; + let segment = AdmittedSegment::decode(&bytes, maximum_policy())?; + let canonical = physical_pool_name::segment(segment.digest()); + let stem = canonical + .strip_suffix(".seg") + .ok_or_else(|| io::Error::other("canonical segment name lost its suffix"))?; + fixture.write_named(&format!("{}.seg", stem.to_uppercase()), &bytes)?; + + let pool = fixture.open()?; + let error = require_error(filesystem_inventory_segments::read( + &pool, + 1, + maximum_policy(), + ))?; + assert!(matches!( + error, + FilesystemMigrationInventoryError::Name { + pool: MigrationInventoryPool::Segments, + source: RecoveryPoolNameError::UppercaseDigest, + .. + } + )); + drop(pool); + fixture.remove()?; + Ok(()) +} + +#[test] +fn corrupt_segment_bytes_refuse_inventory() -> Result<(), Box> { + let fixture = SegmentPoolFixture::create("migration-segment-corruption-refusal")?; + let mut bytes = one_zero_bytes()?; + let segment = AdmittedSegment::decode(&bytes, maximum_policy())?; + let name = physical_pool_name::segment(segment.digest()); + let first = bytes + .first_mut() + .ok_or_else(|| io::Error::other("segment fixture is empty"))?; + *first ^= u8::MAX; + fixture.write_named(&name, &bytes)?; + + let pool = fixture.open()?; + let error = require_error(filesystem_inventory_segments::read( + &pool, + 1, + maximum_policy(), + ))?; + let FilesystemMigrationInventoryError::Artifact { + pool: observed_pool, + source, + .. + } = error + else { + return Err(io::Error::other("corrupt segment returned wrong refusal").into()); + }; + assert_eq!(observed_pool, MigrationInventoryPool::Segments); + assert!(matches!( + source.as_ref(), + CatalogRestartError::Segment { .. } + )); + drop(pool); + fixture.remove()?; + Ok(()) +} + +#[cfg(unix)] +#[test] +fn linked_segment_entry_refuses_inventory() -> Result<(), Box> { + use std::os::unix::fs::symlink; + + let fixture = SegmentPoolFixture::create("migration-segment-link-refusal")?; + let bytes = one_zero_bytes()?; + let segment = AdmittedSegment::decode(&bytes, maximum_policy())?; + let name = physical_pool_name::segment(segment.digest()); + fs::write(fixture.path().join("linked-target"), &bytes)?; + symlink("../linked-target", fixture.pool_path().join(name))?; + + let pool = fixture.open()?; + let error = require_error(filesystem_inventory_segments::read( + &pool, + 1, + maximum_policy(), + ))?; + let FilesystemMigrationInventoryError::Artifact { + pool: observed_pool, + source, + .. + } = error + else { + return Err(io::Error::other("linked segment returned wrong refusal").into()); + }; + assert_eq!(observed_pool, MigrationInventoryPool::Segments); + assert!(matches!( + source.as_ref(), + CatalogRestartError::Io { + phase: CatalogRestartPhase::OpenSegment, + .. + } + )); + drop(pool); + fixture.remove()?; + Ok(()) +} + +#[test] +fn segment_pool_above_remaining_limit_refuses_before_names() -> Result<(), Box> { + let fixture = SegmentPoolFixture::create("migration-segment-limit-refusal")?; + fixture.write_named("unknown", b"not admitted")?; + + let pool = fixture.open()?; + let error = require_error(filesystem_inventory_segments::read( + &pool, + 0, + maximum_policy(), + ))?; + assert!(matches!( + error, + FilesystemMigrationInventoryError::EntryLimitExceeded { + pool: MigrationInventoryPool::Segments, + maximum: 0, + observed_at_least: 1, + } + )); + drop(pool); + fixture.remove()?; + Ok(()) +} + +fn require_error( + result: Result, +) -> Result { + result.map_or_else(Ok, |_value| { + Err(io::Error::other( + "filesystem migration segment inventory unexpectedly succeeded", + )) + }) +} diff --git a/src/adapters/store_migration/filesystem_inventory_segments_test_fixture.rs b/src/adapters/store_migration/filesystem_inventory_segments_test_fixture.rs new file mode 100644 index 0000000..7a39904 --- /dev/null +++ b/src/adapters/store_migration/filesystem_inventory_segments_test_fixture.rs @@ -0,0 +1,79 @@ +//! Deterministic filesystem migration segment-pool fixture. + +use std::error::Error; +use std::fs; +use std::path::Path; + +use cap_std::ambient_authority; +use cap_std::fs::Dir; + +use crate::LayoutEntryLimit; +use crate::adapters::filesystem_test_sandbox::TestDirectory; +use crate::adapters::test_support::decode_hex; +use crate::adapters::{ + AdmittedSegment, SegmentDigest, SegmentReadPolicy, SegmentRecordLimit, physical_pool_name, +}; + +const SEGMENT_HEX: &str = + include_str!("../../../conformance/segment-store/v1/one-zero-segment.hex"); +const EMPTY_SEGMENT_HEX: &str = + include_str!("../../../conformance/segment-store/v1/empty-segment.hex"); + +pub(super) struct SegmentPoolFixture { + sandbox: TestDirectory, +} + +impl SegmentPoolFixture { + pub(super) fn create(name: &str) -> Result> { + let sandbox = TestDirectory::create(name)?; + fs::create_dir(sandbox.path().join("segments"))?; + Ok(Self { sandbox }) + } + + pub(super) fn path(&self) -> &Path { + self.sandbox.path() + } + + pub(super) fn pool_path(&self) -> std::path::PathBuf { + self.path().join("segments") + } + + pub(super) fn open(&self) -> Result> { + Ok(Dir::open_ambient_dir( + self.pool_path(), + ambient_authority(), + )?) + } + + pub(super) fn write_segment(&self, bytes: &[u8]) -> Result> { + let segment = AdmittedSegment::decode(bytes, maximum_policy())?; + fs::write( + self.pool_path() + .join(physical_pool_name::segment(segment.digest())), + bytes, + )?; + Ok(segment.digest()) + } + + pub(super) fn write_named(&self, name: &str, bytes: &[u8]) -> Result<(), Box> { + fs::write(self.pool_path().join(name), bytes)?; + Ok(()) + } + + pub(super) fn remove(self) -> Result<(), Box> { + self.sandbox.remove()?; + Ok(()) + } +} + +pub(super) fn one_zero_bytes() -> Result, Box> { + Ok(decode_hex(SEGMENT_HEX.trim())?) +} + +pub(super) fn empty_bytes() -> Result, Box> { + Ok(decode_hex(EMPTY_SEGMENT_HEX.trim())?) +} + +pub(super) const fn maximum_policy() -> SegmentReadPolicy { + SegmentReadPolicy::new(SegmentRecordLimit::MAXIMUM, LayoutEntryLimit::MAXIMUM) +} diff --git a/src/adapters/store_migration/filesystem_inventory_segments_tests.rs b/src/adapters/store_migration/filesystem_inventory_segments_tests.rs new file mode 100644 index 0000000..caa92ac --- /dev/null +++ b/src/adapters/store_migration/filesystem_inventory_segments_tests.rs @@ -0,0 +1,36 @@ +//! Filesystem migration segment-pool admission laws. + +use std::error::Error; + +use super::StoreMigrationInventoryEntry; +use super::filesystem_inventory_segments; +use super::filesystem_inventory_segments_test_fixture::{ + SegmentPoolFixture, empty_bytes, maximum_policy, one_zero_bytes, +}; +use crate::adapters::AdmittedSegment; + +#[test] +fn every_canonical_segment_is_admitted_into_migration_inventory() -> Result<(), Box> { + let fixture = SegmentPoolFixture::create("migration-segment-inventory")?; + let first_bytes = one_zero_bytes()?; + let second_bytes = empty_bytes()?; + let first = AdmittedSegment::decode(&first_bytes, maximum_policy())?; + let second = AdmittedSegment::decode(&second_bytes, maximum_policy())?; + let _first_digest = fixture.write_segment(&first_bytes)?; + let _second_digest = fixture.write_segment(&second_bytes)?; + + let pool = fixture.open()?; + let inventory = filesystem_inventory_segments::read(&pool, 2, maximum_policy())?; + let mut expected = [ + StoreMigrationInventoryEntry::from_segment(&first), + StoreMigrationInventoryEntry::from_segment(&second), + ]; + expected.sort_unstable(); + + assert_eq!(inventory.entries(), expected.as_slice()); + assert!(inventory.contains(first.digest())); + assert!(inventory.contains(second.digest())); + drop(pool); + fixture.remove()?; + Ok(()) +} diff --git a/src/adapters/store_migration/migration_catalog_admission.rs b/src/adapters/store_migration/migration_catalog_admission.rs new file mode 100644 index 0000000..bcb97b2 --- /dev/null +++ b/src/adapters/store_migration/migration_catalog_admission.rs @@ -0,0 +1,106 @@ +//! This module owns bounded catalog admission for migration pool inventory. + +use crate::adapters::{ + AdmittedSegment, CatalogAdmissionError, ChecksummedCatalog, SegmentDigest, SegmentReadPolicy, +}; +use crate::{CatalogDigest, CatalogGeneration, CatalogLength}; + +use super::{migration_catalog_plan, migration_catalog_records}; + +pub(super) struct AdmittedMigrationCatalog<'a> { + catalog: ChecksummedCatalog<'a>, +} + +impl AdmittedMigrationCatalog<'_> { + pub(super) const fn generation(&self) -> CatalogGeneration { + self.catalog.generation() + } + + pub(super) const fn length(&self) -> CatalogLength { + self.catalog.length() + } + + pub(super) const fn digest(&self) -> CatalogDigest { + self.catalog.digest() + } +} + +pub(super) enum MigrationCatalogAdmissionError { + Catalog(Box), + SegmentSource { + digest: SegmentDigest, + source: E, + }, + SegmentCoordinate { + expected: SegmentDigest, + observed: SegmentDigest, + }, +} + +pub(super) enum MigrationSegmentLoadError { + Missing, + Source(E), +} + +pub(super) fn admit( + catalog: ChecksummedCatalog<'_>, + policy: SegmentReadPolicy, + mut load: F, +) -> Result, MigrationCatalogAdmissionError> +where + F: FnMut(SegmentDigest) -> Result, MigrationSegmentLoadError>, +{ + let mut plan = migration_catalog_plan::plan(catalog)?; + plan.sort_unstable_by_key(|entry| entry.physical_order()); + for entries in + plan.chunk_by(|first, second| first.entry.segment_digest() == second.entry.segment_digest()) + { + admit_segment(entries, policy, &mut load)?; + } + Ok(AdmittedMigrationCatalog { catalog }) +} + +fn admit_segment( + entries: &[migration_catalog_plan::PlannedEntry], + policy: SegmentReadPolicy, + load: &mut F, +) -> Result<(), MigrationCatalogAdmissionError> +where + F: FnMut(SegmentDigest) -> Result, MigrationSegmentLoadError>, +{ + let Some(first) = entries.first() else { + return Ok(()); + }; + let expected = first.entry.segment_digest(); + let encoded = match load(expected) { + Ok(encoded) => encoded, + Err(MigrationSegmentLoadError::Missing) => { + return Err(catalog_error(CatalogAdmissionError::MissingSegment { + digest: expected, + })); + } + Err(MigrationSegmentLoadError::Source(source)) => { + return Err(MigrationCatalogAdmissionError::SegmentSource { + digest: expected, + source, + }); + } + }; + let segment = AdmittedSegment::decode(&encoded, policy).map_err(|source| { + catalog_error(CatalogAdmissionError::Segment { + digest: expected, + source: Box::new(source), + }) + })?; + if segment.digest() != expected { + return Err(MigrationCatalogAdmissionError::SegmentCoordinate { + expected, + observed: segment.digest(), + }); + } + migration_catalog_records::validate(entries, &segment).map_err(catalog_error) +} + +pub(super) fn catalog_error(source: CatalogAdmissionError) -> MigrationCatalogAdmissionError { + MigrationCatalogAdmissionError::Catalog(Box::new(source)) +} diff --git a/src/adapters/store_migration/migration_catalog_plan.rs b/src/adapters/store_migration/migration_catalog_plan.rs new file mode 100644 index 0000000..deefce8 --- /dev/null +++ b/src/adapters/store_migration/migration_catalog_plan.rs @@ -0,0 +1,51 @@ +//! This module owns bounded physical catalog-entry planning for migration. + +use crate::adapters::{ + CatalogAdmissionError, CatalogAllocationPhase, ChecksummedCatalog, DecodedCatalogEntry, + SegmentDigest, +}; + +use super::migration_catalog_admission::{MigrationCatalogAdmissionError, catalog_error}; + +#[derive(Clone, Copy)] +pub(super) struct PlannedEntry { + pub(super) ordinal: usize, + pub(super) entry: DecodedCatalogEntry, +} + +impl PlannedEntry { + pub(super) const fn physical_order(self) -> (SegmentDigest, u64, usize) { + ( + self.entry.segment_digest(), + self.entry.record_offset(), + self.ordinal, + ) + } +} + +pub(super) fn plan( + catalog: ChecksummedCatalog<'_>, +) -> Result, MigrationCatalogAdmissionError> { + let requested = usize::try_from(catalog.entry_count()).map_err(|_source| { + catalog_error(CatalogAdmissionError::EntryCountHostWidth { + observed: catalog.entry_count(), + }) + })?; + let mut plan = Vec::new(); + plan.try_reserve_exact(requested).map_err(|source| { + catalog_error(CatalogAdmissionError::Allocation { + phase: CatalogAllocationPhase::EntryPlan, + requested, + source, + }) + })?; + let entries = catalog + .entries() + .map_err(|source| catalog_error(CatalogAdmissionError::Catalog { source }))?; + for (ordinal, entry) in entries.enumerate() { + let entry = + entry.map_err(|source| catalog_error(CatalogAdmissionError::Catalog { source }))?; + plan.push(PlannedEntry { ordinal, entry }); + } + Ok(plan) +} diff --git a/src/adapters/store_migration/migration_catalog_records.rs b/src/adapters/store_migration/migration_catalog_records.rs new file mode 100644 index 0000000..0156c87 --- /dev/null +++ b/src/adapters/store_migration/migration_catalog_records.rs @@ -0,0 +1,93 @@ +//! This module owns exact migration catalog-to-segment record binding. + +use crate::adapters::{ + AdmittedSegment, AdmittedSegmentRecord, CatalogAdmissionError, DecodedCatalogEntry, +}; + +use super::migration_catalog_plan::PlannedEntry; + +pub(super) fn validate( + entries: &[PlannedEntry], + segment: &AdmittedSegment<'_>, +) -> Result<(), CatalogAdmissionError> { + let digest = segment.digest(); + let mut pending = entries.iter().peekable(); + let mut cursor = segment.record_cursor(); + while let Some(located) = + cursor + .next_record() + .map_err(|source| CatalogAdmissionError::Segment { + digest, + source: Box::new(source), + })? + { + refuse_preceding(&mut pending, located.offset)?; + validate_at_offset(&mut pending, located.offset, located.record)?; + } + cursor + .finish() + .map_err(|source| CatalogAdmissionError::Segment { + digest, + source: Box::new(source), + })?; + pending + .next() + .map_or(Ok(()), |entry| Err(location_error(entry.entry))) +} + +fn refuse_preceding( + pending: &mut std::iter::Peekable>, + record_offset: u64, +) -> Result<(), CatalogAdmissionError> { + match pending.peek() { + Some(entry) if entry.entry.record_offset() < record_offset => { + Err(location_error(entry.entry)) + } + Some(_) | None => Ok(()), + } +} + +fn validate_at_offset( + pending: &mut std::iter::Peekable>, + record_offset: u64, + record: AdmittedSegmentRecord<'_>, +) -> Result<(), CatalogAdmissionError> { + while matches!(pending.peek(), Some(entry) if entry.entry.record_offset() == record_offset) { + let Some(entry) = pending.next() else { + break; + }; + validate_record(entry.entry, record)?; + } + Ok(()) +} + +fn validate_record( + entry: DecodedCatalogEntry, + record: AdmittedSegmentRecord<'_>, +) -> Result<(), CatalogAdmissionError> { + if record.header().record_length() != entry.record_length() { + return Err(location_error(entry)); + } + if record.identity() != entry.identity() { + return Err(CatalogAdmissionError::RecordIdentityMismatch { + expected: entry.identity(), + observed: record.identity(), + }); + } + if record.checksum() != entry.checksum() { + return Err(CatalogAdmissionError::RecordChecksumMismatch { + expected: entry.checksum(), + observed: record.checksum(), + }); + } + Ok(()) +} + +const fn location_error(entry: DecodedCatalogEntry) -> CatalogAdmissionError { + CatalogAdmissionError::LocationNotTopLevel { + identity: entry.identity(), + segment_digest: entry.segment_digest(), + record_offset: entry.record_offset(), + record_length: entry.record_length().get(), + } +} diff --git a/src/adapters/store_migration/migration_inventory_entry.rs b/src/adapters/store_migration/migration_inventory_entry.rs index 1ce8bc0..e04b89f 100644 --- a/src/adapters/store_migration/migration_inventory_entry.rs +++ b/src/adapters/store_migration/migration_inventory_entry.rs @@ -2,6 +2,8 @@ use crate::{AdmittedCatalog, AdmittedSegment}; +use super::migration_catalog_admission::AdmittedMigrationCatalog; + const SEGMENT_KIND: u8 = 1; const CATALOG_KIND: u8 = 2; const ENCODED_LENGTH: usize = 56; @@ -32,6 +34,15 @@ impl StoreMigrationInventoryEntry { )) } + pub(super) const fn from_migration_catalog(catalog: &AdmittedMigrationCatalog<'_>) -> Self { + Self(encode( + CATALOG_KIND, + catalog.generation().get(), + catalog.length().get(), + catalog.digest().as_bytes(), + )) + } + /// Returns the exact 56 canonical bytes. pub const fn encoded(&self) -> &[u8; ENCODED_LENGTH] { &self.0 diff --git a/src/lib.rs b/src/lib.rs index 2c0f2ac..f82f81d 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -68,16 +68,18 @@ pub use adapters::{ CatalogSnapshotError, CatalogSuccessor, CatalogTransitionError, ChecksummedCatalog, ChecksummedPublicationHead, ChecksummedSegmentRecord, ClosedSegment, EmptyDispositionSetDigest, FilesystemCatalogPublicationError, FilesystemCatalogPublisher, FilesystemCatalogSnapshot, + FilesystemMigrationInventoryError, FilesystemMigrationInventoryOperation, FilesystemPlatformAdmission, FilesystemPlatformAdmissionError, FilesystemRecoveryInventoryReader, FilesystemRecoveryNextHeadFinalizationOpenError, FilesystemRecoveryNextHeadFinalizer, FilesystemRecoverySegmentResumeOpenError, FilesystemRecoverySegmentResumer, FilesystemRecoverySegmentStage, FilesystemRecoveryStageCompleter, FilesystemRecoveryStageCompletionOpenError, FilesystemRecoveryStageDiscardOpenError, FilesystemRecoveryStageDiscarder, - FilesystemRecoveryStageError, FilesystemSegmentStage, FilesystemWriterLock, - ImmutablePoolInventoryDigest, InitialGcStateDigest, InitialRetentionStateDigest, - LayoutDecodeError, LayoutDecodePolicy, LayoutEncodeError, LayoutIdBinaryParseError, - LayoutIdTextParseError, MigrationSynchronizationMask, OpenedReusableSegment, + FilesystemRecoveryStageError, FilesystemSegmentStage, FilesystemStoreMigrationInventoryReader, + FilesystemWriterLock, ImmutablePoolInventoryDigest, InitialGcStateDigest, + InitialRetentionStateDigest, LayoutDecodeError, LayoutDecodePolicy, LayoutEncodeError, + LayoutIdBinaryParseError, LayoutIdTextParseError, MigrationInventoryNamespace, + MigrationInventoryPool, MigrationSynchronizationMask, OpenedReusableSegment, PublicationHeadDecodeError, RecoveryCatalogStage, RecoveryCatalogStageError, RecoveryEntryName, RecoveryEntryNameError, RecoveryEntryRole, RecoveryInventory, RecoveryInventoryEntry, RecoveryInventoryError, RecoveryInventoryLimit, RecoveryInventoryLimitError, From 991264f4b03342bc8110416e0ad2a773061ee776 Mon Sep 17 00:00:00 2001 From: James Ross Date: Thu, 30 Jul 2026 06:47:41 -0700 Subject: [PATCH 042/111] Add: Observe filesystem migration authority --- CHANGELOG.md | 4 +- .../segment-store-v2/migration-inventory.md | 9 +- docs/formats/segment-store-v2/recovery.md | 50 +++--- docs/formats/segment-store-v2/requirements.md | 4 +- src/adapters/filesystem_catalog_publisher.rs | 4 +- src/adapters/filesystem_platform_admission.rs | 33 +++- src/adapters/filesystem_platform_profile.rs | 114 +++++-------- .../filesystem_platform_profile_tests.rs | 84 +++++++++ src/adapters/filesystem_root_identity.rs | 30 ++++ src/adapters/filesystem_store_initializer.rs | 19 ++- src/adapters/mod.rs | 1 + src/adapters/store_migration.rs | 12 ++ .../canonical_migration_intent.rs | 20 ++- .../filesystem_inventory_reader.rs | 17 +- .../filesystem_migration_authority.rs | 160 ++++++++++++++++++ .../filesystem_migration_authority_error.rs | 117 +++++++++++++ ...ystem_migration_authority_error_display.rs | 112 ++++++++++++ .../filesystem_migration_authority_tests.rs | 109 ++++++++++++ ...lesystem_migration_authority_validation.rs | 80 +++++++++ .../migration_catalog_coordinates.rs | 52 ++++++ .../migration_intent_encoder.rs | 46 ++--- .../store_migration/store_root_identity.rs | 43 +++++ src/lib.rs | 51 +++--- 23 files changed, 997 insertions(+), 174 deletions(-) create mode 100644 src/adapters/filesystem_platform_profile_tests.rs create mode 100644 src/adapters/filesystem_root_identity.rs create mode 100644 src/adapters/store_migration/filesystem_migration_authority.rs create mode 100644 src/adapters/store_migration/filesystem_migration_authority_error.rs create mode 100644 src/adapters/store_migration/filesystem_migration_authority_error_display.rs create mode 100644 src/adapters/store_migration/filesystem_migration_authority_tests.rs create mode 100644 src/adapters/store_migration/filesystem_migration_authority_validation.rs create mode 100644 src/adapters/store_migration/migration_catalog_coordinates.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index adf1891..72be8af 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,7 +14,9 @@ after its public API and format compatibility policies are established. exact catalog, predecessor, root, definition, store, empty-state, and checksum, digest, and synchronization-mask coordinates; migration fuzzing drives all three decoders, streamed inventory is bounded, writer-locked filesystem - inventory completely admits every immutable-pool artifact, and + inventory completely admits every immutable-pool artifact, filesystem + migration authority derives and revalidates one canonical intent from exact + Linux root, namespace, head, catalog, and inventory coordinates, and `StoreMigrationPhase` freezes 21 transitions with explicit storage and verification-first execution. Retention preflight combines expected-generation planning with deterministic diff --git a/docs/formats/segment-store-v2/migration-inventory.md b/docs/formats/segment-store-v2/migration-inventory.md index 09c233c..2eed8dc 100644 --- a/docs/formats/segment-store-v2/migration-inventory.md +++ b/docs/formats/segment-store-v2/migration-inventory.md @@ -57,5 +57,10 @@ out-of-order evidence, and reproduces the frozen digest. and pinned capabilities for both immutable pools. It inventories every regular entry, including artifacts not reachable from the current publication head, and reproduces the frozen digest without retaining every artifact body at -once. Migration-session integration that revalidates this inventory -immediately before the first namespace mutation remains in progress. +once. `FilesystemStoreMigrationAuthority` combines that digest with an +identity-stable fixed-width `HEAD`, its selected admitted catalog, the exact +version-1 root namespace, and the admitted physical root coordinates. Its +`verify_current` operation repeats the complete observation and refuses any +different canonical intent before mutation. Filesystem migration storage that +invokes this verification immediately before its first namespace mutation +remains in progress. diff --git a/docs/formats/segment-store-v2/recovery.md b/docs/formats/segment-store-v2/recovery.md index b526f9b..9e56d4a 100644 --- a/docs/formats/segment-store-v2/recovery.md +++ b/docs/formats/segment-store-v2/recovery.md @@ -1,13 +1,10 @@ # Migration and Recovery -This page owns the version-2 filesystem namespace, format marker, reader fence, -one-way migration, fixed-stage recovery, GC reservation, and -recovery-disposition reservation. +This page owns version-2 filesystem migration and recovery. ## Exact filesystem namespace -Version 2 preserves the version-1 files and directories and admits these new -coordinates: +Version 2 preserves version-1 files and directories; it adds: ```text reader.lock @@ -50,8 +47,7 @@ Operations are capability-relative and never follow links. | 60 | 4 | reserved | zero | | 64 | 32 | checksum | BLAKE3-256 over bytes `0..64` | -The definition and checksum domains are -`keep.segment-store-definition/v2\0` and +The definition and checksum domains are `keep.segment-store-definition/v2\0` and `keep.segment-store-marker-checksum/v2\0`. A missing marker is version 1 only when the exact version-1 namespace admits. An unsupported, corrupt, substituted, or same-name/different-digest marker refuses. @@ -84,7 +80,6 @@ published segment. ## Migration records Migration is a one-way explicit migration under exclusive writer authority. -Version 1 is never extended in place without durable migration evidence. `migration.intent` is exactly 256 bytes: @@ -101,9 +96,9 @@ Version 1 is never extended in place without durable migration evidence. | 40 | 32 | catalog digest named by version-1 `HEAD` | exact admitted digest | | 72 | 32 | predecessor catalog digest | zero for generation 1 | | 104 | 32 | immutable-pool inventory digest | canonical complete inventory | -| 136 | 8 | root device identity | admitted platform value | -| 144 | 8 | root mount identity | admitted platform value | -| 152 | 8 | root file identity | admitted platform value | +| 136 | 8 | root device identity | admitted Linux `dev_t` | +| 144 | 8 | root mount identity | admitted Linux `statx.stx_mnt_id` | +| 152 | 8 | root file identity | admitted Linux `statx.stx_ino` | | 160 | 32 | target format-definition digest | exact registered v2 digest | | 192 | 32 | new store identifier | deterministic derivation below | | 224 | 32 | checksum | BLAKE3-256 over bytes `0..224` | @@ -119,6 +114,10 @@ each migration inventory entry is exactly 56 bytes, and the fixed maximum is 2,097,152 entries. The intent therefore binds the exact catalog generation, length, and digest named by the admitted version-1 `HEAD`. +On Linux, the root device coordinate is `dev_t`, reconstructed from +`statx.stx_dev_major` and `statx.stx_dev_minor`; mount and file use +`statx.stx_mnt_id` and `statx.stx_ino`. Each is big-endian `u64`. + The deterministically derived store identifier is: ```text @@ -191,15 +190,18 @@ Migration performs these ordered steps: 9. Publish `migration.receipt` from `migration.receipt.next` through the fixed-stage protocol. -The [migration crash-point specification](migration-crash.md) owns that -protocol and spans `KEEP-CRASH-053` through `KEEP-CRASH-073`. - +The [migration crash-point specification](migration-crash.md) owns +`KEEP-CRASH-053` through `KEEP-CRASH-073`. Migration never rewrites or deletes admitted version-1 immutable bytes and provides no automatic downgrade. -Version-1 admission refuses once any migration stage, `migration.intent`, -`reader.lock`, `FORMAT`, or version-2 directory is present. Once the canonical -intent is durable, only version-2 migration recovery may continue. +`FilesystemStoreMigrationAuthority` retains the writer lock and pinned root +and pools. It admits the version-1 namespace, Linux root identity, `HEAD`, +complete immutable-pool inventory, and selected catalog. Before mutation, it +requires the same canonical intent. +Version-1 admission refuses after a migration stage, `migration.intent`, +`reader.lock`, `FORMAT`, or version-2 directory exists. After durable intent, +only version-2 migration recovery may continue. ## Partial migration recovery @@ -219,15 +221,13 @@ The migration recovery boundary admits only these ordered prefixes: -A partial migration retry revalidates the intent and every existing byte, -continues idempotently at the first absent canonical step, and never replaces -an existing entry. A missing predecessor, changed version-1 coordinate, -out-of-order name, wrong file kind, substituted byte, conflicting receipt, -unknown entry, or changed root identity is unrecoverable ambiguity. +A partial migration retry revalidates intent and existing bytes, resumes at the +first absent canonical step, and never replaces an entry. A missing predecessor, +changed version-1 coordinate, out-of-order name, wrong kind or bytes, conflicting +receipt, unknown entry, or changed root identity is unrecoverable ambiguity. -Process death before durable canonical intent leaves version 1 plus at most its -non-authoritative stage. Process death after durable intent leaves -recovery-required version-2 migration state. +Death before durable intent leaves v1 plus at most its non-authoritative stage. +Death after durable intent leaves recovery-required v2 migration state. ## Retention publication recovery diff --git a/docs/formats/segment-store-v2/requirements.md b/docs/formats/segment-store-v2/requirements.md index 3ea5b2c..960656f 100644 --- a/docs/formats/segment-store-v2/requirements.md +++ b/docs/formats/segment-store-v2/requirements.md @@ -30,12 +30,12 @@ case is not evidence. | --- | --- | --- | --- | | `KEEP-MIGRATION-001` | Exact version-1 stores remain admitted until a durable migration artifact exists | compatibility fixtures | Planned in #19 | | `KEEP-MIGRATION-002` | Format marker, intent, and receipt have complete fixed byte tables, named domains, bounds, checksums, deterministic store identity, and exact initial-state digests | exact admission in `tests/store_format_marker.rs`, `tests/store_migration_intent.rs`, and `tests/store_migration_receipt.rs`; canonical construction in `tests/store_migration_intent_encoding.rs` and `tests/store_migration_receipt_encoding.rs`; seeded `migration_format` fuzz target | Implemented | -| `KEEP-MIGRATION-003` | Migration revalidates version-1 head, catalog, pools, root identity, and writer authority before mutation | bounded canonical pool inventory in `tests/store_migration_inventory.rs`; writer-locked filesystem pool admission in `filesystem_inventory_*_tests`; verification-first execution in `tests/store_migration_execution.rs`; mutation-time integration remains | In progress in #19 | +| `KEEP-MIGRATION-003` | Migration revalidates version-1 head, catalog, pools, root identity, and writer authority before mutation | bounded canonical pool inventory in `tests/store_migration_inventory.rs`; writer-locked filesystem pool admission in `filesystem_inventory_*_tests`; exact authority observation and drift refusal in `filesystem_migration_authority_tests`; verification-first execution in `tests/store_migration_execution.rs`; filesystem storage integration remains | In progress in #19 | | `KEEP-MIGRATION-004` | Every partial migration prefix continues idempotently under writer authority | state-machine and recovery tests | Planned in #19 | | `KEEP-MIGRATION-005` | Unknown, out-of-order, substituted, corrupt, conflicting, or changed evidence is unrecoverable ambiguity | corruption and mutation matrix | Planned in #19 | | `KEEP-MIGRATION-006` | Migration never rewrites or deletes admitted version-1 immutable bytes | byte-for-byte before/after witness | Planned in #19 | | `KEEP-MIGRATION-007` | Process death around every intent stage, canonical link, namespace prefix, marker stage, receipt stage, cleanup, and synchronization boundary reaches a documented lawful state | ordered phases and capabilities in `tests/store_migration_phase.rs` and `tests/store_migration_storage.rs`; exact phase-failure execution in `tests/store_migration_execution.rs`; `KEEP-CRASH-053..=073` process-death matrix remains | In progress in #19 | -| `KEEP-MIGRATION-008` | Version-1 admission refuses every version-2 or partial-migration artifact after migration begins | compatibility and fuzz tests | Planned in #19 | +| `KEEP-MIGRATION-008` | Version-1 admission refuses every version-2 or partial-migration artifact after migration begins | `FORMAT` refusal before mutation in `filesystem_migration_authority_tests`; remaining compatibility and fuzz matrix | In progress in #19 | diff --git a/src/adapters/filesystem_catalog_publisher.rs b/src/adapters/filesystem_catalog_publisher.rs index b4c191b..16999a2 100644 --- a/src/adapters/filesystem_catalog_publisher.rs +++ b/src/adapters/filesystem_catalog_publisher.rs @@ -90,7 +90,7 @@ impl FilesystemCatalogPublisher { policy: CatalogRestartPolicy, ) -> io::Result { Self::open( - FilesystemPlatformAdmission::unchecked_for_repository_tasks(lock), + FilesystemPlatformAdmission::unchecked_for_repository_tasks(lock)?, policy, ) } @@ -101,7 +101,7 @@ impl FilesystemCatalogPublisher { policy: CatalogRestartPolicy, ) -> io::Result { Self::open( - FilesystemPlatformAdmission::unchecked_for_tests(lock), + FilesystemPlatformAdmission::unchecked_for_tests(lock)?, policy, ) } diff --git a/src/adapters/filesystem_platform_admission.rs b/src/adapters/filesystem_platform_admission.rs index 626dab3..8d31009 100644 --- a/src/adapters/filesystem_platform_admission.rs +++ b/src/adapters/filesystem_platform_admission.rs @@ -1,6 +1,7 @@ //! This module owns proof that a filesystem root passed platform admission. use super::FilesystemWriterLock; +use super::filesystem_root_identity::FilesystemRootIdentity; /// Exclusive writer authority over a platform-admitted filesystem root. /// @@ -9,24 +10,44 @@ use super::FilesystemWriterLock; #[must_use] pub struct FilesystemPlatformAdmission { lock: FilesystemWriterLock, + root_identity: FilesystemRootIdentity, } impl FilesystemPlatformAdmission { - pub(super) const fn initialized(lock: FilesystemWriterLock) -> Self { - Self { lock } + pub(super) const fn initialized( + lock: FilesystemWriterLock, + root_identity: FilesystemRootIdentity, + ) -> Self { + Self { + lock, + root_identity, + } } #[cfg(test)] - pub(super) const fn unchecked_for_tests(lock: FilesystemWriterLock) -> Self { - Self { lock } + pub(super) fn unchecked_for_tests(lock: FilesystemWriterLock) -> std::io::Result { + Self::unchecked(lock) } #[cfg(feature = "repository-tasks")] - pub(super) const fn unchecked_for_repository_tasks(lock: FilesystemWriterLock) -> Self { - Self { lock } + pub(super) fn unchecked_for_repository_tasks( + lock: FilesystemWriterLock, + ) -> std::io::Result { + Self::unchecked(lock) } pub(super) fn into_lock(self) -> FilesystemWriterLock { self.lock } + + pub(super) fn into_parts(self) -> (FilesystemWriterLock, FilesystemRootIdentity) { + (self.lock, self.root_identity) + } + + #[cfg(any(test, feature = "repository-tasks"))] + fn unchecked(lock: FilesystemWriterLock) -> std::io::Result { + let directory = lock.clone_directory()?; + let root_identity = super::filesystem_platform_profile::root_identity(&directory)?; + Ok(Self::initialized(lock, root_identity)) + } } diff --git a/src/adapters/filesystem_platform_profile.rs b/src/adapters/filesystem_platform_profile.rs index 62e561d..1b6bbda 100644 --- a/src/adapters/filesystem_platform_profile.rs +++ b/src/adapters/filesystem_platform_profile.rs @@ -5,6 +5,8 @@ use std::path::Path; use cap_std::fs::Dir; +use super::filesystem_root_identity::FilesystemRootIdentity; + #[cfg(target_os = "linux")] const PROTOCOL_DIRECTORIES: [&str; 3] = ["staging", "segments", "catalogs"]; @@ -17,6 +19,7 @@ struct LinuxDirectoryProperties { device_major: u32, device_minor: u32, mount_id: u64, + inode: u64, } #[cfg(target_os = "linux")] @@ -82,9 +85,43 @@ fn linux_directory_properties(file: &std::fs::File) -> io::Result io::Result { + let file = directory.try_clone()?.into_std_file(); + let properties = linux_directory_properties(&file)?; + Ok(linux_root_identity(properties)) +} + +#[cfg(target_os = "linux")] +fn linux_root_identity(properties: LinuxDirectoryProperties) -> FilesystemRootIdentity { + let device = rustix::fs::makedev(properties.device_major, properties.device_minor); + FilesystemRootIdentity::new(device, properties.mount_id, properties.inode) +} + +#[cfg(all(not(target_os = "linux"), any(test, feature = "repository-tasks")))] +pub(super) fn root_identity(directory: &Dir) -> io::Result { + use cap_fs_ext::MetadataExt; + + let metadata = directory.dir_metadata()?; + Ok(FilesystemRootIdentity::new( + metadata.dev(), + metadata.dev(), + metadata.ino(), + )) +} + +#[cfg(all(not(target_os = "linux"), not(any(test, feature = "repository-tasks"))))] +pub(super) fn root_identity(_directory: &Dir) -> io::Result { + Err(io::Error::new( + io::ErrorKind::Unsupported, + "filesystem root identity currently requires the admitted Linux ext4 profile", + )) +} + #[cfg(target_os = "linux")] fn admit_linux_properties( filesystem_type: rustix::fs::FsWord, @@ -130,78 +167,5 @@ fn unsupported_linux_profile() -> io::Error { } #[cfg(all(test, target_os = "linux"))] -mod tests { - use super::{ - LinuxDirectoryProperties, PROTOCOL_DIRECTORIES, admit_linux_child_properties, - admit_linux_properties, - }; - - use rustix::fs::{NFS_SUPER_MAGIC, StatVfsMountFlags}; - - const EXT4_SUPER_MAGIC: rustix::fs::FsWord = 0x0000_ef53; - const EXT4_CASEFOLD_FLAG: u32 = 0x4000_0000; - - #[test] - fn only_writable_case_sensitive_ext4_is_admitted() { - assert!(admit_linux_properties(EXT4_SUPER_MAGIC, StatVfsMountFlags::empty(), 0).is_ok()); - assert_unsupported(&admit_linux_properties( - EXT4_SUPER_MAGIC, - StatVfsMountFlags::empty(), - EXT4_CASEFOLD_FLAG, - )); - assert_unsupported(&admit_linux_properties( - EXT4_SUPER_MAGIC, - StatVfsMountFlags::RDONLY, - 0, - )); - assert_unsupported(&admit_linux_properties( - NFS_SUPER_MAGIC, - StatVfsMountFlags::empty(), - 0, - )); - } - - #[test] - fn every_protocol_child_must_share_the_root_filesystem_and_mount() { - assert_eq!(PROTOCOL_DIRECTORIES, ["staging", "segments", "catalogs"]); - let root = properties(8, 1, 41); - let mut casefolded = root; - casefolded.inode_flags = EXT4_CASEFOLD_FLAG; - let mut read_only = root; - read_only.mount_flags = StatVfsMountFlags::RDONLY; - let mut foreign_format = root; - foreign_format.filesystem_type = NFS_SUPER_MAGIC; - - assert!(admit_linux_child_properties(root, root).is_ok()); - assert_unsupported(&admit_linux_child_properties(root, properties(8, 2, 41))); - assert_unsupported(&admit_linux_child_properties(root, properties(8, 1, 42))); - assert_unsupported(&admit_linux_child_properties(root, casefolded)); - assert_unsupported(&admit_linux_child_properties(root, read_only)); - assert_unsupported(&admit_linux_child_properties(root, foreign_format)); - } - - fn assert_unsupported(result: &std::io::Result<()>) { - assert!(matches!( - result, - Err(error) - if error.kind() == std::io::ErrorKind::Unsupported - && error.to_string() - == "store namespace does not satisfy one local writable case-sensitive ext4 profile" - )); - } - - const fn properties( - device_major: u32, - device_minor: u32, - mount_id: u64, - ) -> LinuxDirectoryProperties { - LinuxDirectoryProperties { - filesystem_type: EXT4_SUPER_MAGIC, - mount_flags: StatVfsMountFlags::empty(), - inode_flags: 0, - device_major, - device_minor, - mount_id, - } - } -} +#[path = "filesystem_platform_profile_tests.rs"] +mod tests; diff --git a/src/adapters/filesystem_platform_profile_tests.rs b/src/adapters/filesystem_platform_profile_tests.rs new file mode 100644 index 0000000..dd9d63e --- /dev/null +++ b/src/adapters/filesystem_platform_profile_tests.rs @@ -0,0 +1,84 @@ +//! Linux filesystem platform-profile laws. + +use super::{ + LinuxDirectoryProperties, PROTOCOL_DIRECTORIES, admit_linux_child_properties, + admit_linux_properties, linux_root_identity, +}; +use rustix::fs::{NFS_SUPER_MAGIC, StatVfsMountFlags}; + +const EXT4_SUPER_MAGIC: rustix::fs::FsWord = 0x0000_ef53; +const EXT4_CASEFOLD_FLAG: u32 = 0x4000_0000; + +#[test] +fn only_writable_case_sensitive_ext4_is_admitted() { + assert!(admit_linux_properties(EXT4_SUPER_MAGIC, StatVfsMountFlags::empty(), 0).is_ok()); + assert_unsupported(&admit_linux_properties( + EXT4_SUPER_MAGIC, + StatVfsMountFlags::empty(), + EXT4_CASEFOLD_FLAG, + )); + assert_unsupported(&admit_linux_properties( + EXT4_SUPER_MAGIC, + StatVfsMountFlags::RDONLY, + 0, + )); + assert_unsupported(&admit_linux_properties( + NFS_SUPER_MAGIC, + StatVfsMountFlags::empty(), + 0, + )); +} + +#[test] +fn every_protocol_child_must_share_the_root_filesystem_and_mount() { + assert_eq!(PROTOCOL_DIRECTORIES, ["staging", "segments", "catalogs"]); + let root = properties(8, 1, 41, 1); + let mut casefolded = root; + casefolded.inode_flags = EXT4_CASEFOLD_FLAG; + let mut read_only = root; + read_only.mount_flags = StatVfsMountFlags::RDONLY; + let mut foreign_format = root; + foreign_format.filesystem_type = NFS_SUPER_MAGIC; + + assert!(admit_linux_child_properties(root, root).is_ok()); + assert_unsupported(&admit_linux_child_properties(root, properties(8, 2, 41, 1))); + assert_unsupported(&admit_linux_child_properties(root, properties(8, 1, 42, 1))); + assert_unsupported(&admit_linux_child_properties(root, casefolded)); + assert_unsupported(&admit_linux_child_properties(root, read_only)); + assert_unsupported(&admit_linux_child_properties(root, foreign_format)); +} + +#[test] +fn root_identity_uses_linux_device_mount_and_inode_coordinates() { + let identity = linux_root_identity(properties(8, 1, 41, 73)); + assert_eq!(identity.device(), rustix::fs::makedev(8, 1)); + assert_eq!(identity.mount(), 41); + assert_eq!(identity.file(), 73); +} + +fn assert_unsupported(result: &std::io::Result<()>) { + assert!(matches!( + result, + Err(error) + if error.kind() == std::io::ErrorKind::Unsupported + && error.to_string() + == "store namespace does not satisfy one local writable case-sensitive ext4 profile" + )); +} + +const fn properties( + device_major: u32, + device_minor: u32, + mount_id: u64, + inode: u64, +) -> LinuxDirectoryProperties { + LinuxDirectoryProperties { + filesystem_type: EXT4_SUPER_MAGIC, + mount_flags: StatVfsMountFlags::empty(), + inode_flags: 0, + device_major, + device_minor, + mount_id, + inode, + } +} diff --git a/src/adapters/filesystem_root_identity.rs b/src/adapters/filesystem_root_identity.rs new file mode 100644 index 0000000..e63619f --- /dev/null +++ b/src/adapters/filesystem_root_identity.rs @@ -0,0 +1,30 @@ +//! This module owns one admitted physical filesystem-root coordinate. + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(super) struct FilesystemRootIdentity { + device: u64, + mount: u64, + file: u64, +} + +impl FilesystemRootIdentity { + pub(super) const fn new(device: u64, mount: u64, file: u64) -> Self { + Self { + device, + mount, + file, + } + } + + pub(super) const fn device(self) -> u64 { + self.device + } + + pub(super) const fn mount(self) -> u64 { + self.mount + } + + pub(super) const fn file(self) -> u64 { + self.file + } +} diff --git a/src/adapters/filesystem_store_initializer.rs b/src/adapters/filesystem_store_initializer.rs index bbcc392..80456bb 100644 --- a/src/adapters/filesystem_store_initializer.rs +++ b/src/adapters/filesystem_store_initializer.rs @@ -83,7 +83,17 @@ fn initialize_storage( let lock = storage.into_lock().map_err(|source| { StoreInitializationError::io(StoreInitializationPhase::OpenAndLockWriterFile, source) })?; - Ok(FilesystemPlatformAdmission::initialized(lock)) + let directory = lock.clone_directory().map_err(|source| { + StoreInitializationError::io(StoreInitializationPhase::AdmitPlatform, source) + })?; + let root_identity = + filesystem_platform_profile::root_identity(&directory).map_err(|source| { + StoreInitializationError::io(StoreInitializationPhase::AdmitPlatform, source) + })?; + Ok(FilesystemPlatformAdmission::initialized( + lock, + root_identity, + )) } fn reopen_root( @@ -96,5 +106,10 @@ fn reopen_root( .map_err(|source| FilesystemPlatformAdmissionError::Namespace { source })?; filesystem_initialization_namespace::admit_published(&directory) .map_err(|source| FilesystemPlatformAdmissionError::Namespace { source })?; - Ok(FilesystemPlatformAdmission::initialized(lock)) + let root_identity = filesystem_platform_profile::root_identity(&directory) + .map_err(|source| FilesystemPlatformAdmissionError::Platform { source })?; + Ok(FilesystemPlatformAdmission::initialized( + lock, + root_identity, + )) } diff --git a/src/adapters/mod.rs b/src/adapters/mod.rs index 20952e2..3c783ca 100644 --- a/src/adapters/mod.rs +++ b/src/adapters/mod.rs @@ -117,6 +117,7 @@ mod filesystem_recovery_stage_materialization; mod filesystem_recovery_stage_sync; #[cfg(all(test, unix))] mod filesystem_recovery_stage_tests; +mod filesystem_root_identity; mod filesystem_segment_stage; #[cfg(test)] mod filesystem_segment_stage_tests; diff --git a/src/adapters/store_migration.rs b/src/adapters/store_migration.rs index 93cdfc9..0083394 100644 --- a/src/adapters/store_migration.rs +++ b/src/adapters/store_migration.rs @@ -34,6 +34,12 @@ mod filesystem_inventory_segments_refusal_tests; mod filesystem_inventory_segments_test_fixture; #[cfg(test)] mod filesystem_inventory_segments_tests; +mod filesystem_migration_authority; +mod filesystem_migration_authority_error; +mod filesystem_migration_authority_error_display; +#[cfg(test)] +mod filesystem_migration_authority_tests; +mod filesystem_migration_authority_validation; mod format_definition_digest; mod format_marker_decode_error; mod format_marker_decode_error_display; @@ -44,6 +50,7 @@ mod immutable_pool_inventory_digest; mod initial_gc_state_digest; mod initial_retention_state_digest; mod migration_catalog_admission; +mod migration_catalog_coordinates; mod migration_catalog_plan; mod migration_catalog_records; mod migration_error; @@ -84,6 +91,11 @@ pub use filesystem_inventory_error::{ MigrationInventoryNamespace, MigrationInventoryPool, }; pub use filesystem_inventory_reader::FilesystemStoreMigrationInventoryReader; +pub use filesystem_migration_authority::FilesystemStoreMigrationAuthority; +pub use filesystem_migration_authority_error::{ + FilesystemMigrationAuthorityArtifact, FilesystemMigrationAuthorityError, + StoreRootIdentityCoordinate, +}; pub use format_definition_digest::StoreFormatDefinitionDigest; pub use format_marker_decode_error::StoreFormatMarkerDecodeError; pub use format_marker_digest::StoreFormatMarkerDigest; diff --git a/src/adapters/store_migration/canonical_migration_intent.rs b/src/adapters/store_migration/canonical_migration_intent.rs index 28a95ca..3478a4f 100644 --- a/src/adapters/store_migration/canonical_migration_intent.rs +++ b/src/adapters/store_migration/canonical_migration_intent.rs @@ -1,6 +1,8 @@ //! This boundary module owns canonical owned migration-intent bytes. use super::admitted_migration_intent::StoreMigrationIntentFields; +use super::migration_catalog_coordinates::MigrationCatalogCoordinates; +use super::store_root_identity::StoreRootIdentities; use super::{ ImmutablePoolInventoryDigest, StoreFormatDefinitionDigest, StoreIdentifier, StoreMigrationIntentDigest, StoreRootDeviceIdentity, StoreRootFileIdentity, @@ -53,14 +55,24 @@ impl CanonicalStoreMigrationIntent { root_file_identity: StoreRootFileIdentity, ) -> Self { migration_intent_encoder::encode( - snapshot, + MigrationCatalogCoordinates::from_snapshot(snapshot), inventory_digest, - root_device_identity, - root_mount_identity, - root_file_identity, + StoreRootIdentities::new( + root_device_identity, + root_mount_identity, + root_file_identity, + ), ) } + pub(super) fn from_coordinates( + catalog: MigrationCatalogCoordinates, + inventory_digest: ImmutablePoolInventoryDigest, + roots: StoreRootIdentities, + ) -> Self { + migration_intent_encoder::encode(catalog, inventory_digest, roots) + } + /// Returns the exact canonical intent bytes. pub const fn encoded(&self) -> &[u8] { &self.encoded diff --git a/src/adapters/store_migration/filesystem_inventory_reader.rs b/src/adapters/store_migration/filesystem_inventory_reader.rs index 06b00ea..7d989ab 100644 --- a/src/adapters/store_migration/filesystem_inventory_reader.rs +++ b/src/adapters/store_migration/filesystem_inventory_reader.rs @@ -14,6 +14,7 @@ use super::filesystem_inventory_segments::FilesystemMigrationSegmentInventory; use super::{ ImmutablePoolInventoryDigest, StoreMigrationInventoryEntryCount, StoreMigrationInventoryHasher, }; +use crate::adapters::filesystem_root_identity::FilesystemRootIdentity; use crate::adapters::{FilesystemPlatformAdmission, FilesystemWriterLock, SegmentReadPolicy}; const SEGMENTS_NAME: &str = "segments"; @@ -29,6 +30,7 @@ pub struct FilesystemStoreMigrationInventoryReader { segments: PinnedMigrationPoolDirectory, catalogs: PinnedMigrationPoolDirectory, policy: SegmentReadPolicy, + root_identity: FilesystemRootIdentity, _lock: FilesystemWriterLock, } @@ -47,7 +49,7 @@ impl FilesystemStoreMigrationInventoryReader { admission: FilesystemPlatformAdmission, policy: SegmentReadPolicy, ) -> Result { - let lock = admission.into_lock(); + let (lock, root_identity) = admission.into_parts(); let root = lock.clone_directory() .map_err(|source| FilesystemMigrationInventoryError::Io { @@ -70,6 +72,7 @@ impl FilesystemStoreMigrationInventoryReader { segments, catalogs, policy, + root_identity, _lock: lock, }) } @@ -131,6 +134,18 @@ impl FilesystemStoreMigrationInventoryReader { self.segments.verify(&self.root)?; self.catalogs.verify(&self.root) } + + pub(super) const fn root(&self) -> &Dir { + &self.root + } + + pub(super) const fn catalogs(&self) -> &Dir { + self.catalogs.directory() + } + + pub(super) const fn root_identity(&self) -> FilesystemRootIdentity { + self.root_identity + } } fn hash_inventory( diff --git a/src/adapters/store_migration/filesystem_migration_authority.rs b/src/adapters/store_migration/filesystem_migration_authority.rs new file mode 100644 index 0000000..e263f18 --- /dev/null +++ b/src/adapters/store_migration/filesystem_migration_authority.rs @@ -0,0 +1,160 @@ +//! This module owns exact writer-locked filesystem migration authority. + +use super::filesystem_inventory_file::{self, FilesystemInventoryFilePolicy}; +use super::filesystem_migration_authority_error::{ + FilesystemMigrationAuthorityArtifact as Artifact, FilesystemMigrationAuthorityError as Error, + StoreRootIdentityCoordinate as RootCoordinate, +}; +use super::filesystem_migration_authority_validation::{ + artifact_error, require_root, verify_catalog, +}; +use super::migration_catalog_coordinates::MigrationCatalogCoordinates; +use super::store_root_identity::StoreRootIdentities; +use super::{CanonicalStoreMigrationIntent, FilesystemStoreMigrationInventoryReader}; +use crate::adapters::{ + CatalogRestartArtifact, CatalogRestartPhase, ChecksummedCatalog, ChecksummedPublicationHead, + FilesystemPlatformAdmission, SegmentReadPolicy, filesystem_initialization_namespace, + filesystem_platform_profile, physical_pool_name, +}; + +const HEAD_NAME: &str = "HEAD"; +const HEAD_LENGTH: u64 = 128; + +/// Exclusive authority to observe and migrate one pinned version-1 filesystem root. +/// +/// The authority retains the admitted writer lock and pinned root and immutable +/// pool capabilities for its entire lifetime. Its synchronous, +/// capability-relative filesystem I/O performs no protocol mutation and uses +/// neither a network nor an asynchronous runtime. +#[must_use] +pub struct FilesystemStoreMigrationAuthority { + inventory: FilesystemStoreMigrationInventoryReader, +} + +impl FilesystemStoreMigrationAuthority { + /// Pins one admitted filesystem root for migration observation. + /// + /// This synchronous constructor opens pinned directory capabilities but + /// materializes no artifact bodies and performs no protocol mutation. + /// + /// # Errors + /// + /// Returns [`FilesystemMigrationAuthorityError`](super::FilesystemMigrationAuthorityError) + /// when the root capability cannot be cloned or either immutable pool + /// cannot be pinned without following links. + pub fn open( + admission: FilesystemPlatformAdmission, + policy: SegmentReadPolicy, + ) -> Result { + let inventory = FilesystemStoreMigrationInventoryReader::open(admission, policy) + .map_err(|source| Error::Inventory { source })?; + Ok(Self { inventory }) + } + + /// Observes one canonical intent from exact current version-1 authority. + /// + /// The synchronous call admits the exact published root namespace, physical + /// root coordinate, fixed-width head, complete immutable-pool inventory, + /// and head-selected catalog. Peak content allocation is bounded by one + /// catalog and one segment in addition to the bounded semantic inventory. + /// + /// # Errors + /// + /// Returns [`FilesystemMigrationAuthorityError`](super::FilesystemMigrationAuthorityError) + /// at the exact namespace, root, artifact, coordinate, or inventory refusal. + pub fn observe_intent(&self) -> Result { + self.verify_namespace()?; + let roots = self.verify_root_identity()?; + let head_bytes = self.read_head()?; + let head = ChecksummedPublicationHead::decode(&head_bytes) + .map_err(|source| Error::Head { source })?; + let inventory_digest = self + .inventory + .read() + .map_err(|source| Error::Inventory { source })?; + let coordinates = self.read_catalog(head)?; + if self.read_head()? != head_bytes { + return Err(Error::HeadChanged); + } + self.verify_namespace()?; + let _current_roots = self.verify_root_identity()?; + Ok(CanonicalStoreMigrationIntent::from_coordinates( + coordinates, + inventory_digest, + roots, + )) + } + + /// Re-observes and compares every coordinate in one canonical intent. + /// + /// This has the same synchronous I/O and bounded-allocation behavior as + /// [`Self::observe_intent`] and performs no protocol mutation. + /// + /// # Errors + /// + /// Returns the exact observation refusal or + /// [`FilesystemMigrationAuthorityError::IntentChanged`] with both intent + /// digests when current authority no longer reproduces `expected`. + pub fn verify_current(&self, expected: &CanonicalStoreMigrationIntent) -> Result<(), Error> { + let observed = self.observe_intent()?; + if &observed == expected { + Ok(()) + } else { + Err(Error::IntentChanged { + expected: expected.digest(), + observed: observed.digest(), + }) + } + } + + fn verify_namespace(&self) -> Result<(), Error> { + filesystem_initialization_namespace::admit_published(self.inventory.root()) + .map_err(|source| Error::Namespace { source }) + } + + fn verify_root_identity(&self) -> Result { + let expected = self.inventory.root_identity(); + let observed = filesystem_platform_profile::root_identity(self.inventory.root()) + .map_err(|source| Error::RootIdentity { source })?; + require_root(RootCoordinate::Device, expected.device(), observed.device())?; + require_root(RootCoordinate::Mount, expected.mount(), observed.mount())?; + require_root(RootCoordinate::File, expected.file(), observed.file())?; + Ok(StoreRootIdentities::from_filesystem(observed)) + } + + fn read_head(&self) -> Result, Error> { + let policy = FilesystemInventoryFilePolicy::new( + CatalogRestartArtifact::Head, + CatalogRestartPhase::OpenHead, + CatalogRestartPhase::ReadHead, + HEAD_LENGTH, + ); + filesystem_inventory_file::read(self.inventory.root(), HEAD_NAME, policy) + .map_err(|source| artifact_error(Artifact::Head, source)) + } + + fn read_catalog( + &self, + head: ChecksummedPublicationHead<'_>, + ) -> Result { + let artifact = Artifact::Catalog { + generation: head.generation(), + digest: head.catalog_digest(), + }; + let name = physical_pool_name::catalog(head.generation(), head.catalog_digest()); + let policy = FilesystemInventoryFilePolicy::new( + CatalogRestartArtifact::Catalog, + CatalogRestartPhase::OpenCatalog, + CatalogRestartPhase::ReadCatalog, + head.catalog_length().get(), + ); + let bytes = filesystem_inventory_file::read(self.inventory.catalogs(), &name, policy) + .map_err(|source| artifact_error(artifact, source))?; + let catalog = ChecksummedCatalog::decode(&bytes).map_err(|source| Error::Catalog { + generation: head.generation(), + digest: head.catalog_digest(), + source: Box::new(source), + })?; + verify_catalog(head, catalog) + } +} diff --git a/src/adapters/store_migration/filesystem_migration_authority_error.rs b/src/adapters/store_migration/filesystem_migration_authority_error.rs new file mode 100644 index 0000000..4e98601 --- /dev/null +++ b/src/adapters/store_migration/filesystem_migration_authority_error.rs @@ -0,0 +1,117 @@ +//! This boundary module owns filesystem migration-authority failures. + +use std::io; + +use super::{FilesystemMigrationInventoryError, StoreMigrationIntentDigest}; +use crate::adapters::{CatalogDecodeError, CatalogRestartError, PublicationHeadDecodeError}; +use crate::{CatalogDigest, CatalogGeneration, CatalogLength}; + +/// Published version-1 artifact observed while establishing migration authority. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum FilesystemMigrationAuthorityArtifact { + /// The mutable published `HEAD`. + Head, + /// The immutable catalog selected by `HEAD`. + Catalog { + /// Catalog generation named by `HEAD`. + generation: CatalogGeneration, + /// Catalog digest named by `HEAD`. + digest: CatalogDigest, + }, +} + +/// Physical store-root coordinate compared during migration admission. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum StoreRootIdentityCoordinate { + /// Platform device coordinate. + Device, + /// Platform mount coordinate. + Mount, + /// Platform file coordinate. + File, +} + +/// Failure to observe or revalidate exact filesystem migration authority. +#[derive(Debug)] +pub enum FilesystemMigrationAuthorityError { + /// Complete immutable-pool inventory could not be admitted. + Inventory { + /// Preserved inventory refusal. + source: FilesystemMigrationInventoryError, + }, + /// The exact published version-1 root namespace could not be admitted. + Namespace { + /// Preserved capability-relative filesystem source. + source: io::Error, + }, + /// The physical root identity could not be observed. + RootIdentity { + /// Preserved platform source. + source: io::Error, + }, + /// One physical root coordinate changed under retained authority. + RootIdentityChanged { + /// Coordinate that changed. + coordinate: StoreRootIdentityCoordinate, + /// Coordinate retained by platform admission. + expected: u64, + /// Coordinate observed immediately before migration. + observed: u64, + }, + /// One selected artifact could not be read completely. + Artifact { + /// Exact artifact being observed. + artifact: FilesystemMigrationAuthorityArtifact, + /// Preserved bounded-read refusal. + source: Box, + }, + /// One selected artifact changed physical identity while being read. + ArtifactChanged { + /// Exact artifact that changed. + artifact: FilesystemMigrationAuthorityArtifact, + }, + /// The published head bytes were malformed. + Head { + /// Preserved head-decoding refusal. + source: PublicationHeadDecodeError, + }, + /// The selected catalog bytes were malformed. + Catalog { + /// Catalog generation selected by the head. + generation: CatalogGeneration, + /// Catalog digest selected by the head. + digest: CatalogDigest, + /// Preserved catalog-decoding refusal. + source: Box, + }, + /// Head and selected catalog generation coordinates disagreed. + CatalogGeneration { + /// Generation required by the head. + expected: CatalogGeneration, + /// Generation observed in the catalog. + observed: CatalogGeneration, + }, + /// Head and selected catalog length coordinates disagreed. + CatalogLength { + /// Length required by the head. + expected: CatalogLength, + /// Length observed in the catalog. + observed: CatalogLength, + }, + /// Head and selected catalog digest coordinates disagreed. + CatalogDigest { + /// Digest required by the head. + expected: CatalogDigest, + /// Digest observed in the catalog. + observed: CatalogDigest, + }, + /// The mutable head changed during authority observation. + HeadChanged, + /// Re-observation did not reproduce the supplied canonical intent. + IntentChanged { + /// Intent authorized by the caller. + expected: StoreMigrationIntentDigest, + /// Intent derived from current filesystem authority. + observed: StoreMigrationIntentDigest, + }, +} diff --git a/src/adapters/store_migration/filesystem_migration_authority_error_display.rs b/src/adapters/store_migration/filesystem_migration_authority_error_display.rs new file mode 100644 index 0000000..9fc8c14 --- /dev/null +++ b/src/adapters/store_migration/filesystem_migration_authority_error_display.rs @@ -0,0 +1,112 @@ +//! This module owns filesystem migration-authority error presentation. + +use std::error::Error; +use std::fmt; + +use super::filesystem_migration_authority_error::{ + FilesystemMigrationAuthorityArtifact, FilesystemMigrationAuthorityError, + StoreRootIdentityCoordinate, +}; + +impl fmt::Display for FilesystemMigrationAuthorityArtifact { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::Head => formatter.write_str("HEAD"), + Self::Catalog { generation, digest } => { + write!(formatter, "catalog {generation:?}/{digest:?}") + } + } + } +} + +impl fmt::Display for StoreRootIdentityCoordinate { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter.write_str(match self { + Self::Device => "device", + Self::Mount => "mount", + Self::File => "file", + }) + } +} + +impl fmt::Display for FilesystemMigrationAuthorityError { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::Inventory { .. } => { + formatter.write_str("filesystem migration inventory was refused") + } + Self::Namespace { .. } => { + formatter.write_str("filesystem migration root namespace was refused") + } + Self::RootIdentity { .. } => { + formatter.write_str("filesystem migration root identity could not be observed") + } + Self::RootIdentityChanged { + coordinate, + expected, + observed, + } => write!( + formatter, + "filesystem migration root {coordinate} changed: expected {expected}, observed \ + {observed}" + ), + Self::Artifact { artifact, .. } => { + write!(formatter, "filesystem migration could not read {artifact}") + } + Self::ArtifactChanged { artifact } => { + write!( + formatter, + "filesystem migration {artifact} changed identity" + ) + } + Self::Head { .. } => formatter.write_str("filesystem migration HEAD was malformed"), + Self::Catalog { + generation, digest, .. + } => write!( + formatter, + "filesystem migration catalog {generation:?}/{digest:?} was malformed" + ), + Self::CatalogGeneration { expected, observed } => write!( + formatter, + "filesystem migration catalog generation disagreed: expected {expected:?}, \ + observed {observed:?}" + ), + Self::CatalogLength { expected, observed } => write!( + formatter, + "filesystem migration catalog length disagreed: expected {expected:?}, observed \ + {observed:?}" + ), + Self::CatalogDigest { expected, observed } => write!( + formatter, + "filesystem migration catalog digest disagreed: expected {expected:?}, observed \ + {observed:?}" + ), + Self::HeadChanged => { + formatter.write_str("filesystem migration HEAD changed during observation") + } + Self::IntentChanged { expected, observed } => write!( + formatter, + "filesystem migration intent changed: expected {expected:?}, observed {observed:?}" + ), + } + } +} + +impl Error for FilesystemMigrationAuthorityError { + fn source(&self) -> Option<&(dyn Error + 'static)> { + match self { + Self::Inventory { source } => Some(source), + Self::Namespace { source } | Self::RootIdentity { source } => Some(source), + Self::Artifact { source, .. } => Some(source), + Self::Head { source } => Some(source), + Self::Catalog { source, .. } => Some(source), + Self::RootIdentityChanged { .. } + | Self::ArtifactChanged { .. } + | Self::CatalogGeneration { .. } + | Self::CatalogLength { .. } + | Self::CatalogDigest { .. } + | Self::HeadChanged + | Self::IntentChanged { .. } => None, + } + } +} diff --git a/src/adapters/store_migration/filesystem_migration_authority_tests.rs b/src/adapters/store_migration/filesystem_migration_authority_tests.rs new file mode 100644 index 0000000..5afbc84 --- /dev/null +++ b/src/adapters/store_migration/filesystem_migration_authority_tests.rs @@ -0,0 +1,109 @@ +//! Writer-locked filesystem migration authority laws. + +use std::error::Error; +use std::fs; + +use super::FilesystemMigrationAuthorityError; +use super::filesystem_migration_authority::FilesystemStoreMigrationAuthority; +use crate::adapters::filesystem_test_sandbox::TestDirectory; +use crate::adapters::test_support::decode_hex; +use crate::adapters::{AdmittedSegment, FilesystemPlatformAdmission, physical_pool_name}; + +const SEGMENT_HEX: &str = + include_str!("../../../conformance/segment-store/v1/one-zero-segment.hex"); +const CATALOG_HEX: &str = + include_str!("../../../conformance/segment-store/v1/one-zero-catalog.hex"); +const HEAD_HEX: &str = include_str!("../../../conformance/segment-store/v1/one-zero-head.hex"); +const EMPTY_SEGMENT_HEX: &str = + include_str!("../../../conformance/segment-store/v1/empty-segment.hex"); +const CATALOG_NAME: &str = + "0000000000000001-04b82519b0399baefd0b9c0f32a871052e4c47e3a00226ab03b21661470f7320.cat"; +const SEGMENT_NAME: &str = "b7542dced2ab770894a14d1d04b066e3a899942602c5986d35ba6df6c1a35cfc.seg"; +const INVENTORY_DIGEST: &str = "40bf5d49c34847ac9cf46a256f343cee80cd980d1405d2dd02ceff8f58d674f9"; + +#[test] +fn exact_published_v1_authority_constructs_and_revalidates_one_intent() -> Result<(), Box> +{ + let (sandbox, authority) = open_authority("migration-authority-current")?; + let intent = authority.observe_intent()?; + authority.verify_current(&intent)?; + + assert_eq!(intent.catalog_generation().get(), 1); + assert_eq!( + intent.inventory_digest().as_bytes().as_slice(), + decode_hex(INVENTORY_DIGEST)? + ); + drop(authority); + sandbox.remove()?; + Ok(()) +} + +#[test] +fn immutable_pool_drift_refuses_the_retained_intent() -> Result<(), Box> { + let (sandbox, authority) = open_authority("migration-authority-inventory-drift")?; + let intent = authority.observe_intent()?; + let bytes = decode_hex(EMPTY_SEGMENT_HEX.trim())?; + let segment = AdmittedSegment::decode(&bytes, maximum_policy())?; + fs::write( + sandbox + .path() + .join("segments") + .join(physical_pool_name::segment(segment.digest())), + bytes, + )?; + + let error = authority + .verify_current(&intent) + .err() + .ok_or("changed inventory unexpectedly retained authority")?; + assert!(matches!( + error, + FilesystemMigrationAuthorityError::IntentChanged { expected, observed } + if expected == intent.digest() && observed != expected + )); + drop(authority); + sandbox.remove()?; + Ok(()) +} + +#[test] +fn version_two_namespace_evidence_refuses_before_mutation() -> Result<(), Box> { + let (sandbox, authority) = open_authority("migration-authority-v2-evidence")?; + let intent = authority.observe_intent()?; + fs::write(sandbox.path().join("FORMAT"), [])?; + + let error = authority + .verify_current(&intent) + .err() + .ok_or("version-two evidence unexpectedly retained authority")?; + assert!(matches!( + error, + FilesystemMigrationAuthorityError::Namespace { source } + if source.kind() == std::io::ErrorKind::InvalidData + )); + drop(authority); + sandbox.remove()?; + Ok(()) +} + +fn open_authority( + name: &str, +) -> Result<(TestDirectory, FilesystemStoreMigrationAuthority), Box> { + let sandbox = TestDirectory::create(name)?; + let admission = FilesystemPlatformAdmission::initialize_unchecked_for_tests(sandbox.path())?; + fs::write( + sandbox.path().join("segments").join(SEGMENT_NAME), + decode_hex(SEGMENT_HEX.trim())?, + )?; + fs::write( + sandbox.path().join("catalogs").join(CATALOG_NAME), + decode_hex(CATALOG_HEX.trim())?, + )?; + fs::write(sandbox.path().join("HEAD"), decode_hex(HEAD_HEX.trim())?)?; + let authority = FilesystemStoreMigrationAuthority::open(admission, maximum_policy())?; + Ok((sandbox, authority)) +} + +const fn maximum_policy() -> crate::adapters::SegmentReadPolicy { + super::filesystem_inventory_catalogs_test_fixture::maximum_policy() +} diff --git a/src/adapters/store_migration/filesystem_migration_authority_validation.rs b/src/adapters/store_migration/filesystem_migration_authority_validation.rs new file mode 100644 index 0000000..fe68a46 --- /dev/null +++ b/src/adapters/store_migration/filesystem_migration_authority_validation.rs @@ -0,0 +1,80 @@ +//! This module owns migration-authority coordinate validation. + +use super::filesystem_inventory_file::FilesystemInventoryFileError; +use super::filesystem_migration_authority_error::{ + FilesystemMigrationAuthorityArtifact as Artifact, FilesystemMigrationAuthorityError as Error, + StoreRootIdentityCoordinate as RootCoordinate, +}; +use super::migration_catalog_coordinates::MigrationCatalogCoordinates; +use crate::adapters::{ChecksummedCatalog, ChecksummedPublicationHead}; + +pub(super) const fn require_root( + coordinate: RootCoordinate, + expected: u64, + observed: u64, +) -> Result<(), Error> { + if expected == observed { + Ok(()) + } else { + Err(Error::RootIdentityChanged { + coordinate, + expected, + observed, + }) + } +} + +pub(super) fn artifact_error(artifact: Artifact, source: FilesystemInventoryFileError) -> Error { + match source { + FilesystemInventoryFileError::Artifact(source) => Error::Artifact { artifact, source }, + FilesystemInventoryFileError::Changed => Error::ArtifactChanged { artifact }, + } +} + +pub(super) fn verify_catalog( + head: ChecksummedPublicationHead<'_>, + catalog: ChecksummedCatalog<'_>, +) -> Result { + require_generation(head.generation(), catalog.generation())?; + require_length(head.catalog_length(), catalog.length())?; + require_digest(head.catalog_digest(), catalog.digest())?; + Ok(MigrationCatalogCoordinates::new( + catalog.generation(), + catalog.length(), + catalog.digest(), + catalog.previous_catalog_digest(), + )) +} + +fn require_generation( + expected: crate::CatalogGeneration, + observed: crate::CatalogGeneration, +) -> Result<(), Error> { + if expected == observed { + Ok(()) + } else { + Err(Error::CatalogGeneration { expected, observed }) + } +} + +fn require_length( + expected: crate::CatalogLength, + observed: crate::CatalogLength, +) -> Result<(), Error> { + if expected == observed { + Ok(()) + } else { + Err(Error::CatalogLength { expected, observed }) + } +} + +fn require_digest( + expected: crate::CatalogDigest, + observed: crate::CatalogDigest, +) -> Result<(), Error> { + if expected == observed { + Ok(()) + } else { + Err(Error::CatalogDigest { expected, observed }) + } +} diff --git a/src/adapters/store_migration/migration_catalog_coordinates.rs b/src/adapters/store_migration/migration_catalog_coordinates.rs new file mode 100644 index 0000000..d3baaaf --- /dev/null +++ b/src/adapters/store_migration/migration_catalog_coordinates.rs @@ -0,0 +1,52 @@ +//! This module owns admitted catalog coordinates for intent encoding. + +use crate::{CatalogDigest, CatalogGeneration, CatalogLength, CatalogSnapshot}; + +#[derive(Clone, Copy)] +pub(super) struct MigrationCatalogCoordinates { + generation: CatalogGeneration, + length: CatalogLength, + digest: CatalogDigest, + predecessor: Option, +} + +impl MigrationCatalogCoordinates { + pub(super) const fn new( + generation: CatalogGeneration, + length: CatalogLength, + digest: CatalogDigest, + predecessor: Option, + ) -> Self { + Self { + generation, + length, + digest, + predecessor, + } + } + + pub(super) const fn from_snapshot(snapshot: &CatalogSnapshot<'_, '_, '_>) -> Self { + Self::new( + snapshot.generation(), + snapshot.catalog_length(), + snapshot.catalog_digest(), + snapshot.previous_catalog_digest(), + ) + } + + pub(super) const fn generation(self) -> CatalogGeneration { + self.generation + } + + pub(super) const fn length(self) -> CatalogLength { + self.length + } + + pub(super) const fn digest(self) -> CatalogDigest { + self.digest + } + + pub(super) const fn predecessor(self) -> Option { + self.predecessor + } +} diff --git a/src/adapters/store_migration/migration_intent_encoder.rs b/src/adapters/store_migration/migration_intent_encoder.rs index faa7572..1aebbf1 100644 --- a/src/adapters/store_migration/migration_intent_encoder.rs +++ b/src/adapters/store_migration/migration_intent_encoder.rs @@ -1,41 +1,27 @@ //! This boundary module owns canonical migration-intent encoding. use super::admitted_migration_intent::StoreMigrationIntentFields; +use super::migration_catalog_coordinates::MigrationCatalogCoordinates; use super::migration_intent_format::StoreIdentifierFields; +use super::store_root_identity::StoreRootIdentities; use super::{ CanonicalStoreMigrationIntent, ImmutablePoolInventoryDigest, StoreFormatDefinitionDigest, - StoreIdentifier, StoreRootDeviceIdentity, StoreRootFileIdentity, StoreRootMountIdentity, - migration_intent_format as format, + StoreIdentifier, migration_intent_format as format, }; -use crate::CatalogSnapshot; - -#[derive(Clone, Copy)] -struct RootIdentities { - device: StoreRootDeviceIdentity, - mount: StoreRootMountIdentity, - file: StoreRootFileIdentity, -} pub(super) fn encode( - snapshot: &CatalogSnapshot<'_, '_, '_>, + catalog: MigrationCatalogCoordinates, inventory_digest: ImmutablePoolInventoryDigest, - root_device_identity: StoreRootDeviceIdentity, - root_mount_identity: StoreRootMountIdentity, - root_file_identity: StoreRootFileIdentity, + roots: StoreRootIdentities, ) -> CanonicalStoreMigrationIntent { let fields = StoreIdentifierFields { - catalog_generation: snapshot.generation(), - catalog_length: snapshot.catalog_length(), - catalog_digest: snapshot.catalog_digest(), - predecessor_catalog_digest: snapshot.previous_catalog_digest(), + catalog_generation: catalog.generation(), + catalog_length: catalog.length(), + catalog_digest: catalog.digest(), + predecessor_catalog_digest: catalog.predecessor(), inventory_digest, target_definition_digest: StoreFormatDefinitionDigest::VERSION_TWO, }; - let roots = RootIdentities { - device: root_device_identity, - mount: root_mount_identity, - file: root_file_identity, - }; let store_identifier = format::store_identifier(&fields); let mut encoded = [0_u8; format::ENCODED_LENGTH]; let (preimage, checksum_slot) = encoded.split_at_mut(format::CHECKSUM_OFFSET); @@ -50,9 +36,9 @@ pub(super) fn encode( catalog_digest: fields.catalog_digest, predecessor_catalog_digest: fields.predecessor_catalog_digest, inventory_digest: fields.inventory_digest, - root_device_identity: roots.device, - root_mount_identity: roots.mount, - root_file_identity: roots.file, + root_device_identity: roots.device(), + root_mount_identity: roots.mount(), + root_file_identity: roots.file(), target_definition_digest: fields.target_definition_digest, store_identifier, }, @@ -63,7 +49,7 @@ pub(super) fn encode( fn write_preimage( output: &mut [u8], fields: &StoreIdentifierFields, - roots: RootIdentities, + roots: StoreRootIdentities, store_identifier: StoreIdentifier, ) { let (magic, output) = output.split_at_mut(16); @@ -89,11 +75,11 @@ fn write_preimage( let (inventory_digest, output) = output.split_at_mut(32); inventory_digest.copy_from_slice(fields.inventory_digest.as_bytes()); let (device_identity, output) = output.split_at_mut(8); - device_identity.copy_from_slice(&roots.device.get().to_be_bytes()); + device_identity.copy_from_slice(&roots.device().get().to_be_bytes()); let (mount_identity, output) = output.split_at_mut(8); - mount_identity.copy_from_slice(&roots.mount.get().to_be_bytes()); + mount_identity.copy_from_slice(&roots.mount().get().to_be_bytes()); let (file_identity, output) = output.split_at_mut(8); - file_identity.copy_from_slice(&roots.file.get().to_be_bytes()); + file_identity.copy_from_slice(&roots.file().get().to_be_bytes()); let (definition_digest, store_identifier_slot) = output.split_at_mut(32); definition_digest.copy_from_slice(fields.target_definition_digest.as_bytes()); store_identifier_slot.copy_from_slice(store_identifier.as_bytes()); diff --git a/src/adapters/store_migration/store_root_identity.rs b/src/adapters/store_migration/store_root_identity.rs index 39bc4bf..3ddce7b 100644 --- a/src/adapters/store_migration/store_root_identity.rs +++ b/src/adapters/store_migration/store_root_identity.rs @@ -1,5 +1,7 @@ //! This module owns physical store-root recovery coordinates. +use crate::adapters::filesystem_root_identity::FilesystemRootIdentity; + macro_rules! root_identity { ($name:ident, $documentation:literal) => { #[doc = $documentation] @@ -36,3 +38,44 @@ root_identity!( StoreRootFileIdentity, "Platform file identity bound into a migration intent." ); + +#[derive(Clone, Copy)] +pub(super) struct StoreRootIdentities { + device: StoreRootDeviceIdentity, + mount: StoreRootMountIdentity, + file: StoreRootFileIdentity, +} + +impl StoreRootIdentities { + pub(super) const fn new( + device: StoreRootDeviceIdentity, + mount: StoreRootMountIdentity, + file: StoreRootFileIdentity, + ) -> Self { + Self { + device, + mount, + file, + } + } + + pub(super) const fn from_filesystem(identity: FilesystemRootIdentity) -> Self { + Self::new( + StoreRootDeviceIdentity::from_admitted(identity.device()), + StoreRootMountIdentity::from_admitted(identity.mount()), + StoreRootFileIdentity::from_admitted(identity.file()), + ) + } + + pub(super) const fn device(self) -> StoreRootDeviceIdentity { + self.device + } + + pub(super) const fn mount(self) -> StoreRootMountIdentity { + self.mount + } + + pub(super) const fn file(self) -> StoreRootFileIdentity { + self.file + } +} diff --git a/src/lib.rs b/src/lib.rs index f82f81d..18cd737 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -68,6 +68,7 @@ pub use adapters::{ CatalogSnapshotError, CatalogSuccessor, CatalogTransitionError, ChecksummedCatalog, ChecksummedPublicationHead, ChecksummedSegmentRecord, ClosedSegment, EmptyDispositionSetDigest, FilesystemCatalogPublicationError, FilesystemCatalogPublisher, FilesystemCatalogSnapshot, + FilesystemMigrationAuthorityArtifact, FilesystemMigrationAuthorityError, FilesystemMigrationInventoryError, FilesystemMigrationInventoryOperation, FilesystemPlatformAdmission, FilesystemPlatformAdmissionError, FilesystemRecoveryInventoryReader, FilesystemRecoveryNextHeadFinalizationOpenError, @@ -75,27 +76,28 @@ pub use adapters::{ FilesystemRecoverySegmentResumer, FilesystemRecoverySegmentStage, FilesystemRecoveryStageCompleter, FilesystemRecoveryStageCompletionOpenError, FilesystemRecoveryStageDiscardOpenError, FilesystemRecoveryStageDiscarder, - FilesystemRecoveryStageError, FilesystemSegmentStage, FilesystemStoreMigrationInventoryReader, - FilesystemWriterLock, ImmutablePoolInventoryDigest, InitialGcStateDigest, - InitialRetentionStateDigest, LayoutDecodeError, LayoutDecodePolicy, LayoutEncodeError, - LayoutIdBinaryParseError, LayoutIdTextParseError, MigrationInventoryNamespace, - MigrationInventoryPool, MigrationSynchronizationMask, OpenedReusableSegment, - PublicationHeadDecodeError, RecoveryCatalogStage, RecoveryCatalogStageError, RecoveryEntryName, - RecoveryEntryNameError, RecoveryEntryRole, RecoveryInventory, RecoveryInventoryEntry, - RecoveryInventoryError, RecoveryInventoryLimit, RecoveryInventoryLimitError, - RecoveryInventoryOperation, RecoveryInventoryStorage, RecoveryNameClassificationError, - RecoveryNameManifest, RecoveryNamedEntry, RecoveryNamespace, RecoveryNextHeadFinalizationError, - RecoveryNextHeadFinalizationOutcome, RecoveryNextHeadFinalizationPlanError, - RecoveryNextHeadFinalizationReadiness, RecoveryNextHeadFinalizationReceipt, - RecoveryNextHeadFinalizationRequest, RecoveryNextHeadFinalizationStorage, - RecoveryNextHeadFinalizationStorageError, RecoveryNextHeadFinalizationTarget, - RecoveryNextHeadStage, RecoveryNextHeadStageError, RecoveryPoolNameError, - RecoveryRequiredEntry, RecoverySegmentResumeError, RecoverySegmentResumePlanError, - RecoverySegmentResumeRequest, RecoverySegmentResumeStorage, RecoverySegmentResumeStorageError, - RecoverySegmentStage, RecoverySegmentStageError, RecoverySegmentTruncation, RecoveryStage, - RecoveryStageAssessment, RecoveryStageAssessmentError, RecoveryStageByteAdmissionError, - RecoveryStageCompletionError, RecoveryStageCompletionPlanError, RecoveryStageCompletionPool, - RecoveryStageCompletionReceipt, RecoveryStageCompletionRequest, RecoveryStageCompletionStorage, + FilesystemRecoveryStageError, FilesystemSegmentStage, FilesystemStoreMigrationAuthority, + FilesystemStoreMigrationInventoryReader, FilesystemWriterLock, ImmutablePoolInventoryDigest, + InitialGcStateDigest, InitialRetentionStateDigest, LayoutDecodeError, LayoutDecodePolicy, + LayoutEncodeError, LayoutIdBinaryParseError, LayoutIdTextParseError, + MigrationInventoryNamespace, MigrationInventoryPool, MigrationSynchronizationMask, + OpenedReusableSegment, PublicationHeadDecodeError, RecoveryCatalogStage, + RecoveryCatalogStageError, RecoveryEntryName, RecoveryEntryNameError, RecoveryEntryRole, + RecoveryInventory, RecoveryInventoryEntry, RecoveryInventoryError, RecoveryInventoryLimit, + RecoveryInventoryLimitError, RecoveryInventoryOperation, RecoveryInventoryStorage, + RecoveryNameClassificationError, RecoveryNameManifest, RecoveryNamedEntry, RecoveryNamespace, + RecoveryNextHeadFinalizationError, RecoveryNextHeadFinalizationOutcome, + RecoveryNextHeadFinalizationPlanError, RecoveryNextHeadFinalizationReadiness, + RecoveryNextHeadFinalizationReceipt, RecoveryNextHeadFinalizationRequest, + RecoveryNextHeadFinalizationStorage, RecoveryNextHeadFinalizationStorageError, + RecoveryNextHeadFinalizationTarget, RecoveryNextHeadStage, RecoveryNextHeadStageError, + RecoveryPoolNameError, RecoveryRequiredEntry, RecoverySegmentResumeError, + RecoverySegmentResumePlanError, RecoverySegmentResumeRequest, RecoverySegmentResumeStorage, + RecoverySegmentResumeStorageError, RecoverySegmentStage, RecoverySegmentStageError, + RecoverySegmentTruncation, RecoveryStage, RecoveryStageAssessment, + RecoveryStageAssessmentError, RecoveryStageByteAdmissionError, RecoveryStageCompletionError, + RecoveryStageCompletionPlanError, RecoveryStageCompletionPool, RecoveryStageCompletionReceipt, + RecoveryStageCompletionRequest, RecoveryStageCompletionStorage, RecoveryStageCompletionStorageError, RecoveryStageCompletionTarget, RecoveryStageDiscardError, RecoveryStageDiscardOutcome, RecoveryStageDiscardPlanError, RecoveryStageDiscardReason, RecoveryStageDiscardReceipt, RecoveryStageDiscardRequest, RecoveryStageDiscardStorage, @@ -117,9 +119,10 @@ pub use adapters::{ StoreMigrationInventoryEntryCount, StoreMigrationInventoryEntryCountError, StoreMigrationInventoryError, StoreMigrationInventoryHasher, StoreMigrationPhase, StoreMigrationReceiptDecodeError, StoreMigrationStorage, StoreRootDeviceIdentity, - StoreRootFileIdentity, StoreRootMountIdentity, WriterLockAcquireError, WriterLockAcquirePhase, - admit_recovery_stage_bytes, assess_recovery_stage, classify_recovery_catalog_stage, - classify_recovery_names, classify_recovery_next_head_stage, classify_recovery_segment_stage, + StoreRootFileIdentity, StoreRootIdentityCoordinate, StoreRootMountIdentity, + WriterLockAcquireError, WriterLockAcquirePhase, admit_recovery_stage_bytes, + assess_recovery_stage, classify_recovery_catalog_stage, classify_recovery_names, + classify_recovery_next_head_stage, classify_recovery_segment_stage, execute_recovery_next_head_finalization, execute_recovery_segment_resume, execute_recovery_stage_completion, execute_recovery_stage_discard, execute_store_migration, fingerprint_recovery_stage, initialize_store, plan_recovery_next_head_finalization, From 5b26a1dbc9095f853bd4fda63e69195b74353228 Mon Sep 17 00:00:00 2001 From: James Ross Date: Tue, 4 Aug 2026 02:17:37 -0700 Subject: [PATCH 043/111] Stream recovery reads and reject trailing artifacts --- src/adapters/catalog_restart_io.rs | 153 ++++++++++++- ...lesystem_recovery_stage_materialization.rs | 213 +++++++++++++++++- 2 files changed, 351 insertions(+), 15 deletions(-) diff --git a/src/adapters/catalog_restart_io.rs b/src/adapters/catalog_restart_io.rs index 0a62c94..41513c6 100644 --- a/src/adapters/catalog_restart_io.rs +++ b/src/adapters/catalog_restart_io.rs @@ -9,6 +9,8 @@ use cap_std::fs::{Dir, File, OpenOptions}; use super::{CatalogRestartArtifact, CatalogRestartError, CatalogRestartPhase}; +const CATALOG_RESTART_READ_BUFFER_LENGTH: usize = 8_192; + pub(super) fn open_root(root: &Path) -> Result { Dir::open_ambient_dir(root, ambient_authority()) .map_err(|source| CatalogRestartError::io(CatalogRestartPhase::OpenRoot, source)) @@ -46,6 +48,7 @@ pub(super) fn read_exact( byte_count: expected, source: None, })?; + let mut encoded = Vec::new(); encoded .try_reserve_exact(host_length) @@ -54,22 +57,79 @@ pub(super) fn read_exact( byte_count: expected, source: Some(source), })?; - encoded.resize(host_length, 0); - file.read_exact(&mut encoded) - .map_err(|source| CatalogRestartError::io(phase, source))?; - reject_trailing_bytes(&mut file, artifact, phase, expected)?; + read_exact_to(&mut file, artifact, phase, expected, |chunk| { + encoded.extend_from_slice(chunk); + Ok(()) + })?; Ok(encoded) } -fn reject_trailing_bytes( - file: &mut File, +pub(super) fn read_exact_to( + source: &mut R, + artifact: CatalogRestartArtifact, + phase: CatalogRestartPhase, + expected: u64, + mut on_chunk: F, +) -> Result<(), CatalogRestartError> +where + R: Read, + F: FnMut(&[u8]) -> Result<(), CatalogRestartError>, +{ + let mut observed = 0_u64; + let mut buffer = [0_u8; CATALOG_RESTART_READ_BUFFER_LENGTH]; + let chunk_length = u64::try_from(buffer.len()) + .map_err(|_source| CatalogRestartError::LengthArithmetic { artifact, expected })?; + + while observed < expected { + let remaining = expected + .checked_sub(observed) + .ok_or(CatalogRestartError::LengthArithmetic { artifact, expected })?; + let offered = remaining + .min(chunk_length) + .try_into() + .map_err(|_source| CatalogRestartError::LengthArithmetic { artifact, expected })?; + let read_buffer = buffer + .get_mut(..offered) + .ok_or(CatalogRestartError::LengthArithmetic { artifact, expected })?; + match source.read(read_buffer) { + Ok(0) => { + return Err(CatalogRestartError::io( + phase, + io::Error::new( + io::ErrorKind::UnexpectedEof, + "restart artifact ended before the expected boundary", + ), + )); + } + Ok(count) => { + let bytes = read_buffer + .get(..count) + .ok_or(CatalogRestartError::LengthArithmetic { artifact, expected })?; + on_chunk(bytes)?; + let increment = u64::try_from(count).map_err(|_source| { + CatalogRestartError::LengthArithmetic { artifact, expected } + })?; + observed = observed + .checked_add(increment) + .ok_or(CatalogRestartError::LengthArithmetic { artifact, expected })?; + } + Err(source) if source.kind() == io::ErrorKind::Interrupted => {} + Err(source) => return Err(CatalogRestartError::io(phase, source)), + } + } + reject_trailing_bytes(source, artifact, phase, expected)?; + Ok(()) +} + +fn reject_trailing_bytes( + source: &mut R, artifact: CatalogRestartArtifact, phase: CatalogRestartPhase, expected: u64, ) -> Result<(), CatalogRestartError> { let mut trailing = [0_u8; 1]; loop { - match file.read(&mut trailing) { + match source.read(&mut trailing) { Ok(0) => return Ok(()), Ok(observed) => { let increment = u64::try_from(observed).map_err(|_source| { @@ -90,3 +150,82 @@ fn reject_trailing_bytes( } } } + +#[cfg(test)] +mod tests { + use std::io::{Cursor, ErrorKind}; + + use super::*; + + #[test] + fn read_exact_to_streams_chunks() { + let mut source = Cursor::new(vec![b'a', b'b', b'c', b'd', b'e', b'f', b'g']); + let mut observed = Vec::>::new(); + + let result = read_exact_to( + &mut source, + CatalogRestartArtifact::Head, + CatalogRestartPhase::ReadCatalog, + 7, + |chunk| { + observed.push(chunk.to_vec()); + Ok(()) + }, + ); + + assert!(result.is_ok()); + assert_eq!(observed.concat(), b"abcdefg"); + } + + #[test] + fn read_exact_to_rejects_short_artifacts() { + let mut source = Cursor::new(vec![b'a', b'b']); + let mut seen = 0_u8; + + let result = read_exact_to( + &mut source, + CatalogRestartArtifact::Head, + CatalogRestartPhase::ReadCatalog, + 4, + |_chunk| { + seen = seen.checked_add(1).expect("unexpected chunk overflow"); + Ok(()) + }, + ); + + let error = result.unwrap_err(); + assert_eq!(seen, 1); + assert!(matches!( + error, + CatalogRestartError::Io { + phase: CatalogRestartPhase::ReadCatalog, + ref source, + } if source.kind() == ErrorKind::UnexpectedEof + )); + } + + #[test] + fn read_exact_to_rejects_trailing_bytes() { + let mut source = Cursor::new(vec![b'a', b'b', b'c']); + + let result = read_exact_to( + &mut source, + CatalogRestartArtifact::Head, + CatalogRestartPhase::ReadCatalog, + 2, + |_| Ok(()), + ); + + let error = result.unwrap_err(); + let expected = 2_u64; + assert!(matches!( + error, + CatalogRestartError::Length { + artifact: CatalogRestartArtifact::Head, + minimum, + maximum, + observed: 3 + } if minimum == expected && maximum == expected + )); + } +} diff --git a/src/adapters/filesystem_recovery_stage_materialization.rs b/src/adapters/filesystem_recovery_stage_materialization.rs index 0f769a2..d005b48 100644 --- a/src/adapters/filesystem_recovery_stage_materialization.rs +++ b/src/adapters/filesystem_recovery_stage_materialization.rs @@ -1,6 +1,6 @@ //! This module owns exact writable recovery-stage materialization. -use std::io::{Read, Seek, SeekFrom}; +use std::io::{self, Read, Seek, SeekFrom}; use cap_std::fs::File; @@ -14,12 +14,7 @@ pub(super) fn read_and_position( let mut encoded = allocate(stage, length)?; file.seek(SeekFrom::Start(0)) .map_err(|source| FilesystemRecoveryStageError::Position { stage, source })?; - file.read_exact(&mut encoded) - .map_err(|source| FilesystemRecoveryStageError::Materialize { - stage, - expected: length, - source, - })?; + read_exact(file, stage, length, &mut encoded)?; verify_position(file, stage, length)?; Ok(encoded.into_boxed_slice()) } @@ -42,10 +37,87 @@ fn allocate( source, } })?; - encoded.resize(host_length, 0); Ok(encoded) } +fn read_exact( + file: &mut File, + stage: RecoveryStage, + length: RecoveryStageLength, + encoded: &mut Vec, +) -> Result<(), FilesystemRecoveryStageError> { + let expected = length.get(); + let observed = file + .by_ref() + .take(expected) + .read_to_end(encoded) + .map_err(|source| FilesystemRecoveryStageError::Materialize { + stage, + expected: length, + source, + })?; + let observed = + u64::try_from(observed).map_err(|_source| FilesystemRecoveryStageError::LengthChanged { + stage, + expected: length, + observed: expected, + })?; + if observed < expected { + return Err(FilesystemRecoveryStageError::Materialize { + stage, + expected: length, + source: io::Error::new( + io::ErrorKind::UnexpectedEof, + "recovery stage ended before the expected boundary", + ), + }); + } + reject_trailing_bytes(file, stage, length)?; + Ok(()) +} + +fn reject_trailing_bytes( + file: &mut File, + stage: RecoveryStage, + expected: RecoveryStageLength, +) -> Result<(), FilesystemRecoveryStageError> { + let mut trailing = [0_u8; 1]; + loop { + match file.read(&mut trailing) { + Ok(0) => return Ok(()), + Ok(read_bytes) => { + let increment = u64::try_from(read_bytes).map_err(|_source| { + FilesystemRecoveryStageError::LengthChanged { + stage, + expected, + observed: expected.get(), + } + })?; + let observed = expected.get().checked_add(increment).ok_or( + FilesystemRecoveryStageError::LengthChanged { + stage, + expected, + observed: expected.get(), + }, + )?; + return Err(FilesystemRecoveryStageError::LengthChanged { + stage, + expected, + observed, + }); + } + Err(source) if source.kind() == io::ErrorKind::Interrupted => continue, + Err(source) => { + return Err(FilesystemRecoveryStageError::Materialize { + stage, + expected, + source, + }); + } + } + } +} + pub(super) fn verify_position( file: &mut File, stage: RecoveryStage, @@ -64,3 +136,128 @@ pub(super) fn verify_position( }) } } + +#[cfg(test)] +mod tests { + use std::error::Error; + use std::fs; + use std::path::PathBuf; + use std::sync::atomic::{AtomicU64, Ordering}; + + use cap_fs_ext::{FollowSymlinks, OpenOptionsFollowExt, OpenOptionsSyncExt}; + use cap_std::fs::OpenOptions; + use cap_std::{ambient_authority, fs::Dir}; + + use super::*; + + static NEXT_DIRECTORY: AtomicU64 = AtomicU64::new(0); + + #[test] + fn read_exact_reads_expected_bytes_without_trailing() -> Result<(), Box> { + let sandbox = TestDirectory::create("stage-materialization-exact")?; + let path = sandbox.path().join("stage.bin"); + fs::write(&path, b"abcdef")?; + let mut file = open_for_tests(&path)?; + let encoded = super::read_and_position( + &mut file, + RecoveryStage::Segment, + RecoveryStageLength::from_validated(6), + )?; + assert_eq!(encoded.as_ref(), b"abcdef"); + drop(file); + sandbox.remove()?; + Ok(()) + } + + #[test] + fn read_and_position_rejects_short_stage() -> Result<(), Box> { + let sandbox = TestDirectory::create("stage-materialization-short")?; + let path = sandbox.path().join("stage.bin"); + fs::write(&path, b"abc")?; + let mut file = open_for_tests(&path)?; + let error = super::read_and_position( + &mut file, + RecoveryStage::Segment, + RecoveryStageLength::from_validated(5), + ) + .expect_err("short stage materialization was admitted"); + + assert!(matches!( + error, + FilesystemRecoveryStageError::Materialize { + stage: RecoveryStage::Segment, + expected, + source, + } if expected.get() == 5 && source.kind() == std::io::ErrorKind::UnexpectedEof + )); + drop(file); + sandbox.remove()?; + Ok(()) + } + + #[test] + fn read_and_position_rejects_trailing_bytes() -> Result<(), Box> { + let sandbox = TestDirectory::create("stage-materialization-trailing")?; + let path = sandbox.path().join("stage.bin"); + fs::write(&path, b"abcdef")?; + let mut file = open_for_tests(&path)?; + let error = super::read_and_position( + &mut file, + RecoveryStage::Segment, + RecoveryStageLength::from_validated(3), + ) + .expect_err("trailing-stage materialization was admitted"); + + assert!(matches!( + error, + FilesystemRecoveryStageError::LengthChanged { + stage: RecoveryStage::Segment, + expected, + observed: 4, + } if expected.get() == 3 + )); + drop(file); + sandbox.remove()?; + Ok(()) + } + + fn open_for_tests(path: &PathBuf) -> Result> { + let directory = Dir::open_ambient_dir( + path.parent() + .expect("directory parent exists for stage fixture"), + ambient_authority(), + )?; + let mut options = OpenOptions::new(); + options.read(true).follow(FollowSymlinks::No).nonblock(true); + let file = directory.open_with( + path.file_name() + .expect("file path has file name for fixture") + .to_str() + .expect("file name is UTF-8 for fixture"), + &options, + )?; + Ok(file) + } + + struct TestDirectory { + path: PathBuf, + } + + impl TestDirectory { + fn create(name: &str) -> std::io::Result { + let sequence = NEXT_DIRECTORY.fetch_add(1, Ordering::Relaxed); + let path = + std::env::temp_dir().join(format!("keep-{name}-{}-{sequence}", std::process::id())); + fs::create_dir(&path)?; + Ok(Self { path }) + } + + fn path(&self) -> &std::path::Path { + &self.path + } + + fn remove(self) -> std::io::Result<()> { + fs::remove_dir_all(self.path) + } + } +} From 2fffbdca475b15d7daa2c4a5df42ec61ba0f5f86 Mon Sep 17 00:00:00 2001 From: James Ross Date: Tue, 4 Aug 2026 02:27:42 -0700 Subject: [PATCH 044/111] Test: add streaming large-input callback memory harness --- src/adapters/catalog_restart_io.rs | 158 ++++++++++++++++++++++++++++- 1 file changed, 157 insertions(+), 1 deletion(-) diff --git a/src/adapters/catalog_restart_io.rs b/src/adapters/catalog_restart_io.rs index 41513c6..140d5e1 100644 --- a/src/adapters/catalog_restart_io.rs +++ b/src/adapters/catalog_restart_io.rs @@ -153,10 +153,52 @@ fn reject_trailing_bytes( #[cfg(test)] mod tests { - use std::io::{Cursor, ErrorKind}; + use std::io; + use std::io::{Cursor, ErrorKind, Read}; + use std::mem::size_of; use super::*; + #[test] + fn read_exact_to_streams_large_virtual_file_with_small_callback_memory() { + const TOTAL_BYTES: u64 = 64_u64 * 1024 * 1024; + const READER_STRIDE: u64 = 2_u64 * 1024; + const CALLBACK_BUDGET_BYTES: usize = 16 * 1024; + + let mut source = SyntheticStreamingReader::new(TOTAL_BYTES, READER_STRIDE); + let mut budget = StreamingCallbackBudget::new(TOTAL_BYTES, CALLBACK_BUDGET_BYTES); + + let result = read_exact_to( + &mut source, + CatalogRestartArtifact::Head, + CatalogRestartPhase::ReadCatalog, + TOTAL_BYTES, + |chunk| budget.consume(chunk), + ); + + assert!(result.is_ok(), "{result:?}"); + assert!(budget.observed_bytes() > 0); + assert_eq!(budget.observed_bytes(), TOTAL_BYTES); + assert!( + budget.max_chunk() >= READER_STRIDE as usize, + "reader stride should be observed" + ); + assert!( + budget.max_chunk() <= budget.callback_limit(), + "callback should remain in budget" + ); + assert!( + budget.max_chunk() <= CATALOG_RESTART_READ_BUFFER_LENGTH, + "read buffer bounds should hold" + ); + assert!(budget.total_chunks() > 0); + assert!( + size_of::() < 128, + "callback state should stay compact" + ); + assert_eq!(budget.observed_bytes(), TOTAL_BYTES); + } + #[test] fn read_exact_to_streams_chunks() { let mut source = Cursor::new(vec![b'a', b'b', b'c', b'd', b'e', b'f', b'g']); @@ -228,4 +270,118 @@ mod tests { } if minimum == expected && maximum == expected )); } + + struct SyntheticStreamingReader { + remaining: u64, + emit_stride: u64, + } + + impl SyntheticStreamingReader { + fn new(total: u64, emit_stride: u64) -> Self { + Self { + remaining: total, + emit_stride, + } + } + } + + impl Read for SyntheticStreamingReader { + fn read(&mut self, sink: &mut [u8]) -> io::Result { + if self.remaining == 0 { + return Ok(0); + } + + let sink_capacity = match u64::try_from(sink.len()) { + Ok(capacity) => capacity, + Err(_) => { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + "sink capacity exceeds supported range", + )); + } + }; + let emitted: usize = match self + .emit_stride + .min(self.remaining) + .min(sink_capacity) + .try_into() + { + Ok(size) => size, + Err(_) => { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + "requested read size exceeds supported range", + )); + } + }; + + sink[..emitted].fill(0x5a); + self.remaining -= u64::try_from(emitted) + .map_err(|_| io::Error::new(io::ErrorKind::InvalidInput, "emit size overflow"))?; + Ok(emitted) + } + } + + struct StreamingCallbackBudget { + observed_bytes: u64, + total_chunks: u64, + max_chunk: usize, + callback_limit: usize, + expected_total: u64, + } + + impl StreamingCallbackBudget { + fn new(expected_total: u64, callback_limit: usize) -> Self { + Self { + observed_bytes: 0, + total_chunks: 0, + max_chunk: 0, + callback_limit, + expected_total, + } + } + + fn consume(&mut self, chunk: &[u8]) -> Result<(), CatalogRestartError> { + self.total_chunks = + self.total_chunks + .checked_add(1) + .ok_or(CatalogRestartError::LengthArithmetic { + artifact: CatalogRestartArtifact::Head, + expected: self.expected_total, + })?; + + self.max_chunk = self.max_chunk.max(chunk.len()); + + self.observed_bytes = self + .observed_bytes + .checked_add(u64::try_from(chunk.len()).map_err(|_source| { + CatalogRestartError::LengthArithmetic { + artifact: CatalogRestartArtifact::Head, + expected: self.expected_total, + } + })?) + .ok_or(CatalogRestartError::LengthArithmetic { + artifact: CatalogRestartArtifact::Head, + expected: self.expected_total, + })?; + + Ok(()) + } + + fn observed_bytes(&self) -> u64 { + self.observed_bytes + } + + fn max_chunk(&self) -> usize { + self.max_chunk + } + + fn callback_limit(&self) -> usize { + self.callback_limit + } + + fn total_chunks(&self) -> u64 { + self.total_chunks + } + } } From ee876cf6a4ea911597ba4461b7005f62b6afd0f2 Mon Sep 17 00:00:00 2001 From: James Ross Date: Tue, 4 Aug 2026 02:33:24 -0700 Subject: [PATCH 045/111] Test: add streaming write_exact_to regression coverage --- src/adapters/catalog_restart_io.rs | 198 ++++++++++++++++++++++++++++- 1 file changed, 196 insertions(+), 2 deletions(-) diff --git a/src/adapters/catalog_restart_io.rs b/src/adapters/catalog_restart_io.rs index 140d5e1..9bbccfa 100644 --- a/src/adapters/catalog_restart_io.rs +++ b/src/adapters/catalog_restart_io.rs @@ -1,4 +1,5 @@ -//! This module owns exact capability-relative restart artifact reads. +//! This module owns exact capability-relative restart artifact reads and +//! bounded streaming writes. use std::io::{self, Read}; use std::path::Path; @@ -64,6 +65,66 @@ pub(super) fn read_exact( Ok(encoded) } +#[cfg(test)] +pub(super) fn write_exact_to( + source: &mut R, + destination: &mut W, + artifact: CatalogRestartArtifact, + phase: CatalogRestartPhase, + expected: u64, +) -> Result<(), CatalogRestartError> +where + R: Read, + W: io::Write, +{ + let mut observed = 0_u64; + let mut buffer = [0_u8; CATALOG_RESTART_READ_BUFFER_LENGTH]; + let chunk_length = u64::try_from(buffer.len()) + .map_err(|_source| CatalogRestartError::LengthArithmetic { artifact, expected })?; + + while observed < expected { + let remaining = expected + .checked_sub(observed) + .ok_or(CatalogRestartError::LengthArithmetic { artifact, expected })?; + let offered = remaining + .min(chunk_length) + .try_into() + .map_err(|_source| CatalogRestartError::LengthArithmetic { artifact, expected })?; + let read_buffer = buffer + .get_mut(..offered) + .ok_or(CatalogRestartError::LengthArithmetic { artifact, expected })?; + match source.read(read_buffer) { + Ok(0) => { + return Err(CatalogRestartError::io( + phase, + io::Error::new( + io::ErrorKind::UnexpectedEof, + "restart artifact ended before the expected boundary", + ), + )); + } + Ok(count) => { + let bytes = read_buffer + .get(..count) + .ok_or(CatalogRestartError::LengthArithmetic { artifact, expected })?; + destination + .write_all(bytes) + .map_err(|source| CatalogRestartError::io(phase, source))?; + let increment = u64::try_from(count).map_err(|_source| { + CatalogRestartError::LengthArithmetic { artifact, expected } + })?; + observed = observed + .checked_add(increment) + .ok_or(CatalogRestartError::LengthArithmetic { artifact, expected })?; + } + Err(source) if source.kind() == io::ErrorKind::Interrupted => {} + Err(source) => return Err(CatalogRestartError::io(phase, source)), + } + } + reject_trailing_bytes(source, artifact, phase, expected)?; + Ok(()) +} + pub(super) fn read_exact_to( source: &mut R, artifact: CatalogRestartArtifact, @@ -154,7 +215,7 @@ fn reject_trailing_bytes( #[cfg(test)] mod tests { use std::io; - use std::io::{Cursor, ErrorKind, Read}; + use std::io::{Cursor, ErrorKind, Read, Write}; use std::mem::size_of; use super::*; @@ -199,6 +260,32 @@ mod tests { assert_eq!(budget.observed_bytes(), TOTAL_BYTES); } + #[test] + fn write_exact_to_streams_large_virtual_file_with_small_writer_state() { + const TOTAL_BYTES: u64 = 64_u64 * 1024 * 1024; + const READER_STRIDE: u64 = 2_u64 * 1024; + const WRITER_BUDGET_BYTES: usize = 4 * 1024; + + let mut source = SyntheticStreamingReader::new(TOTAL_BYTES, READER_STRIDE); + let mut sink = StreamingWriteSink::new(WRITER_BUDGET_BYTES); + + let result = write_exact_to( + &mut source, + &mut sink, + CatalogRestartArtifact::Head, + CatalogRestartPhase::ReadCatalog, + TOTAL_BYTES, + ); + + assert!(result.is_ok(), "{result:?}"); + assert_eq!(sink.observed_bytes(), TOTAL_BYTES); + assert!(sink.total_chunks() > 0); + assert!(sink.max_chunk() <= WRITER_BUDGET_BYTES); + assert!(sink.max_chunk() <= CATALOG_RESTART_READ_BUFFER_LENGTH); + assert!(sink.max_chunk() >= READER_STRIDE as usize); + assert!(size_of::() < 64); + } + #[test] fn read_exact_to_streams_chunks() { let mut source = Cursor::new(vec![b'a', b'b', b'c', b'd', b'e', b'f', b'g']); @@ -271,6 +358,57 @@ mod tests { )); } + #[test] + fn write_exact_to_rejects_short_artifacts() { + let mut source = Cursor::new(vec![b'a', b'b']); + let mut sink = StreamingWriteSink::new(16 * 1024); + + let result = write_exact_to( + &mut source, + &mut sink, + CatalogRestartArtifact::Head, + CatalogRestartPhase::ReadCatalog, + 4, + ); + + let error = result.unwrap_err(); + assert_eq!(sink.observed_bytes(), 2); + assert!(matches!( + error, + CatalogRestartError::Io { + phase: CatalogRestartPhase::ReadCatalog, + ref source, + } if source.kind() == ErrorKind::UnexpectedEof + )); + } + + #[test] + fn write_exact_to_rejects_trailing_bytes() { + let mut source = Cursor::new(vec![b'a', b'b', b'c']); + let mut sink = StreamingWriteSink::new(16 * 1024); + + let result = write_exact_to( + &mut source, + &mut sink, + CatalogRestartArtifact::Head, + CatalogRestartPhase::ReadCatalog, + 2, + ); + + let error = result.unwrap_err(); + let expected = 2_u64; + assert_eq!(sink.observed_bytes(), 2); + assert!(matches!( + error, + CatalogRestartError::Length { + artifact: CatalogRestartArtifact::Head, + minimum, + maximum, + observed: 3 + } if minimum == expected && maximum == expected + )); + } + struct SyntheticStreamingReader { remaining: u64, emit_stride: u64, @@ -384,4 +522,60 @@ mod tests { self.total_chunks } } + + struct StreamingWriteSink { + observed_bytes: u64, + observed_chunks: u64, + max_chunk: usize, + writer_memory_limit: usize, + } + + impl StreamingWriteSink { + fn new(writer_memory_limit: usize) -> Self { + Self { + observed_bytes: 0, + observed_chunks: 0, + max_chunk: 0, + writer_memory_limit, + } + } + + fn observed_bytes(&self) -> u64 { + self.observed_bytes + } + + fn total_chunks(&self) -> u64 { + self.observed_chunks + } + + fn max_chunk(&self) -> usize { + self.max_chunk + } + } + + impl Write for StreamingWriteSink { + fn write(&mut self, bytes: &[u8]) -> io::Result { + self.observed_chunks = self.observed_chunks.checked_add(1).ok_or_else(|| { + io::Error::new(io::ErrorKind::InvalidInput, "chunk count overflow") + })?; + let observed = u64::try_from(bytes.len()).map_err(|_| { + io::Error::new(io::ErrorKind::InvalidInput, "chunk length overflow") + })?; + self.observed_bytes = self.observed_bytes.checked_add(observed).ok_or_else(|| { + io::Error::new(io::ErrorKind::InvalidInput, "write count overflow") + })?; + self.max_chunk = self.max_chunk.max(bytes.len()); + if bytes.len() > self.writer_memory_limit { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + "sink memory budget exceeded", + )); + } + Ok(bytes.len()) + } + + fn flush(&mut self) -> io::Result<()> { + Ok(()) + } + } } From a0f60f63549b66141cd8870bb7fe11903bd74e7c Mon Sep 17 00:00:00 2001 From: James Ross Date: Tue, 4 Aug 2026 02:39:52 -0700 Subject: [PATCH 046/111] refactor: shape catalog restart streaming transfer API --- src/adapters/catalog_restart_io.rs | 206 +++++++++++++++-------------- 1 file changed, 108 insertions(+), 98 deletions(-) diff --git a/src/adapters/catalog_restart_io.rs b/src/adapters/catalog_restart_io.rs index 9bbccfa..a54e532 100644 --- a/src/adapters/catalog_restart_io.rs +++ b/src/adapters/catalog_restart_io.rs @@ -1,5 +1,5 @@ //! This module owns exact capability-relative restart artifact reads and -//! bounded streaming writes. +//! bounded, exact-transfer streaming. use std::io::{self, Read}; use std::path::Path; @@ -12,6 +12,39 @@ use super::{CatalogRestartArtifact, CatalogRestartError, CatalogRestartPhase}; const CATALOG_RESTART_READ_BUFFER_LENGTH: usize = 8_192; +#[derive(Clone, Copy, Debug)] +pub(super) struct ExactTransfer { + artifact: CatalogRestartArtifact, + phase: CatalogRestartPhase, + expected: u64, +} + +impl ExactTransfer { + const fn new( + artifact: CatalogRestartArtifact, + phase: CatalogRestartPhase, + expected: u64, + ) -> Self { + Self { + artifact, + phase, + expected, + } + } + + fn artifact(&self) -> CatalogRestartArtifact { + self.artifact + } + + fn phase(&self) -> CatalogRestartPhase { + self.phase + } + + fn expected(&self) -> u64 { + self.expected + } +} + pub(super) fn open_root(root: &Path) -> Result { Dir::open_ambient_dir(root, ambient_authority()) .map_err(|source| CatalogRestartError::io(CatalogRestartPhase::OpenRoot, source)) @@ -43,99 +76,59 @@ pub(super) fn read_exact( phase: CatalogRestartPhase, expected: u64, ) -> Result, CatalogRestartError> { - let host_length = - usize::try_from(expected).map_err(|_source| CatalogRestartError::Allocation { - artifact, - byte_count: expected, + let transfer = ExactTransfer::new(artifact, phase, expected); + let host_length = usize::try_from(transfer.expected()).map_err(|_source| { + CatalogRestartError::Allocation { + artifact: transfer.artifact(), + byte_count: transfer.expected(), source: None, - })?; + } + })?; let mut encoded = Vec::new(); encoded .try_reserve_exact(host_length) .map_err(|source| CatalogRestartError::Allocation { - artifact, + artifact: transfer.artifact(), byte_count: expected, source: Some(source), })?; - read_exact_to(&mut file, artifact, phase, expected, |chunk| { + copy_exact_to_chunks(&mut file, transfer, |chunk| { encoded.extend_from_slice(chunk); Ok(()) })?; Ok(encoded) } -#[cfg(test)] -pub(super) fn write_exact_to( +pub(super) fn copy_exact( source: &mut R, destination: &mut W, - artifact: CatalogRestartArtifact, - phase: CatalogRestartPhase, - expected: u64, -) -> Result<(), CatalogRestartError> + transfer: ExactTransfer, +) -> Result where R: Read, W: io::Write, { - let mut observed = 0_u64; - let mut buffer = [0_u8; CATALOG_RESTART_READ_BUFFER_LENGTH]; - let chunk_length = u64::try_from(buffer.len()) - .map_err(|_source| CatalogRestartError::LengthArithmetic { artifact, expected })?; - - while observed < expected { - let remaining = expected - .checked_sub(observed) - .ok_or(CatalogRestartError::LengthArithmetic { artifact, expected })?; - let offered = remaining - .min(chunk_length) - .try_into() - .map_err(|_source| CatalogRestartError::LengthArithmetic { artifact, expected })?; - let read_buffer = buffer - .get_mut(..offered) - .ok_or(CatalogRestartError::LengthArithmetic { artifact, expected })?; - match source.read(read_buffer) { - Ok(0) => { - return Err(CatalogRestartError::io( - phase, - io::Error::new( - io::ErrorKind::UnexpectedEof, - "restart artifact ended before the expected boundary", - ), - )); - } - Ok(count) => { - let bytes = read_buffer - .get(..count) - .ok_or(CatalogRestartError::LengthArithmetic { artifact, expected })?; - destination - .write_all(bytes) - .map_err(|source| CatalogRestartError::io(phase, source))?; - let increment = u64::try_from(count).map_err(|_source| { - CatalogRestartError::LengthArithmetic { artifact, expected } - })?; - observed = observed - .checked_add(increment) - .ok_or(CatalogRestartError::LengthArithmetic { artifact, expected })?; - } - Err(source) if source.kind() == io::ErrorKind::Interrupted => {} - Err(source) => return Err(CatalogRestartError::io(phase, source)), - } - } - reject_trailing_bytes(source, artifact, phase, expected)?; - Ok(()) + copy_exact_to_chunks(source, transfer, |chunk| { + destination + .write_all(chunk) + .map_err(|source| CatalogRestartError::io(transfer.phase(), source)) + }) } -pub(super) fn read_exact_to( +pub(super) fn copy_exact_to_chunks( source: &mut R, - artifact: CatalogRestartArtifact, - phase: CatalogRestartPhase, - expected: u64, + transfer: ExactTransfer, mut on_chunk: F, -) -> Result<(), CatalogRestartError> +) -> Result where R: Read, F: FnMut(&[u8]) -> Result<(), CatalogRestartError>, { + let artifact = transfer.artifact(); + let phase = transfer.phase(); + let expected = transfer.expected(); + let mut observed = 0_u64; let mut buffer = [0_u8; CATALOG_RESTART_READ_BUFFER_LENGTH]; let chunk_length = u64::try_from(buffer.len()) @@ -179,7 +172,7 @@ where } } reject_trailing_bytes(source, artifact, phase, expected)?; - Ok(()) + Ok(observed) } fn reject_trailing_bytes( @@ -229,11 +222,13 @@ mod tests { let mut source = SyntheticStreamingReader::new(TOTAL_BYTES, READER_STRIDE); let mut budget = StreamingCallbackBudget::new(TOTAL_BYTES, CALLBACK_BUDGET_BYTES); - let result = read_exact_to( + let result = copy_exact_to_chunks( &mut source, - CatalogRestartArtifact::Head, - CatalogRestartPhase::ReadCatalog, - TOTAL_BYTES, + ExactTransfer::new( + CatalogRestartArtifact::Head, + CatalogRestartPhase::ReadCatalog, + TOTAL_BYTES, + ), |chunk| budget.consume(chunk), ); @@ -261,7 +256,7 @@ mod tests { } #[test] - fn write_exact_to_streams_large_virtual_file_with_small_writer_state() { + fn copy_exact_streams_large_virtual_file_with_small_writer_state() { const TOTAL_BYTES: u64 = 64_u64 * 1024 * 1024; const READER_STRIDE: u64 = 2_u64 * 1024; const WRITER_BUDGET_BYTES: usize = 4 * 1024; @@ -269,15 +264,20 @@ mod tests { let mut source = SyntheticStreamingReader::new(TOTAL_BYTES, READER_STRIDE); let mut sink = StreamingWriteSink::new(WRITER_BUDGET_BYTES); - let result = write_exact_to( + let result = copy_exact( &mut source, &mut sink, - CatalogRestartArtifact::Head, - CatalogRestartPhase::ReadCatalog, - TOTAL_BYTES, + ExactTransfer::new( + CatalogRestartArtifact::Head, + CatalogRestartPhase::ReadCatalog, + TOTAL_BYTES, + ), ); - assert!(result.is_ok(), "{result:?}"); + let observed = result.unwrap_or_else(|error| { + panic!("copy should succeed for expected length: {error:?}"); + }); + assert_eq!(observed, TOTAL_BYTES); assert_eq!(sink.observed_bytes(), TOTAL_BYTES); assert!(sink.total_chunks() > 0); assert!(sink.max_chunk() <= WRITER_BUDGET_BYTES); @@ -291,11 +291,13 @@ mod tests { let mut source = Cursor::new(vec![b'a', b'b', b'c', b'd', b'e', b'f', b'g']); let mut observed = Vec::>::new(); - let result = read_exact_to( + let result = copy_exact_to_chunks( &mut source, - CatalogRestartArtifact::Head, - CatalogRestartPhase::ReadCatalog, - 7, + ExactTransfer::new( + CatalogRestartArtifact::Head, + CatalogRestartPhase::ReadCatalog, + 7, + ), |chunk| { observed.push(chunk.to_vec()); Ok(()) @@ -311,11 +313,13 @@ mod tests { let mut source = Cursor::new(vec![b'a', b'b']); let mut seen = 0_u8; - let result = read_exact_to( + let result = copy_exact_to_chunks( &mut source, - CatalogRestartArtifact::Head, - CatalogRestartPhase::ReadCatalog, - 4, + ExactTransfer::new( + CatalogRestartArtifact::Head, + CatalogRestartPhase::ReadCatalog, + 4, + ), |_chunk| { seen = seen.checked_add(1).expect("unexpected chunk overflow"); Ok(()) @@ -337,11 +341,13 @@ mod tests { fn read_exact_to_rejects_trailing_bytes() { let mut source = Cursor::new(vec![b'a', b'b', b'c']); - let result = read_exact_to( + let result = copy_exact_to_chunks( &mut source, - CatalogRestartArtifact::Head, - CatalogRestartPhase::ReadCatalog, - 2, + ExactTransfer::new( + CatalogRestartArtifact::Head, + CatalogRestartPhase::ReadCatalog, + 2, + ), |_| Ok(()), ); @@ -359,16 +365,18 @@ mod tests { } #[test] - fn write_exact_to_rejects_short_artifacts() { + fn copy_exact_rejects_short_artifacts() { let mut source = Cursor::new(vec![b'a', b'b']); let mut sink = StreamingWriteSink::new(16 * 1024); - let result = write_exact_to( + let result = copy_exact( &mut source, &mut sink, - CatalogRestartArtifact::Head, - CatalogRestartPhase::ReadCatalog, - 4, + ExactTransfer::new( + CatalogRestartArtifact::Head, + CatalogRestartPhase::ReadCatalog, + 4, + ), ); let error = result.unwrap_err(); @@ -383,16 +391,18 @@ mod tests { } #[test] - fn write_exact_to_rejects_trailing_bytes() { + fn copy_exact_rejects_trailing_bytes() { let mut source = Cursor::new(vec![b'a', b'b', b'c']); let mut sink = StreamingWriteSink::new(16 * 1024); - let result = write_exact_to( + let result = copy_exact( &mut source, &mut sink, - CatalogRestartArtifact::Head, - CatalogRestartPhase::ReadCatalog, - 2, + ExactTransfer::new( + CatalogRestartArtifact::Head, + CatalogRestartPhase::ReadCatalog, + 2, + ), ); let error = result.unwrap_err(); From 593e448b8c71d0d128bfb699a20c1c96e69f544e Mon Sep 17 00:00:00 2001 From: James Ross Date: Tue, 4 Aug 2026 02:42:09 -0700 Subject: [PATCH 047/111] feat: expose transfer-specific restart IO copy API --- src/adapters/catalog_restart_io.rs | 39 +++++++++++++++++++++--------- 1 file changed, 27 insertions(+), 12 deletions(-) diff --git a/src/adapters/catalog_restart_io.rs b/src/adapters/catalog_restart_io.rs index a54e532..bca44ce 100644 --- a/src/adapters/catalog_restart_io.rs +++ b/src/adapters/catalog_restart_io.rs @@ -1,7 +1,7 @@ //! This module owns exact capability-relative restart artifact reads and //! bounded, exact-transfer streaming. -use std::io::{self, Read}; +use std::io::{self, Read, Write}; use std::path::Path; use cap_fs_ext::{FollowSymlinks, OpenOptionsFollowExt, OpenOptionsSyncExt}; @@ -20,7 +20,7 @@ pub(super) struct ExactTransfer { } impl ExactTransfer { - const fn new( + pub(super) const fn new( artifact: CatalogRestartArtifact, phase: CatalogRestartPhase, expected: u64, @@ -32,15 +32,15 @@ impl ExactTransfer { } } - fn artifact(&self) -> CatalogRestartArtifact { + pub(super) const fn artifact(&self) -> CatalogRestartArtifact { self.artifact } - fn phase(&self) -> CatalogRestartPhase { + pub(super) const fn phase(&self) -> CatalogRestartPhase { self.phase } - fn expected(&self) -> u64 { + pub(super) const fn expected(&self) -> u64 { self.expected } } @@ -93,10 +93,10 @@ pub(super) fn read_exact( byte_count: expected, source: Some(source), })?; - copy_exact_to_chunks(&mut file, transfer, |chunk| { - encoded.extend_from_slice(chunk); - Ok(()) - })?; + let mut sink = VecWrite { + encoded: &mut encoded, + }; + copy_exact(&mut file, &mut sink, transfer)?; Ok(encoded) } @@ -205,6 +205,21 @@ fn reject_trailing_bytes( } } +struct VecWrite<'a> { + encoded: &'a mut Vec, +} + +impl Write for VecWrite<'_> { + fn write(&mut self, bytes: &[u8]) -> io::Result { + self.encoded.extend_from_slice(bytes); + Ok(bytes.len()) + } + + fn flush(&mut self) -> io::Result<()> { + Ok(()) + } +} + #[cfg(test)] mod tests { use std::io; @@ -287,7 +302,7 @@ mod tests { } #[test] - fn read_exact_to_streams_chunks() { + fn copy_exact_to_chunks_streams() { let mut source = Cursor::new(vec![b'a', b'b', b'c', b'd', b'e', b'f', b'g']); let mut observed = Vec::>::new(); @@ -309,7 +324,7 @@ mod tests { } #[test] - fn read_exact_to_rejects_short_artifacts() { + fn copy_exact_to_chunks_rejects_short_artifacts() { let mut source = Cursor::new(vec![b'a', b'b']); let mut seen = 0_u8; @@ -338,7 +353,7 @@ mod tests { } #[test] - fn read_exact_to_rejects_trailing_bytes() { + fn copy_exact_to_chunks_rejects_trailing_bytes() { let mut source = Cursor::new(vec![b'a', b'b', b'c']); let result = copy_exact_to_chunks( From 57aaa76be49f1745fa140f24f61089fdeb562935 Mon Sep 17 00:00:00 2001 From: James Ross Date: Tue, 4 Aug 2026 02:59:00 -0700 Subject: [PATCH 048/111] Fix: enforce checked conversions in streaming IO adapters --- src/adapters/catalog_restart_io.rs | 132 ++++++++++++------ ...lesystem_recovery_stage_materialization.rs | 62 ++++---- src/adapters/filesystem_root_identity.rs | 4 + 3 files changed, 132 insertions(+), 66 deletions(-) diff --git a/src/adapters/catalog_restart_io.rs b/src/adapters/catalog_restart_io.rs index bca44ce..cfd32ae 100644 --- a/src/adapters/catalog_restart_io.rs +++ b/src/adapters/catalog_restart_io.rs @@ -222,6 +222,7 @@ impl Write for VecWrite<'_> { #[cfg(test)] mod tests { + use std::error::Error; use std::io; use std::io::{Cursor, ErrorKind, Read, Write}; use std::mem::size_of; @@ -229,15 +230,22 @@ mod tests { use super::*; #[test] - fn read_exact_to_streams_large_virtual_file_with_small_callback_memory() { + fn read_exact_to_streams_large_virtual_file_with_small_callback_memory() + -> Result<(), Box> { const TOTAL_BYTES: u64 = 64_u64 * 1024 * 1024; - const READER_STRIDE: u64 = 2_u64 * 1024; + const READER_STRIDE_BYTES: usize = 2_usize * 1024; const CALLBACK_BUDGET_BYTES: usize = 16 * 1024; - - let mut source = SyntheticStreamingReader::new(TOTAL_BYTES, READER_STRIDE); + let Ok(reader_stride) = u64::try_from(READER_STRIDE_BYTES) else { + return Err(Box::new(io::Error::new( + io::ErrorKind::InvalidData, + "reader stride is outside supported range", + ))); + }; + + let mut source = SyntheticStreamingReader::new(TOTAL_BYTES, reader_stride); let mut budget = StreamingCallbackBudget::new(TOTAL_BYTES, CALLBACK_BUDGET_BYTES); - let result = copy_exact_to_chunks( + let _observed = copy_exact_to_chunks( &mut source, ExactTransfer::new( CatalogRestartArtifact::Head, @@ -245,13 +253,12 @@ mod tests { TOTAL_BYTES, ), |chunk| budget.consume(chunk), - ); + )?; - assert!(result.is_ok(), "{result:?}"); assert!(budget.observed_bytes() > 0); assert_eq!(budget.observed_bytes(), TOTAL_BYTES); assert!( - budget.max_chunk() >= READER_STRIDE as usize, + budget.max_chunk() >= READER_STRIDE_BYTES, "reader stride should be observed" ); assert!( @@ -268,18 +275,26 @@ mod tests { "callback state should stay compact" ); assert_eq!(budget.observed_bytes(), TOTAL_BYTES); + Ok(()) } #[test] - fn copy_exact_streams_large_virtual_file_with_small_writer_state() { + fn copy_exact_streams_large_virtual_file_with_small_writer_state() -> Result<(), Box> + { const TOTAL_BYTES: u64 = 64_u64 * 1024 * 1024; - const READER_STRIDE: u64 = 2_u64 * 1024; + const READER_STRIDE_BYTES: usize = 2_usize * 1024; const WRITER_BUDGET_BYTES: usize = 4 * 1024; - - let mut source = SyntheticStreamingReader::new(TOTAL_BYTES, READER_STRIDE); + let Ok(reader_stride) = u64::try_from(READER_STRIDE_BYTES) else { + return Err(Box::new(io::Error::new( + io::ErrorKind::InvalidData, + "reader stride is outside supported range", + ))); + }; + + let mut source = SyntheticStreamingReader::new(TOTAL_BYTES, reader_stride); let mut sink = StreamingWriteSink::new(WRITER_BUDGET_BYTES); - let result = copy_exact( + let observed = copy_exact( &mut source, &mut sink, ExactTransfer::new( @@ -287,26 +302,23 @@ mod tests { CatalogRestartPhase::ReadCatalog, TOTAL_BYTES, ), - ); - - let observed = result.unwrap_or_else(|error| { - panic!("copy should succeed for expected length: {error:?}"); - }); + )?; assert_eq!(observed, TOTAL_BYTES); assert_eq!(sink.observed_bytes(), TOTAL_BYTES); assert!(sink.total_chunks() > 0); assert!(sink.max_chunk() <= WRITER_BUDGET_BYTES); assert!(sink.max_chunk() <= CATALOG_RESTART_READ_BUFFER_LENGTH); - assert!(sink.max_chunk() >= READER_STRIDE as usize); + assert!(sink.max_chunk() >= READER_STRIDE_BYTES); assert!(size_of::() < 64); + Ok(()) } #[test] - fn copy_exact_to_chunks_streams() { + fn copy_exact_to_chunks_streams() -> Result<(), Box> { let mut source = Cursor::new(vec![b'a', b'b', b'c', b'd', b'e', b'f', b'g']); let mut observed = Vec::>::new(); - let result = copy_exact_to_chunks( + let _observed = copy_exact_to_chunks( &mut source, ExactTransfer::new( CatalogRestartArtifact::Head, @@ -317,14 +329,14 @@ mod tests { observed.push(chunk.to_vec()); Ok(()) }, - ); + )?; - assert!(result.is_ok()); assert_eq!(observed.concat(), b"abcdefg"); + Ok(()) } #[test] - fn copy_exact_to_chunks_rejects_short_artifacts() { + fn copy_exact_to_chunks_rejects_short_artifacts() -> Result<(), Box> { let mut source = Cursor::new(vec![b'a', b'b']); let mut seen = 0_u8; @@ -336,12 +348,25 @@ mod tests { 4, ), |_chunk| { - seen = seen.checked_add(1).expect("unexpected chunk overflow"); + seen = match seen.checked_add(1) { + Some(total) => total, + None => { + return Err(CatalogRestartError::LengthArithmetic { + artifact: CatalogRestartArtifact::Head, + expected: 4, + }); + } + }; Ok(()) }, ); - let error = result.unwrap_err(); + let Err(error) = result else { + return Err(Box::new(io::Error::new( + io::ErrorKind::InvalidData, + "short artifact should have been rejected", + ))); + }; assert_eq!(seen, 1); assert!(matches!( error, @@ -350,10 +375,11 @@ mod tests { ref source, } if source.kind() == ErrorKind::UnexpectedEof )); + Ok(()) } #[test] - fn copy_exact_to_chunks_rejects_trailing_bytes() { + fn copy_exact_to_chunks_rejects_trailing_bytes() -> Result<(), Box> { let mut source = Cursor::new(vec![b'a', b'b', b'c']); let result = copy_exact_to_chunks( @@ -366,7 +392,12 @@ mod tests { |_| Ok(()), ); - let error = result.unwrap_err(); + let Err(error) = result else { + return Err(Box::new(io::Error::new( + io::ErrorKind::InvalidData, + "trailing bytes should have been rejected", + ))); + }; let expected = 2_u64; assert!(matches!( error, @@ -377,10 +408,11 @@ mod tests { observed: 3 } if minimum == expected && maximum == expected )); + Ok(()) } #[test] - fn copy_exact_rejects_short_artifacts() { + fn copy_exact_rejects_short_artifacts() -> Result<(), Box> { let mut source = Cursor::new(vec![b'a', b'b']); let mut sink = StreamingWriteSink::new(16 * 1024); @@ -394,7 +426,12 @@ mod tests { ), ); - let error = result.unwrap_err(); + let Err(error) = result else { + return Err(Box::new(io::Error::new( + io::ErrorKind::InvalidData, + "short artifact should have been rejected", + ))); + }; assert_eq!(sink.observed_bytes(), 2); assert!(matches!( error, @@ -403,10 +440,11 @@ mod tests { ref source, } if source.kind() == ErrorKind::UnexpectedEof )); + Ok(()) } #[test] - fn copy_exact_rejects_trailing_bytes() { + fn copy_exact_rejects_trailing_bytes() -> Result<(), Box> { let mut source = Cursor::new(vec![b'a', b'b', b'c']); let mut sink = StreamingWriteSink::new(16 * 1024); @@ -420,7 +458,12 @@ mod tests { ), ); - let error = result.unwrap_err(); + let Err(error) = result else { + return Err(Box::new(io::Error::new( + io::ErrorKind::InvalidData, + "trailing bytes should have been rejected", + ))); + }; let expected = 2_u64; assert_eq!(sink.observed_bytes(), 2); assert!(matches!( @@ -432,6 +475,7 @@ mod tests { observed: 3 } if minimum == expected && maximum == expected )); + Ok(()) } struct SyntheticStreamingReader { @@ -454,14 +498,11 @@ mod tests { return Ok(0); } - let sink_capacity = match u64::try_from(sink.len()) { - Ok(capacity) => capacity, - Err(_) => { - return Err(io::Error::new( - io::ErrorKind::InvalidInput, - "sink capacity exceeds supported range", - )); - } + let Ok(sink_capacity) = u64::try_from(sink.len()) else { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + "sink capacity exceeds supported range", + )); }; let emitted: usize = match self .emit_stride @@ -478,9 +519,16 @@ mod tests { } }; - sink[..emitted].fill(0x5a); - self.remaining -= u64::try_from(emitted) + let read_window = sink.get_mut(..emitted).ok_or_else(|| { + io::Error::new(io::ErrorKind::InvalidInput, "read window overflow") + })?; + read_window.fill(0x5a); + let emitted_u64 = u64::try_from(emitted) .map_err(|_| io::Error::new(io::ErrorKind::InvalidInput, "emit size overflow"))?; + self.remaining = self + .remaining + .checked_sub(emitted_u64) + .ok_or_else(|| io::Error::new(io::ErrorKind::InvalidInput, "emit underflow"))?; Ok(emitted) } } diff --git a/src/adapters/filesystem_recovery_stage_materialization.rs b/src/adapters/filesystem_recovery_stage_materialization.rs index d005b48..c1a98aa 100644 --- a/src/adapters/filesystem_recovery_stage_materialization.rs +++ b/src/adapters/filesystem_recovery_stage_materialization.rs @@ -93,20 +93,20 @@ fn reject_trailing_bytes( observed: expected.get(), } })?; - let observed = expected.get().checked_add(increment).ok_or( + let observed = expected.get().checked_add(increment).ok_or_else(|| { FilesystemRecoveryStageError::LengthChanged { stage, expected, observed: expected.get(), - }, - )?; + } + })?; return Err(FilesystemRecoveryStageError::LengthChanged { stage, expected, observed, }); } - Err(source) if source.kind() == io::ErrorKind::Interrupted => continue, + Err(source) if source.kind() == io::ErrorKind::Interrupted => {} Err(source) => { return Err(FilesystemRecoveryStageError::Materialize { stage, @@ -141,7 +141,8 @@ pub(super) fn verify_position( mod tests { use std::error::Error; use std::fs; - use std::path::PathBuf; + use std::io; + use std::path::{Path, PathBuf}; use std::sync::atomic::{AtomicU64, Ordering}; use cap_fs_ext::{FollowSymlinks, OpenOptionsFollowExt, OpenOptionsSyncExt}; @@ -175,12 +176,16 @@ mod tests { let path = sandbox.path().join("stage.bin"); fs::write(&path, b"abc")?; let mut file = open_for_tests(&path)?; - let error = super::read_and_position( + let Err(error) = super::read_and_position( &mut file, RecoveryStage::Segment, RecoveryStageLength::from_validated(5), - ) - .expect_err("short stage materialization was admitted"); + ) else { + return Err(Box::new(io::Error::new( + io::ErrorKind::InvalidData, + "short stage materialization should have failed", + ))); + }; assert!(matches!( error, @@ -201,12 +206,16 @@ mod tests { let path = sandbox.path().join("stage.bin"); fs::write(&path, b"abcdef")?; let mut file = open_for_tests(&path)?; - let error = super::read_and_position( + let Err(error) = super::read_and_position( &mut file, RecoveryStage::Segment, RecoveryStageLength::from_validated(3), - ) - .expect_err("trailing-stage materialization was admitted"); + ) else { + return Err(Box::new(io::Error::new( + io::ErrorKind::InvalidData, + "trailing stage materialization should have failed", + ))); + }; assert!(matches!( error, @@ -221,21 +230,26 @@ mod tests { Ok(()) } - fn open_for_tests(path: &PathBuf) -> Result> { - let directory = Dir::open_ambient_dir( - path.parent() - .expect("directory parent exists for stage fixture"), - ambient_authority(), - )?; + fn open_for_tests(path: &Path) -> Result> { + let directory_path = path.parent().ok_or_else(|| { + io::Error::new( + io::ErrorKind::InvalidInput, + "stage fixture path does not have a parent directory", + ) + })?; + let file_name = path + .file_name() + .and_then(|name| name.to_str()) + .ok_or_else(|| { + io::Error::new( + io::ErrorKind::InvalidInput, + "stage fixture path has no UTF-8 file name", + ) + })?; + let directory = Dir::open_ambient_dir(directory_path, ambient_authority())?; let mut options = OpenOptions::new(); options.read(true).follow(FollowSymlinks::No).nonblock(true); - let file = directory.open_with( - path.file_name() - .expect("file path has file name for fixture") - .to_str() - .expect("file name is UTF-8 for fixture"), - &options, - )?; + let file = directory.open_with(file_name, &options)?; Ok(file) } diff --git a/src/adapters/filesystem_root_identity.rs b/src/adapters/filesystem_root_identity.rs index e63619f..36af8bb 100644 --- a/src/adapters/filesystem_root_identity.rs +++ b/src/adapters/filesystem_root_identity.rs @@ -8,6 +8,10 @@ pub(super) struct FilesystemRootIdentity { } impl FilesystemRootIdentity { + #[cfg(any( + target_os = "linux", + all(not(target_os = "linux"), any(test, feature = "repository-tasks")) + ))] pub(super) const fn new(device: u64, mount: u64, file: u64) -> Self { Self { device, From 877fef013e356d1272c1de7627ecf6fb6d2581f9 Mon Sep 17 00:00:00 2001 From: James Ross Date: Sat, 15 Aug 2026 04:02:34 -0700 Subject: [PATCH 049/111] Refactor: isolate catalog restart IO test doubles --- src/adapters/catalog_restart_io.rs | 179 +---------------- .../catalog_restart_io_test_doubles.rs | 180 ++++++++++++++++++ src/adapters/mod.rs | 2 + 3 files changed, 186 insertions(+), 175 deletions(-) create mode 100644 src/adapters/catalog_restart_io_test_doubles.rs diff --git a/src/adapters/catalog_restart_io.rs b/src/adapters/catalog_restart_io.rs index cfd32ae..3af6b0f 100644 --- a/src/adapters/catalog_restart_io.rs +++ b/src/adapters/catalog_restart_io.rs @@ -224,9 +224,12 @@ impl Write for VecWrite<'_> { mod tests { use std::error::Error; use std::io; - use std::io::{Cursor, ErrorKind, Read, Write}; + use std::io::{Cursor, ErrorKind}; use std::mem::size_of; + use super::super::catalog_restart_io_test_doubles::{ + StreamingCallbackBudget, StreamingWriteSink, SyntheticStreamingReader, + }; use super::*; #[test] @@ -477,178 +480,4 @@ mod tests { )); Ok(()) } - - struct SyntheticStreamingReader { - remaining: u64, - emit_stride: u64, - } - - impl SyntheticStreamingReader { - fn new(total: u64, emit_stride: u64) -> Self { - Self { - remaining: total, - emit_stride, - } - } - } - - impl Read for SyntheticStreamingReader { - fn read(&mut self, sink: &mut [u8]) -> io::Result { - if self.remaining == 0 { - return Ok(0); - } - - let Ok(sink_capacity) = u64::try_from(sink.len()) else { - return Err(io::Error::new( - io::ErrorKind::InvalidInput, - "sink capacity exceeds supported range", - )); - }; - let emitted: usize = match self - .emit_stride - .min(self.remaining) - .min(sink_capacity) - .try_into() - { - Ok(size) => size, - Err(_) => { - return Err(io::Error::new( - io::ErrorKind::InvalidInput, - "requested read size exceeds supported range", - )); - } - }; - - let read_window = sink.get_mut(..emitted).ok_or_else(|| { - io::Error::new(io::ErrorKind::InvalidInput, "read window overflow") - })?; - read_window.fill(0x5a); - let emitted_u64 = u64::try_from(emitted) - .map_err(|_| io::Error::new(io::ErrorKind::InvalidInput, "emit size overflow"))?; - self.remaining = self - .remaining - .checked_sub(emitted_u64) - .ok_or_else(|| io::Error::new(io::ErrorKind::InvalidInput, "emit underflow"))?; - Ok(emitted) - } - } - - struct StreamingCallbackBudget { - observed_bytes: u64, - total_chunks: u64, - max_chunk: usize, - callback_limit: usize, - expected_total: u64, - } - - impl StreamingCallbackBudget { - fn new(expected_total: u64, callback_limit: usize) -> Self { - Self { - observed_bytes: 0, - total_chunks: 0, - max_chunk: 0, - callback_limit, - expected_total, - } - } - - fn consume(&mut self, chunk: &[u8]) -> Result<(), CatalogRestartError> { - self.total_chunks = - self.total_chunks - .checked_add(1) - .ok_or(CatalogRestartError::LengthArithmetic { - artifact: CatalogRestartArtifact::Head, - expected: self.expected_total, - })?; - - self.max_chunk = self.max_chunk.max(chunk.len()); - - self.observed_bytes = self - .observed_bytes - .checked_add(u64::try_from(chunk.len()).map_err(|_source| { - CatalogRestartError::LengthArithmetic { - artifact: CatalogRestartArtifact::Head, - expected: self.expected_total, - } - })?) - .ok_or(CatalogRestartError::LengthArithmetic { - artifact: CatalogRestartArtifact::Head, - expected: self.expected_total, - })?; - - Ok(()) - } - - fn observed_bytes(&self) -> u64 { - self.observed_bytes - } - - fn max_chunk(&self) -> usize { - self.max_chunk - } - - fn callback_limit(&self) -> usize { - self.callback_limit - } - - fn total_chunks(&self) -> u64 { - self.total_chunks - } - } - - struct StreamingWriteSink { - observed_bytes: u64, - observed_chunks: u64, - max_chunk: usize, - writer_memory_limit: usize, - } - - impl StreamingWriteSink { - fn new(writer_memory_limit: usize) -> Self { - Self { - observed_bytes: 0, - observed_chunks: 0, - max_chunk: 0, - writer_memory_limit, - } - } - - fn observed_bytes(&self) -> u64 { - self.observed_bytes - } - - fn total_chunks(&self) -> u64 { - self.observed_chunks - } - - fn max_chunk(&self) -> usize { - self.max_chunk - } - } - - impl Write for StreamingWriteSink { - fn write(&mut self, bytes: &[u8]) -> io::Result { - self.observed_chunks = self.observed_chunks.checked_add(1).ok_or_else(|| { - io::Error::new(io::ErrorKind::InvalidInput, "chunk count overflow") - })?; - let observed = u64::try_from(bytes.len()).map_err(|_| { - io::Error::new(io::ErrorKind::InvalidInput, "chunk length overflow") - })?; - self.observed_bytes = self.observed_bytes.checked_add(observed).ok_or_else(|| { - io::Error::new(io::ErrorKind::InvalidInput, "write count overflow") - })?; - self.max_chunk = self.max_chunk.max(bytes.len()); - if bytes.len() > self.writer_memory_limit { - return Err(io::Error::new( - io::ErrorKind::InvalidInput, - "sink memory budget exceeded", - )); - } - Ok(bytes.len()) - } - - fn flush(&mut self) -> io::Result<()> { - Ok(()) - } - } } diff --git a/src/adapters/catalog_restart_io_test_doubles.rs b/src/adapters/catalog_restart_io_test_doubles.rs new file mode 100644 index 0000000..ebe5026 --- /dev/null +++ b/src/adapters/catalog_restart_io_test_doubles.rs @@ -0,0 +1,180 @@ +//! This module owns bounded streaming doubles for catalog restart I/O laws. + +use std::io::{self, Read, Write}; + +use super::{CatalogRestartArtifact, CatalogRestartError}; + +pub(super) struct SyntheticStreamingReader { + remaining: u64, + emit_stride: u64, +} + +impl SyntheticStreamingReader { + pub(super) fn new(total: u64, emit_stride: u64) -> Self { + Self { + remaining: total, + emit_stride, + } + } +} + +impl Read for SyntheticStreamingReader { + fn read(&mut self, sink: &mut [u8]) -> io::Result { + if self.remaining == 0 { + return Ok(0); + } + + let Ok(sink_capacity) = u64::try_from(sink.len()) else { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + "sink capacity exceeds supported range", + )); + }; + let emitted: usize = match self + .emit_stride + .min(self.remaining) + .min(sink_capacity) + .try_into() + { + Ok(size) => size, + Err(_) => { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + "requested read size exceeds supported range", + )); + } + }; + + let read_window = sink + .get_mut(..emitted) + .ok_or_else(|| io::Error::new(io::ErrorKind::InvalidInput, "read window overflow"))?; + read_window.fill(0x5a); + let emitted_u64 = u64::try_from(emitted) + .map_err(|_| io::Error::new(io::ErrorKind::InvalidInput, "emit size overflow"))?; + self.remaining = self + .remaining + .checked_sub(emitted_u64) + .ok_or_else(|| io::Error::new(io::ErrorKind::InvalidInput, "emit underflow"))?; + Ok(emitted) + } +} + +pub(super) struct StreamingCallbackBudget { + observed_bytes: u64, + total_chunks: u64, + max_chunk: usize, + callback_limit: usize, + expected_total: u64, +} + +impl StreamingCallbackBudget { + pub(super) fn new(expected_total: u64, callback_limit: usize) -> Self { + Self { + observed_bytes: 0, + total_chunks: 0, + max_chunk: 0, + callback_limit, + expected_total, + } + } + + pub(super) fn consume(&mut self, chunk: &[u8]) -> Result<(), CatalogRestartError> { + self.total_chunks = + self.total_chunks + .checked_add(1) + .ok_or(CatalogRestartError::LengthArithmetic { + artifact: CatalogRestartArtifact::Head, + expected: self.expected_total, + })?; + + self.max_chunk = self.max_chunk.max(chunk.len()); + + self.observed_bytes = self + .observed_bytes + .checked_add(u64::try_from(chunk.len()).map_err(|_source| { + CatalogRestartError::LengthArithmetic { + artifact: CatalogRestartArtifact::Head, + expected: self.expected_total, + } + })?) + .ok_or(CatalogRestartError::LengthArithmetic { + artifact: CatalogRestartArtifact::Head, + expected: self.expected_total, + })?; + + Ok(()) + } + + pub(super) fn observed_bytes(&self) -> u64 { + self.observed_bytes + } + + pub(super) fn max_chunk(&self) -> usize { + self.max_chunk + } + + pub(super) fn callback_limit(&self) -> usize { + self.callback_limit + } + + pub(super) fn total_chunks(&self) -> u64 { + self.total_chunks + } +} + +pub(super) struct StreamingWriteSink { + observed_bytes: u64, + observed_chunks: u64, + max_chunk: usize, + writer_memory_limit: usize, +} + +impl StreamingWriteSink { + pub(super) fn new(writer_memory_limit: usize) -> Self { + Self { + observed_bytes: 0, + observed_chunks: 0, + max_chunk: 0, + writer_memory_limit, + } + } + + pub(super) fn observed_bytes(&self) -> u64 { + self.observed_bytes + } + + pub(super) fn total_chunks(&self) -> u64 { + self.observed_chunks + } + + pub(super) fn max_chunk(&self) -> usize { + self.max_chunk + } +} + +impl Write for StreamingWriteSink { + fn write(&mut self, bytes: &[u8]) -> io::Result { + self.observed_chunks = self + .observed_chunks + .checked_add(1) + .ok_or_else(|| io::Error::new(io::ErrorKind::InvalidInput, "chunk count overflow"))?; + let observed = u64::try_from(bytes.len()) + .map_err(|_| io::Error::new(io::ErrorKind::InvalidInput, "chunk length overflow"))?; + self.observed_bytes = self + .observed_bytes + .checked_add(observed) + .ok_or_else(|| io::Error::new(io::ErrorKind::InvalidInput, "write count overflow"))?; + self.max_chunk = self.max_chunk.max(bytes.len()); + if bytes.len() > self.writer_memory_limit { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + "sink memory budget exceeded", + )); + } + Ok(bytes.len()) + } + + fn flush(&mut self) -> io::Result<()> { + Ok(()) + } +} diff --git a/src/adapters/mod.rs b/src/adapters/mod.rs index 3c783ca..6d69ae8 100644 --- a/src/adapters/mod.rs +++ b/src/adapters/mod.rs @@ -50,6 +50,8 @@ mod catalog_restart_artifact; mod catalog_restart_byte_limit; mod catalog_restart_error; mod catalog_restart_io; +#[cfg(test)] +mod catalog_restart_io_test_doubles; mod catalog_restart_loader; mod catalog_restart_phase; mod catalog_restart_policy; From c195bb1aa6563c4c1ebfd79b4cf7e3b2ada1d079 Mon Sep 17 00:00:00 2001 From: James Ross Date: Sat, 15 Aug 2026 04:34:55 -0700 Subject: [PATCH 050/111] Fix: isolate root identity from platform admission --- src/adapters/filesystem_platform_profile.rs | 34 +++++++++++++++---- .../filesystem_platform_profile_tests.rs | 10 +++--- .../durability_crash_matrix/error/display.rs | 23 ++++++++++++- 3 files changed, 53 insertions(+), 14 deletions(-) diff --git a/src/adapters/filesystem_platform_profile.rs b/src/adapters/filesystem_platform_profile.rs index 1b6bbda..ca0423c 100644 --- a/src/adapters/filesystem_platform_profile.rs +++ b/src/adapters/filesystem_platform_profile.rs @@ -19,7 +19,6 @@ struct LinuxDirectoryProperties { device_major: u32, device_minor: u32, mount_id: u64, - inode: u64, } #[cfg(target_os = "linux")] @@ -85,21 +84,42 @@ fn linux_directory_properties(file: &std::fs::File) -> io::Result io::Result { let file = directory.try_clone()?.into_std_file(); - let properties = linux_directory_properties(&file)?; - Ok(linux_root_identity(properties)) + linux_file_identity(&file) +} + +#[cfg(target_os = "linux")] +fn linux_file_identity(file: &std::fs::File) -> io::Result { + use rustix::fs::{AtFlags, StatxFlags, statx}; + + let required = StatxFlags::BASIC_STATS | StatxFlags::MNT_ID; + let status = statx(file, ".", AtFlags::empty(), required)?; + let observed = StatxFlags::from_bits_retain(status.stx_mask); + if !observed.contains(required) { + return Err(unsupported_linux_profile()); + } + Ok(linux_root_identity( + status.stx_dev_major, + status.stx_dev_minor, + status.stx_mnt_id, + status.stx_ino, + )) } #[cfg(target_os = "linux")] -fn linux_root_identity(properties: LinuxDirectoryProperties) -> FilesystemRootIdentity { - let device = rustix::fs::makedev(properties.device_major, properties.device_minor); - FilesystemRootIdentity::new(device, properties.mount_id, properties.inode) +fn linux_root_identity( + device_major: u32, + device_minor: u32, + mount_id: u64, + inode: u64, +) -> FilesystemRootIdentity { + let device = rustix::fs::makedev(device_major, device_minor); + FilesystemRootIdentity::new(device, mount_id, inode) } #[cfg(all(not(target_os = "linux"), any(test, feature = "repository-tasks")))] diff --git a/src/adapters/filesystem_platform_profile_tests.rs b/src/adapters/filesystem_platform_profile_tests.rs index dd9d63e..4f4776e 100644 --- a/src/adapters/filesystem_platform_profile_tests.rs +++ b/src/adapters/filesystem_platform_profile_tests.rs @@ -32,7 +32,7 @@ fn only_writable_case_sensitive_ext4_is_admitted() { #[test] fn every_protocol_child_must_share_the_root_filesystem_and_mount() { assert_eq!(PROTOCOL_DIRECTORIES, ["staging", "segments", "catalogs"]); - let root = properties(8, 1, 41, 1); + let root = properties(8, 1, 41); let mut casefolded = root; casefolded.inode_flags = EXT4_CASEFOLD_FLAG; let mut read_only = root; @@ -41,8 +41,8 @@ fn every_protocol_child_must_share_the_root_filesystem_and_mount() { foreign_format.filesystem_type = NFS_SUPER_MAGIC; assert!(admit_linux_child_properties(root, root).is_ok()); - assert_unsupported(&admit_linux_child_properties(root, properties(8, 2, 41, 1))); - assert_unsupported(&admit_linux_child_properties(root, properties(8, 1, 42, 1))); + assert_unsupported(&admit_linux_child_properties(root, properties(8, 2, 41))); + assert_unsupported(&admit_linux_child_properties(root, properties(8, 1, 42))); assert_unsupported(&admit_linux_child_properties(root, casefolded)); assert_unsupported(&admit_linux_child_properties(root, read_only)); assert_unsupported(&admit_linux_child_properties(root, foreign_format)); @@ -50,7 +50,7 @@ fn every_protocol_child_must_share_the_root_filesystem_and_mount() { #[test] fn root_identity_uses_linux_device_mount_and_inode_coordinates() { - let identity = linux_root_identity(properties(8, 1, 41, 73)); + let identity = linux_root_identity(8, 1, 41, 73); assert_eq!(identity.device(), rustix::fs::makedev(8, 1)); assert_eq!(identity.mount(), 41); assert_eq!(identity.file(), 73); @@ -70,7 +70,6 @@ const fn properties( device_major: u32, device_minor: u32, mount_id: u64, - inode: u64, ) -> LinuxDirectoryProperties { LinuxDirectoryProperties { filesystem_type: EXT4_SUPER_MAGIC, @@ -79,6 +78,5 @@ const fn properties( device_major, device_minor, mount_id, - inode, } } diff --git a/xtask/src/durability_crash_matrix/error/display.rs b/xtask/src/durability_crash_matrix/error/display.rs index b9a215d..79f24e8 100644 --- a/xtask/src/durability_crash_matrix/error/display.rs +++ b/xtask/src/durability_crash_matrix/error/display.rs @@ -217,7 +217,9 @@ fn format_boundary( formatter: &mut fmt::Formatter<'_>, ) -> fmt::Result { match error { - DurabilityCrashMatrixError::Io { action, .. } => write!(formatter, "cannot {action}"), + DurabilityCrashMatrixError::Io { action, source } => { + write!(formatter, "cannot {action}: {source}") + } DurabilityCrashMatrixError::NonUnicodeStatePath => { formatter.write_str("post-crash store path is not valid Unicode") } @@ -233,3 +235,22 @@ fn format_boundary( _ => Err(fmt::Error), } } + +#[cfg(test)] +mod tests { + use std::io; + + use super::DurabilityCrashMatrixError; + + #[test] + fn io_boundary_diagnostics_preserve_the_exact_source() { + let error = DurabilityCrashMatrixError::io( + "open crash catalog publisher", + io::Error::new(io::ErrorKind::Unsupported, "profile probe escaped bypass"), + ); + assert_eq!( + error.to_string(), + "cannot open crash catalog publisher: profile probe escaped bypass" + ); + } +} From 45fbb441a5376ad41666b084b2fdecf57ea953b7 Mon Sep 17 00:00:00 2001 From: James Ross Date: Sun, 23 Aug 2026 03:46:12 -0700 Subject: [PATCH 051/111] Add fresh filesystem store migration --- README.md | 10 +- docs/formats/segment-store-v2/README.md | 8 +- .../segment-store-v2/migration-crash.md | 5 +- docs/formats/segment-store-v2/recovery.md | 28 +- docs/formats/segment-store-v2/requirements.md | 13 +- src/adapters/store_migration.rs | 9 + .../filesystem_migration_authority.rs | 28 +- .../filesystem_migration_authority_tests.rs | 36 +-- .../filesystem_migration_fixed_artifact.rs | 242 ++++++++++++++++++ .../filesystem_migration_namespace.rs | 228 +++++++++++++++++ ...ilesystem_migration_namespace_directory.rs | 188 ++++++++++++++ .../filesystem_migration_reader_fence.rs | 62 +++++ .../filesystem_migration_storage.rs | 242 ++++++++++++++++++ .../filesystem_migration_storage_tests.rs | 203 +++++++++++++++ .../filesystem_migration_test_fixture.rs | 40 +++ src/lib.rs | 9 +- 16 files changed, 1285 insertions(+), 66 deletions(-) create mode 100644 src/adapters/store_migration/filesystem_migration_fixed_artifact.rs create mode 100644 src/adapters/store_migration/filesystem_migration_namespace.rs create mode 100644 src/adapters/store_migration/filesystem_migration_namespace_directory.rs create mode 100644 src/adapters/store_migration/filesystem_migration_reader_fence.rs create mode 100644 src/adapters/store_migration/filesystem_migration_storage.rs create mode 100644 src/adapters/store_migration/filesystem_migration_storage_tests.rs create mode 100644 src/adapters/store_migration/filesystem_migration_test_fixture.rs diff --git a/README.md b/README.md index 499bc51..a686c17 100644 --- a/README.md +++ b/README.md @@ -124,8 +124,14 @@ preflight proof; and the exact 17-phase publication vocabulary with a blocking storage capability port are implemented. Private-field proofs retain every receipt coordinate. Ordered storage-port orchestration revalidates current authority, executes all 17 durability phases, and returns a consequential -complete-coordinate receipt. Filesystem migration, retention execution, -recovery, compaction, and garbage collection remain planned. +complete-coordinate receipt. Writer-locked filesystem authority now implements +the 21-phase fresh migration storage protocol: it exclusively publishes all +three fixed records, admits and synchronizes the exact version-2 namespace, +reopens every canonical view, and retains byte-and-inode evidence through final +verification without changing version-1 immutable bytes. Partial-prefix +migration recovery, filesystem retention execution, immutable reader +snapshots, compaction, and garbage collection remain planned; version 2 is not +yet an admitted restart-safe production store. Presence in the reference CAS does not claim durable retention or crash recovery. diff --git a/docs/formats/segment-store-v2/README.md b/docs/formats/segment-store-v2/README.md index 150022b..7d76e94 100644 --- a/docs/formats/segment-store-v2/README.md +++ b/docs/formats/segment-store-v2/README.md @@ -81,8 +81,10 @@ vocabulary with a blocking storage capability port are available. Storage-independent preparation derives exact canonical manifest and head successors from coherent preflight and current-manifest evidence. Ordered storage-port orchestration revalidates authority and returns a complete receipt. -Production filesystem retention publication, recovery, migration, and garbage +Fresh writer-locked filesystem migration execution now publishes all canonical +fixed records and the exact empty version-2 namespace without changing +version-1 immutable bytes. Partial-prefix restart recovery, production +filesystem retention publication, immutable reader snapshots, and garbage collection do not exist yet. Requirements still in progress in issue #19 or -issue #21 are not complete evidence. A store must refuse version-2 state until -the relevant +issue #21 are not complete evidence. A store must refuse version-2 state until the relevant corruption, model-based, crash-injection, recovery, and fuzz evidence exists. diff --git a/docs/formats/segment-store-v2/migration-crash.md b/docs/formats/segment-store-v2/migration-crash.md index b030fd8..d81d565 100644 --- a/docs/formats/segment-store-v2/migration-crash.md +++ b/docs/formats/segment-store-v2/migration-crash.md @@ -104,4 +104,7 @@ prefix, marker, receipt, and cleanup state without depending on a clock, filesystem iteration order, or file existence alone. `StoreMigrationPhase::ALL` freezes the 21 boundaries above in exact order. -Storage execution and process-death evidence remain unimplemented. +Fresh writer-locked filesystem execution now implements that exact order and +has deterministic in-process storage-fault and corruption laws. The +before/during/after process-death matrix and restart classifier remain +unimplemented; this page does not yet claim crash recovery. diff --git a/docs/formats/segment-store-v2/recovery.md b/docs/formats/segment-store-v2/recovery.md index 9e56d4a..3a37fa5 100644 --- a/docs/formats/segment-store-v2/recovery.md +++ b/docs/formats/segment-store-v2/recovery.md @@ -59,7 +59,9 @@ corpus `definition.tsv` bytes. The format-marker digest is BLAKE3-256 of `CanonicalStoreFormatMarker` produces the registered marker; `AdmittedStoreFormatMarker` admits its framing, checksum, definition, and namespace bound. `CanonicalStoreMigrationIntent` retains typed intent coordinates; `CanonicalStoreMigrationReceipt` binds completion; admitted record types verify both. `StoreMigrationStorage` names all 21 durability capabilities; `execute_store_migration` verifies current authority first and returns only after final synchronization. -`FilesystemStoreMigrationInventoryReader` inventories every version-1 immutable artifact under retained writer authority and pinned pool capabilities; migration-session integration and partial-prefix recovery remain unimplemented. +`FilesystemStoreMigrationInventoryReader` inventories immutable version-1 bytes +under retained writer authority and pinned pools. The fresh filesystem writer +executes once; partial-prefix restart admission and recovery remain absent. ## Reader fence @@ -168,9 +170,10 @@ absence of `retention/HEAD` is the canonical empty retention state only while all retention stages and pools are empty. Any retention artifact routes through recovery instead. Direct version-2 initialization is undefined. -The byte-exact offset tables and golden fixtures are requirements -`KEEP-MIGRATION-002` and `KEEP-MIGRATION-007`; no production writer exists -until those planned items become implemented evidence. +The exact offsets and fixtures are requirement `KEEP-MIGRATION-002`. The fresh +writer emits only those canonical records; success is not restart evidence. +Version 2 remains unavailable as production until partial-prefix recovery and +`KEEP-MIGRATION-007` process-death evidence exist. ## One-way migration protocol @@ -198,7 +201,10 @@ provides no automatic downgrade. `FilesystemStoreMigrationAuthority` retains the writer lock and pinned root and pools. It admits the version-1 namespace, Linux root identity, `HEAD`, complete immutable-pool inventory, and selected catalog. Before mutation, it -requires the same canonical intent. +requires the same canonical intent. Its port retains fixed-record handles, +verifies bytes and inode identity at each publication boundary, and admits only +ordered prefixes. It reopens the complete view before receipt staging and +leaves all version-1 immutable bytes untouched. Version-1 admission refuses after a migration stage, `migration.intent`, `reader.lock`, `FORMAT`, or version-2 directory exists. After durable intent, only version-2 migration recovery may continue. @@ -289,12 +295,6 @@ Each point requires before, during, and after process-death evidence. Restart must establish exact catalog visibility, retention head, namespace generation, orphan classification, stage disposition, and recovery report. -## GC and recovery-disposition recovery - -The [GC and disposition record specification](gc.md) owns the exact -`GcRetirementIntent`, `GcRetirementReceipt`, and -`RecoveryDispositionReceipt` grammars and state transitions. Issue #21 owns -their executable parser, corruption, crash, recovery, and fuzz evidence. -Issue #19 admits only the absent `gc/intent`, `gc/receipt`, -`recovery/disposition.next`, and disposition-receipt pool. Any presence is -unsupported mandatory state and refuses without mutation. +GC and recovery-disposition grammar and transitions are owned by the +[GC specification](gc.md). Until issue #21 implements them, any such artifact +is unsupported mandatory state and refuses without mutation. diff --git a/docs/formats/segment-store-v2/requirements.md b/docs/formats/segment-store-v2/requirements.md index 960656f..cdb7009 100644 --- a/docs/formats/segment-store-v2/requirements.md +++ b/docs/formats/segment-store-v2/requirements.md @@ -30,11 +30,11 @@ case is not evidence. | --- | --- | --- | --- | | `KEEP-MIGRATION-001` | Exact version-1 stores remain admitted until a durable migration artifact exists | compatibility fixtures | Planned in #19 | | `KEEP-MIGRATION-002` | Format marker, intent, and receipt have complete fixed byte tables, named domains, bounds, checksums, deterministic store identity, and exact initial-state digests | exact admission in `tests/store_format_marker.rs`, `tests/store_migration_intent.rs`, and `tests/store_migration_receipt.rs`; canonical construction in `tests/store_migration_intent_encoding.rs` and `tests/store_migration_receipt_encoding.rs`; seeded `migration_format` fuzz target | Implemented | -| `KEEP-MIGRATION-003` | Migration revalidates version-1 head, catalog, pools, root identity, and writer authority before mutation | bounded canonical pool inventory in `tests/store_migration_inventory.rs`; writer-locked filesystem pool admission in `filesystem_inventory_*_tests`; exact authority observation and drift refusal in `filesystem_migration_authority_tests`; verification-first execution in `tests/store_migration_execution.rs`; filesystem storage integration remains | In progress in #19 | +| `KEEP-MIGRATION-003` | Migration revalidates version-1 head, catalog, pools, root identity, and writer authority before mutation | bounded canonical pool inventory in `tests/store_migration_inventory.rs`; writer-locked filesystem pool admission in `filesystem_inventory_*_tests`; exact authority observation and drift refusal in `filesystem_migration_authority_tests`; verification-first execution in `tests/store_migration_execution.rs`; fresh filesystem integration and post-publication drift refusal in `filesystem_migration_storage_tests` | Implemented | | `KEEP-MIGRATION-004` | Every partial migration prefix continues idempotently under writer authority | state-machine and recovery tests | Planned in #19 | -| `KEEP-MIGRATION-005` | Unknown, out-of-order, substituted, corrupt, conflicting, or changed evidence is unrecoverable ambiguity | corruption and mutation matrix | Planned in #19 | -| `KEEP-MIGRATION-006` | Migration never rewrites or deletes admitted version-1 immutable bytes | byte-for-byte before/after witness | Planned in #19 | -| `KEEP-MIGRATION-007` | Process death around every intent stage, canonical link, namespace prefix, marker stage, receipt stage, cleanup, and synchronization boundary reaches a documented lawful state | ordered phases and capabilities in `tests/store_migration_phase.rs` and `tests/store_migration_storage.rs`; exact phase-failure execution in `tests/store_migration_execution.rs`; `KEEP-CRASH-053..=073` process-death matrix remains | In progress in #19 | +| `KEEP-MIGRATION-005` | Unknown, out-of-order, substituted, corrupt, conflicting, or changed evidence is unrecoverable ambiguity | forward-execution stage preservation, byte-equal inode-substitution, out-of-order-prefix, and post-publication drift laws in `filesystem_migration_storage_tests`; restart corruption and mutation matrix remains | In progress in #19 | +| `KEEP-MIGRATION-006` | Migration never rewrites or deletes admitted version-1 immutable bytes | exact segment, catalog, and head before/after witness in `filesystem_migration_storage_tests`; restart-path evidence remains | In progress in #19 | +| `KEEP-MIGRATION-007` | Process death around every intent stage, canonical link, namespace prefix, marker stage, receipt stage, cleanup, and synchronization boundary reaches a documented lawful state | ordered phases and capabilities in `tests/store_migration_phase.rs` and `tests/store_migration_storage.rs`; exact phase-failure execution in `tests/store_migration_execution.rs`; production 21-phase forward execution in `filesystem_migration_storage_tests`; `KEEP-CRASH-053..=073` process-death matrix remains | In progress in #19 | | `KEEP-MIGRATION-008` | Version-1 admission refuses every version-2 or partial-migration artifact after migration begins | `FORMAT` refusal before mutation in `filesystem_migration_authority_tests`; remaining compatibility and fuzz matrix | In progress in #19 | @@ -57,7 +57,8 @@ case is not evidence. - Migration is one-way and provides no downgrade. - Retention evidence proves a bounded physical reconstruction claim, not application meaning, causal ownership, future policy, or secure erasure. -- A version-2 format specification is not proof that a version-2 production - writer exists. +- A fresh forward writer is not proof that version 2 is restart-safe or + production-admitted; partial-prefix recovery and crash evidence remain + mandatory. - Benchmarks are required before performance-sensitive retention or migration optimization. diff --git a/src/adapters/store_migration.rs b/src/adapters/store_migration.rs index 0083394..2df8d0b 100644 --- a/src/adapters/store_migration.rs +++ b/src/adapters/store_migration.rs @@ -40,6 +40,15 @@ mod filesystem_migration_authority_error_display; #[cfg(test)] mod filesystem_migration_authority_tests; mod filesystem_migration_authority_validation; +mod filesystem_migration_fixed_artifact; +mod filesystem_migration_namespace; +mod filesystem_migration_namespace_directory; +mod filesystem_migration_reader_fence; +mod filesystem_migration_storage; +#[cfg(test)] +mod filesystem_migration_storage_tests; +#[cfg(test)] +mod filesystem_migration_test_fixture; mod format_definition_digest; mod format_marker_decode_error; mod format_marker_decode_error_display; diff --git a/src/adapters/store_migration/filesystem_migration_authority.rs b/src/adapters/store_migration/filesystem_migration_authority.rs index e263f18..4ddb05a 100644 --- a/src/adapters/store_migration/filesystem_migration_authority.rs +++ b/src/adapters/store_migration/filesystem_migration_authority.rs @@ -8,6 +8,7 @@ use super::filesystem_migration_authority_error::{ use super::filesystem_migration_authority_validation::{ artifact_error, require_root, verify_catalog, }; +use super::filesystem_migration_fixed_artifact::FilesystemMigrationFixedStage; use super::migration_catalog_coordinates::MigrationCatalogCoordinates; use super::store_root_identity::StoreRootIdentities; use super::{CanonicalStoreMigrationIntent, FilesystemStoreMigrationInventoryReader}; @@ -23,12 +24,21 @@ const HEAD_LENGTH: u64 = 128; /// Exclusive authority to observe and migrate one pinned version-1 filesystem root. /// /// The authority retains the admitted writer lock and pinned root and immutable -/// pool capabilities for its entire lifetime. Its synchronous, -/// capability-relative filesystem I/O performs no protocol mutation and uses -/// neither a network nor an asynchronous runtime. +/// pool capabilities for its entire lifetime. Observation performs no protocol +/// mutation. When passed to [`crate::execute_store_migration`], its +/// [`crate::StoreMigrationStorage`] implementation executes only the fresh +/// forward protocol from an exactly admitted version-1 root. It retains opened +/// fixed-record handles through final verification, performs synchronous +/// capability-relative I/O, and uses neither a network nor an asynchronous +/// runtime. Reopening a partial migration prefix remains a separate recovery +/// boundary. #[must_use] pub struct FilesystemStoreMigrationAuthority { inventory: FilesystemStoreMigrationInventoryReader, + pub(super) fixed_stage: Option, + pub(super) published_intent: Option, + pub(super) published_marker: Option, + pub(super) published_receipt: Option, } impl FilesystemStoreMigrationAuthority { @@ -48,7 +58,13 @@ impl FilesystemStoreMigrationAuthority { ) -> Result { let inventory = FilesystemStoreMigrationInventoryReader::open(admission, policy) .map_err(|source| Error::Inventory { source })?; - Ok(Self { inventory }) + Ok(Self { + inventory, + fixed_stage: None, + published_intent: None, + published_marker: None, + published_receipt: None, + }) } /// Observes one canonical intent from exact current version-1 authority. @@ -157,4 +173,8 @@ impl FilesystemStoreMigrationAuthority { })?; verify_catalog(head, catalog) } + + pub(super) const fn root(&self) -> &cap_std::fs::Dir { + self.inventory.root() + } } diff --git a/src/adapters/store_migration/filesystem_migration_authority_tests.rs b/src/adapters/store_migration/filesystem_migration_authority_tests.rs index 5afbc84..9750f86 100644 --- a/src/adapters/store_migration/filesystem_migration_authority_tests.rs +++ b/src/adapters/store_migration/filesystem_migration_authority_tests.rs @@ -4,21 +4,13 @@ use std::error::Error; use std::fs; use super::FilesystemMigrationAuthorityError; -use super::filesystem_migration_authority::FilesystemStoreMigrationAuthority; -use crate::adapters::filesystem_test_sandbox::TestDirectory; use crate::adapters::test_support::decode_hex; -use crate::adapters::{AdmittedSegment, FilesystemPlatformAdmission, physical_pool_name}; +use crate::adapters::{AdmittedSegment, physical_pool_name}; + +use super::filesystem_migration_test_fixture::{maximum_policy, open_authority}; -const SEGMENT_HEX: &str = - include_str!("../../../conformance/segment-store/v1/one-zero-segment.hex"); -const CATALOG_HEX: &str = - include_str!("../../../conformance/segment-store/v1/one-zero-catalog.hex"); -const HEAD_HEX: &str = include_str!("../../../conformance/segment-store/v1/one-zero-head.hex"); const EMPTY_SEGMENT_HEX: &str = include_str!("../../../conformance/segment-store/v1/empty-segment.hex"); -const CATALOG_NAME: &str = - "0000000000000001-04b82519b0399baefd0b9c0f32a871052e4c47e3a00226ab03b21661470f7320.cat"; -const SEGMENT_NAME: &str = "b7542dced2ab770894a14d1d04b066e3a899942602c5986d35ba6df6c1a35cfc.seg"; const INVENTORY_DIGEST: &str = "40bf5d49c34847ac9cf46a256f343cee80cd980d1405d2dd02ceff8f58d674f9"; #[test] @@ -85,25 +77,3 @@ fn version_two_namespace_evidence_refuses_before_mutation() -> Result<(), Box Result<(TestDirectory, FilesystemStoreMigrationAuthority), Box> { - let sandbox = TestDirectory::create(name)?; - let admission = FilesystemPlatformAdmission::initialize_unchecked_for_tests(sandbox.path())?; - fs::write( - sandbox.path().join("segments").join(SEGMENT_NAME), - decode_hex(SEGMENT_HEX.trim())?, - )?; - fs::write( - sandbox.path().join("catalogs").join(CATALOG_NAME), - decode_hex(CATALOG_HEX.trim())?, - )?; - fs::write(sandbox.path().join("HEAD"), decode_hex(HEAD_HEX.trim())?)?; - let authority = FilesystemStoreMigrationAuthority::open(admission, maximum_policy())?; - Ok((sandbox, authority)) -} - -const fn maximum_policy() -> crate::adapters::SegmentReadPolicy { - super::filesystem_inventory_catalogs_test_fixture::maximum_policy() -} diff --git a/src/adapters/store_migration/filesystem_migration_fixed_artifact.rs b/src/adapters/store_migration/filesystem_migration_fixed_artifact.rs new file mode 100644 index 0000000..06705fc --- /dev/null +++ b/src/adapters/store_migration/filesystem_migration_fixed_artifact.rs @@ -0,0 +1,242 @@ +//! This module owns exact fixed-record migration publication. + +use std::io::{self, Read, Write}; + +use cap_fs_ext::{FollowSymlinks, MetadataExt, OpenOptionsFollowExt, OpenOptionsSyncExt}; +use cap_std::fs::{Dir, File, Metadata, OpenOptions}; + +use super::{format_marker_decoder, migration_intent_format, migration_receipt_format}; +use crate::adapters::filesystem_catalog_artifact; + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(super) enum FilesystemMigrationFixedArtifact { + Intent, + Marker, + Receipt, +} + +impl FilesystemMigrationFixedArtifact { + const fn stage_name(self) -> &'static str { + match self { + Self::Intent => "migration.intent.next", + Self::Marker => "FORMAT.next", + Self::Receipt => "migration.receipt.next", + } + } + + const fn canonical_name(self) -> &'static str { + match self { + Self::Intent => "migration.intent", + Self::Marker => "FORMAT", + Self::Receipt => "migration.receipt", + } + } + + const fn encoded_length(self) -> usize { + match self { + Self::Intent => migration_intent_format::ENCODED_LENGTH, + Self::Marker => format_marker_decoder::ENCODED_LENGTH, + Self::Receipt => migration_receipt_format::ENCODED_LENGTH, + } + } +} + +pub(super) struct FilesystemMigrationFixedStage { + artifact: FilesystemMigrationFixedArtifact, + expected: Box<[u8]>, + identity: FixedFileIdentity, + file: File, +} + +impl FilesystemMigrationFixedStage { + pub(super) fn create( + root: &Dir, + artifact: FilesystemMigrationFixedArtifact, + expected: &[u8], + ) -> io::Result { + require_length(artifact, expected)?; + let mut file = filesystem_catalog_artifact::create_exclusive(root, artifact.stage_name())?; + let identity = FixedFileIdentity::read_file(&file)?; + file.write_all(expected)?; + file.flush()?; + Ok(Self { + artifact, + expected: Box::from(expected), + identity, + file, + }) + } + + pub(super) fn synchronize(&self, root: &Dir) -> io::Result<()> { + self.require_handle()?; + self.file.sync_all()?; + self.verify_stage(root) + } + + pub(super) fn link( + &self, + root: &Dir, + artifact: FilesystemMigrationFixedArtifact, + expected: &[u8], + ) -> io::Result<()> { + self.require_record(artifact, expected)?; + self.verify_stage(root)?; + match root.hard_link( + self.artifact.stage_name(), + root, + self.artifact.canonical_name(), + ) { + Ok(()) => {} + Err(source) if source.kind() == io::ErrorKind::AlreadyExists => {} + Err(source) => return Err(source), + } + self.verify_linked_names(root) + } + + pub(super) fn remove(self, root: &Dir) -> io::Result { + self.verify_linked_names(root)?; + root.remove_file(self.artifact.stage_name())?; + require_absent(root, self.artifact.stage_name())?; + self.verify_canonical(root)?; + Ok(self) + } + + pub(super) fn verify_linked(&self, root: &Dir) -> io::Result<()> { + self.verify_linked_names(root) + } + + pub(super) fn verify_canonical(&self, root: &Dir) -> io::Result<()> { + self.require_handle()?; + verify_name( + root, + self.artifact.canonical_name(), + &self.expected, + self.identity, + ) + } + + pub(super) const fn artifact(&self) -> FilesystemMigrationFixedArtifact { + self.artifact + } + + fn require_handle(&self) -> io::Result<()> { + let observed = FixedFileIdentity::read_file(&self.file)?; + if observed == self.identity { + Ok(()) + } else { + Err(invalid_data("migration stage handle changed identity")) + } + } + + fn require_record( + &self, + artifact: FilesystemMigrationFixedArtifact, + expected: &[u8], + ) -> io::Result<()> { + if self.artifact == artifact && self.expected.as_ref() == expected { + Ok(()) + } else { + Err(invalid_data("migration stage record disagreed")) + } + } + + fn verify_stage(&self, root: &Dir) -> io::Result<()> { + verify_name( + root, + self.artifact.stage_name(), + &self.expected, + self.identity, + ) + } + + fn verify_linked_names(&self, root: &Dir) -> io::Result<()> { + self.verify_stage(root)?; + verify_name( + root, + self.artifact.canonical_name(), + &self.expected, + self.identity, + ) + } +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +struct FixedFileIdentity { + device: u64, + inode: u64, +} + +impl FixedFileIdentity { + fn read_file(file: &File) -> io::Result { + file.metadata().map(|metadata| Self::from(&metadata)) + } +} + +impl From<&Metadata> for FixedFileIdentity { + fn from(metadata: &Metadata) -> Self { + Self { + device: metadata.dev(), + inode: metadata.ino(), + } + } +} + +fn verify_name( + root: &Dir, + name: &str, + expected: &[u8], + identity: FixedFileIdentity, +) -> io::Result<()> { + let mut options = OpenOptions::new(); + options.read(true).follow(FollowSymlinks::No).nonblock(true); + let mut file = root.open_with(name, &options)?; + require_metadata(&file.metadata()?, expected.len(), identity)?; + require_metadata(&root.symlink_metadata(name)?, expected.len(), identity)?; + let mut observed = vec![0_u8; expected.len()]; + file.read_exact(&mut observed)?; + let mut trailing = [0_u8; 1]; + if observed != expected || file.read(&mut trailing)? != 0 { + return Err(invalid_data("migration fixed-record bytes disagreed")); + } + require_metadata(&file.metadata()?, expected.len(), identity)?; + require_metadata(&root.symlink_metadata(name)?, expected.len(), identity) +} + +fn require_metadata( + metadata: &Metadata, + expected_length: usize, + expected_identity: FixedFileIdentity, +) -> io::Result<()> { + let expected_length = u64::try_from(expected_length) + .map_err(|_source| invalid_data("migration fixed-record length exceeded u64"))?; + if metadata.is_file() + && metadata.len() == expected_length + && FixedFileIdentity::from(metadata) == expected_identity + { + Ok(()) + } else { + Err(invalid_data( + "migration fixed-record kind, length, or identity disagreed", + )) + } +} + +fn require_length(artifact: FilesystemMigrationFixedArtifact, expected: &[u8]) -> io::Result<()> { + if expected.len() == artifact.encoded_length() { + Ok(()) + } else { + Err(invalid_data("migration fixed-record length disagreed")) + } +} + +fn require_absent(root: &Dir, name: &str) -> io::Result<()> { + match root.symlink_metadata(name) { + Err(source) if source.kind() == io::ErrorKind::NotFound => Ok(()), + Ok(_) => Err(invalid_data("removed migration stage remained visible")), + Err(source) => Err(source), + } +} + +fn invalid_data(message: &'static str) -> io::Error { + io::Error::new(io::ErrorKind::InvalidData, message) +} diff --git a/src/adapters/store_migration/filesystem_migration_namespace.rs b/src/adapters/store_migration/filesystem_migration_namespace.rs new file mode 100644 index 0000000..04b9279 --- /dev/null +++ b/src/adapters/store_migration/filesystem_migration_namespace.rs @@ -0,0 +1,228 @@ +//! This module owns ordered version-2 migration namespace admission. + +use std::io; + +use cap_std::fs::Dir; + +use super::filesystem_migration_namespace_directory::{ + PinnedMigrationDirectory, ambiguous, exact_membership, optional_directory, + require_allowed_membership, require_directory, require_empty, require_exact_membership, + require_regular, required_directory, +}; +use super::filesystem_migration_reader_fence; + +const WRITER_LOCK: &str = "writer.lock"; +const STAGING: &str = "staging"; +const SEGMENTS: &str = "segments"; +const CATALOGS: &str = "catalogs"; +const HEAD: &str = "HEAD"; +const INTENT: &str = "migration.intent"; +const READER_LOCK: &str = "reader.lock"; +const RETENTION: &str = "retention"; +const ROOTS: &str = "roots"; +const MANIFESTS: &str = "manifests"; +const GC: &str = "gc"; +const RECOVERY: &str = "recovery"; +const DISPOSITIONS: &str = "dispositions"; + +const BEFORE_READER: [&str; 6] = [WRITER_LOCK, STAGING, SEGMENTS, CATALOGS, HEAD, INTENT]; +const AFTER_READER: [&str; 7] = [ + WRITER_LOCK, + STAGING, + SEGMENTS, + CATALOGS, + HEAD, + INTENT, + READER_LOCK, +]; +const PREFIX_ROOT: [&str; 10] = [ + WRITER_LOCK, + STAGING, + SEGMENTS, + CATALOGS, + HEAD, + INTENT, + READER_LOCK, + RETENTION, + GC, + RECOVERY, +]; +const MARKER_ROOT: [&str; 11] = [ + WRITER_LOCK, + STAGING, + SEGMENTS, + CATALOGS, + HEAD, + INTENT, + READER_LOCK, + RETENTION, + GC, + RECOVERY, + "FORMAT", +]; +const RECEIPT_ROOT: [&str; 12] = [ + WRITER_LOCK, + STAGING, + SEGMENTS, + CATALOGS, + HEAD, + INTENT, + READER_LOCK, + RETENTION, + GC, + RECOVERY, + "FORMAT", + "migration.receipt", +]; + +pub(super) fn admit_reader_fence(root: &Dir) -> io::Result<()> { + let before = exact_membership(root, &BEFORE_READER)?; + let after = exact_membership(root, &AFTER_READER)?; + if !before && !after { + return Err(ambiguous("reader-fence predecessor namespace disagreed")); + } + let file = if before { + filesystem_migration_reader_fence::create(root)? + } else { + filesystem_migration_reader_fence::open(root)? + }; + filesystem_migration_reader_fence::verify(root, &file)?; + file.sync_all()?; + filesystem_migration_reader_fence::verify(root, &file)?; + verify_reader_root(root) +} + +pub(super) fn admit_namespace_prefix(root: &Dir) -> io::Result<()> { + preflight_prefix(root)?; + let retention = PinnedMigrationDirectory::admit(root, RETENTION)?; + let roots = PinnedMigrationDirectory::admit(retention.directory(), ROOTS)?; + let manifests = PinnedMigrationDirectory::admit(retention.directory(), MANIFESTS)?; + let gc = PinnedMigrationDirectory::admit(root, GC)?; + let recovery = PinnedMigrationDirectory::admit(root, RECOVERY)?; + let dispositions = PinnedMigrationDirectory::admit(recovery.directory(), DISPOSITIONS)?; + roots.verify(retention.directory())?; + manifests.verify(retention.directory())?; + dispositions.verify(recovery.directory())?; + retention.verify(root)?; + gc.verify(root)?; + recovery.verify(root)?; + verify_namespace_prefix(root) +} + +pub(super) fn verify_intent_root(root: &Dir) -> io::Result<()> { + require_v1_and_intent(root)?; + require_exact_membership(root, &BEFORE_READER) +} + +pub(super) fn verify_reader_root(root: &Dir) -> io::Result<()> { + require_v1_and_intent(root)?; + let reader = filesystem_migration_reader_fence::open(root)?; + filesystem_migration_reader_fence::verify(root, &reader)?; + require_exact_membership(root, &AFTER_READER) +} + +pub(super) fn verify_namespace_prefix(root: &Dir) -> io::Result<()> { + verify_prefix_directories(root)?; + require_exact_membership(root, &PREFIX_ROOT) +} + +pub(super) fn verify_namespace_contents(root: &Dir) -> io::Result<()> { + verify_prefix_directories(root) +} + +pub(super) fn verify_marker_view(root: &Dir) -> io::Result<()> { + verify_marker_contents(root)?; + require_exact_membership(root, &MARKER_ROOT) +} + +pub(super) fn verify_marker_contents(root: &Dir) -> io::Result<()> { + verify_prefix_directories(root)?; + require_regular(root, "FORMAT", Some(96)) +} + +pub(super) fn verify_receipt_view(root: &Dir) -> io::Result<()> { + verify_prefix_directories(root)?; + require_regular(root, "FORMAT", Some(96))?; + require_regular(root, "migration.receipt", Some(256))?; + require_exact_membership(root, &RECEIPT_ROOT) +} + +fn preflight_prefix(root: &Dir) -> io::Result<()> { + require_allowed_membership(root, &PREFIX_ROOT)?; + require_base_namespace(root)?; + let retention = optional_directory(root, RETENTION)?; + let gc = optional_directory(root, GC)?; + let recovery = optional_directory(root, RECOVERY)?; + let retention_complete = preflight_retention(retention.as_ref())?; + if gc.is_some() && !retention_complete { + return Err(ambiguous("gc appeared before the retention prefix")); + } + if recovery.is_some() && gc.is_none() { + return Err(ambiguous("recovery appeared before the gc prefix")); + } + if let Some(directory) = gc.as_ref() { + require_empty(directory.directory())?; + } + preflight_recovery(recovery.as_ref()) +} + +fn require_base_namespace(root: &Dir) -> io::Result<()> { + require_v1_and_intent(root)?; + let reader = filesystem_migration_reader_fence::open(root)?; + filesystem_migration_reader_fence::verify(root, &reader) +} + +fn require_v1_and_intent(root: &Dir) -> io::Result<()> { + require_regular(root, WRITER_LOCK, None)?; + require_directory(root, STAGING)?; + require_directory(root, SEGMENTS)?; + require_directory(root, CATALOGS)?; + require_regular(root, HEAD, Some(128))?; + require_regular(root, INTENT, Some(256)) +} + +fn verify_prefix_directories(root: &Dir) -> io::Result<()> { + require_base_namespace(root)?; + let retention = required_directory(root, RETENTION)?; + let roots = required_directory(retention.directory(), ROOTS)?; + let manifests = required_directory(retention.directory(), MANIFESTS)?; + let gc = required_directory(root, GC)?; + let recovery = required_directory(root, RECOVERY)?; + let dispositions = required_directory(recovery.directory(), DISPOSITIONS)?; + require_empty(roots.directory())?; + require_empty(manifests.directory())?; + require_empty(gc.directory())?; + require_empty(dispositions.directory())?; + require_exact_membership(retention.directory(), &[ROOTS, MANIFESTS])?; + require_exact_membership(recovery.directory(), &[DISPOSITIONS]) +} + +fn preflight_retention(retention: Option<&PinnedMigrationDirectory>) -> io::Result { + let Some(retention) = retention else { + return Ok(false); + }; + require_allowed_membership(retention.directory(), &[ROOTS, MANIFESTS])?; + let roots = optional_directory(retention.directory(), ROOTS)?; + let manifests = optional_directory(retention.directory(), MANIFESTS)?; + if manifests.is_some() && roots.is_none() { + return Err(ambiguous("retention manifests appeared before roots")); + } + if let Some(directory) = roots.as_ref() { + require_empty(directory.directory())?; + } + if let Some(directory) = manifests.as_ref() { + require_empty(directory.directory())?; + } + Ok(roots.is_some() && manifests.is_some()) +} + +fn preflight_recovery(recovery: Option<&PinnedMigrationDirectory>) -> io::Result<()> { + let Some(recovery) = recovery else { + return Ok(()); + }; + require_allowed_membership(recovery.directory(), &[DISPOSITIONS])?; + if let Some(dispositions) = optional_directory(recovery.directory(), DISPOSITIONS)? { + require_empty(dispositions.directory())?; + } + Ok(()) +} diff --git a/src/adapters/store_migration/filesystem_migration_namespace_directory.rs b/src/adapters/store_migration/filesystem_migration_namespace_directory.rs new file mode 100644 index 0000000..adb46d2 --- /dev/null +++ b/src/adapters/store_migration/filesystem_migration_namespace_directory.rs @@ -0,0 +1,188 @@ +//! This module owns pinned migration namespace-directory admission. + +use std::ffi::OsStr; +use std::io; + +use cap_fs_ext::MetadataExt; +use cap_std::fs::{Dir, Metadata}; + +use crate::adapters::{ + filesystem_catalog_artifact, filesystem_platform_profile, sync_capable_directory, +}; + +pub(super) struct PinnedMigrationDirectory { + name: &'static str, + identity: DirectoryIdentity, + directory: Dir, +} + +impl PinnedMigrationDirectory { + pub(super) fn admit(parent: &Dir, name: &'static str) -> io::Result { + let created = match parent.create_dir(name) { + Ok(()) => true, + Err(source) if source.kind() == io::ErrorKind::AlreadyExists => false, + Err(source) => return Err(source), + }; + let directory = sync_capable_directory::open(parent, name)?; + require_same_filesystem(parent, &directory)?; + let identity = DirectoryIdentity::from(&directory.dir_metadata()?); + let pinned = Self { + name, + identity, + directory, + }; + pinned.verify(parent)?; + if created { + filesystem_catalog_artifact::synchronize_directory(parent)?; + } + Ok(pinned) + } + + pub(super) fn verify(&self, parent: &Dir) -> io::Result<()> { + let handle = DirectoryIdentity::from(&self.directory.dir_metadata()?); + let current = sync_capable_directory::open(parent, self.name)?; + require_same_filesystem(parent, ¤t)?; + let current = DirectoryIdentity::from(¤t.dir_metadata()?); + let metadata = parent.symlink_metadata(self.name)?; + if metadata.is_dir() + && handle == self.identity + && current == self.identity + && DirectoryIdentity::from(&metadata) == handle + { + Ok(()) + } else { + Err(ambiguous("migration directory changed identity")) + } + } + + pub(super) const fn directory(&self) -> &Dir { + &self.directory + } +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +struct DirectoryIdentity { + device: u64, + inode: u64, +} + +impl From<&Metadata> for DirectoryIdentity { + fn from(metadata: &Metadata) -> Self { + Self { + device: metadata.dev(), + inode: metadata.ino(), + } + } +} + +pub(super) fn optional_directory( + parent: &Dir, + name: &'static str, +) -> io::Result> { + match parent.symlink_metadata(name) { + Err(source) if source.kind() == io::ErrorKind::NotFound => Ok(None), + Err(source) => Err(source), + Ok(metadata) if metadata.is_dir() => { + let directory = sync_capable_directory::open(parent, name)?; + require_same_filesystem(parent, &directory)?; + let pinned = PinnedMigrationDirectory { + name, + identity: DirectoryIdentity::from(&metadata), + directory, + }; + pinned.verify(parent)?; + Ok(Some(pinned)) + } + Ok(_) => Err(ambiguous("migration namespace entry has the wrong kind")), + } +} + +fn require_same_filesystem(parent: &Dir, child: &Dir) -> io::Result<()> { + let parent = filesystem_platform_profile::root_identity(parent)?; + let child = filesystem_platform_profile::root_identity(child)?; + if parent.device() == child.device() && parent.mount() == child.mount() { + Ok(()) + } else { + Err(ambiguous( + "migration namespace crossed the admitted filesystem or mount", + )) + } +} + +pub(super) fn required_directory( + parent: &Dir, + name: &'static str, +) -> io::Result { + optional_directory(parent, name)? + .ok_or_else(|| ambiguous("required migration namespace directory was absent")) +} + +pub(super) fn require_directory(parent: &Dir, name: &str) -> io::Result<()> { + if parent.symlink_metadata(name)?.is_dir() { + Ok(()) + } else { + Err(ambiguous("required migration directory has the wrong kind")) + } +} + +pub(super) fn require_regular(parent: &Dir, name: &str, length: Option) -> io::Result<()> { + let metadata = parent.symlink_metadata(name)?; + if metadata.is_file() && length.is_none_or(|expected| metadata.len() == expected) { + Ok(()) + } else { + Err(ambiguous( + "required migration file has the wrong kind or length", + )) + } +} + +pub(super) fn require_empty(directory: &Dir) -> io::Result<()> { + let mut entries = directory.entries()?; + if entries.next().transpose()?.is_none() { + Ok(()) + } else { + Err(ambiguous("new migration namespace was not empty")) + } +} + +pub(super) fn require_exact_membership(directory: &Dir, expected: &[&str]) -> io::Result<()> { + if exact_membership(directory, expected)? { + Ok(()) + } else { + Err(ambiguous("migration namespace membership disagreed")) + } +} + +pub(super) fn exact_membership(directory: &Dir, expected: &[&str]) -> io::Result { + let mut observed = Vec::with_capacity(expected.len()); + for entry in directory.entries()? { + if observed.len() == expected.len() { + return Ok(false); + } + observed.push(entry?.file_name()); + } + Ok(observed.len() == expected.len() + && observed.iter().all(|name| { + expected + .iter() + .any(|candidate| name == OsStr::new(candidate)) + })) +} + +pub(super) fn require_allowed_membership(directory: &Dir, allowed: &[&str]) -> io::Result<()> { + let mut observed = 0_usize; + for entry in directory.entries()? { + observed = observed + .checked_add(1) + .ok_or_else(|| ambiguous("migration namespace count overflowed"))?; + let name = entry?.file_name(); + if observed > allowed.len() || !allowed.iter().any(|candidate| name == *candidate) { + return Err(ambiguous("migration namespace contains an unknown entry")); + } + } + Ok(()) +} + +pub(super) fn ambiguous(message: &'static str) -> io::Error { + io::Error::new(io::ErrorKind::InvalidData, message) +} diff --git a/src/adapters/store_migration/filesystem_migration_reader_fence.rs b/src/adapters/store_migration/filesystem_migration_reader_fence.rs new file mode 100644 index 0000000..7465fb1 --- /dev/null +++ b/src/adapters/store_migration/filesystem_migration_reader_fence.rs @@ -0,0 +1,62 @@ +//! This module owns persistent migration reader-fence admission. + +use std::io; + +use cap_fs_ext::{FollowSymlinks, MetadataExt, OpenOptionsFollowExt, OpenOptionsSyncExt}; +use cap_std::fs::{Dir, File, Metadata, OpenOptions}; + +use super::filesystem_migration_namespace_directory::ambiguous; + +const READER_LOCK: &str = "reader.lock"; + +pub(super) fn create(root: &Dir) -> io::Result { + let mut options = options(); + options.create_new(true); + root.open_with(READER_LOCK, &options) +} + +pub(super) fn open(root: &Dir) -> io::Result { + root.open_with(READER_LOCK, &options()) +} + +pub(super) fn verify(root: &Dir, file: &File) -> io::Result<()> { + let handle = file.metadata()?; + let entry = root.symlink_metadata(READER_LOCK)?; + if handle.is_file() + && entry.is_file() + && handle.len() == 0 + && entry.len() == 0 + && FileIdentity::from(&handle) == FileIdentity::from(&entry) + { + Ok(()) + } else { + Err(ambiguous( + "reader fence kind, length, or identity disagreed", + )) + } +} + +fn options() -> OpenOptions { + let mut options = OpenOptions::new(); + options + .read(true) + .write(true) + .follow(FollowSymlinks::No) + .nonblock(true); + options +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +struct FileIdentity { + device: u64, + inode: u64, +} + +impl From<&Metadata> for FileIdentity { + fn from(metadata: &Metadata) -> Self { + Self { + device: metadata.dev(), + inode: metadata.ino(), + } + } +} diff --git a/src/adapters/store_migration/filesystem_migration_storage.rs b/src/adapters/store_migration/filesystem_migration_storage.rs new file mode 100644 index 0000000..b7ff6e4 --- /dev/null +++ b/src/adapters/store_migration/filesystem_migration_storage.rs @@ -0,0 +1,242 @@ +//! This module binds filesystem migration authority to the storage port. + +use std::io; + +use super::filesystem_migration_fixed_artifact::{ + FilesystemMigrationFixedArtifact as FixedArtifact, FilesystemMigrationFixedStage, +}; +use super::{ + CanonicalStoreFormatMarker, CanonicalStoreMigrationIntent, CanonicalStoreMigrationReceipt, + FilesystemStoreMigrationAuthority, StoreMigrationStorage, filesystem_migration_namespace, +}; +use crate::adapters::filesystem_catalog_artifact; + +impl StoreMigrationStorage for FilesystemStoreMigrationAuthority { + fn verify_current(&mut self, intent: &CanonicalStoreMigrationIntent) -> io::Result<()> { + verify_authority(self, intent) + } + + fn write_intent_stage(&mut self, intent: &CanonicalStoreMigrationIntent) -> io::Result<()> { + write_stage(self, FixedArtifact::Intent, intent.encoded()) + } + + fn synchronize_intent_stage(&mut self) -> io::Result<()> { + synchronize_stage(self, FixedArtifact::Intent) + } + + fn link_intent(&mut self, intent: &CanonicalStoreMigrationIntent) -> io::Result<()> { + link_stage(self, FixedArtifact::Intent, intent.encoded()) + } + + fn synchronize_root_after_intent(&mut self) -> io::Result<()> { + synchronize_root(self)?; + active_stage(self, FixedArtifact::Intent)?.verify_linked(self.root()) + } + + fn remove_intent_stage(&mut self) -> io::Result<()> { + remove_stage(self, FixedArtifact::Intent) + } + + fn synchronize_root_after_intent_cleanup(&mut self) -> io::Result<()> { + synchronize_root(self)?; + verify_published(self, FixedArtifact::Intent)?; + filesystem_migration_namespace::verify_intent_root(self.root()) + } + + fn admit_reader_fence(&mut self) -> io::Result<()> { + verify_published(self, FixedArtifact::Intent)?; + filesystem_migration_namespace::admit_reader_fence(self.root())?; + verify_published(self, FixedArtifact::Intent) + } + + fn admit_namespace_prefix(&mut self) -> io::Result<()> { + verify_published(self, FixedArtifact::Intent)?; + filesystem_migration_namespace::admit_namespace_prefix(self.root())?; + verify_published(self, FixedArtifact::Intent) + } + + fn synchronize_root_after_namespace(&mut self) -> io::Result<()> { + synchronize_root(self)?; + verify_published(self, FixedArtifact::Intent)?; + filesystem_migration_namespace::verify_namespace_prefix(self.root()) + } + + fn write_marker_stage(&mut self, marker: &CanonicalStoreFormatMarker) -> io::Result<()> { + verify_namespace_view(self)?; + write_stage(self, FixedArtifact::Marker, marker.encoded()) + } + + fn synchronize_marker_stage(&mut self) -> io::Result<()> { + synchronize_stage(self, FixedArtifact::Marker) + } + + fn link_marker(&mut self, marker: &CanonicalStoreFormatMarker) -> io::Result<()> { + link_stage(self, FixedArtifact::Marker, marker.encoded()) + } + + fn synchronize_root_after_marker(&mut self) -> io::Result<()> { + synchronize_root(self)?; + verify_published(self, FixedArtifact::Intent)?; + filesystem_migration_namespace::verify_namespace_contents(self.root())?; + active_stage(self, FixedArtifact::Marker)?.verify_linked(self.root()) + } + + fn remove_marker_stage(&mut self) -> io::Result<()> { + remove_stage(self, FixedArtifact::Marker) + } + + fn synchronize_root_after_marker_cleanup(&mut self) -> io::Result<()> { + synchronize_root(self)?; + verify_marker_view(self) + } + + fn write_receipt_stage(&mut self, receipt: &CanonicalStoreMigrationReceipt) -> io::Result<()> { + verify_marker_view(self)?; + write_stage(self, FixedArtifact::Receipt, receipt.encoded()) + } + + fn synchronize_receipt_stage(&mut self) -> io::Result<()> { + synchronize_stage(self, FixedArtifact::Receipt) + } + + fn link_receipt(&mut self, receipt: &CanonicalStoreMigrationReceipt) -> io::Result<()> { + link_stage(self, FixedArtifact::Receipt, receipt.encoded()) + } + + fn synchronize_root_after_receipt(&mut self) -> io::Result<()> { + synchronize_root(self)?; + verify_published(self, FixedArtifact::Intent)?; + verify_published(self, FixedArtifact::Marker)?; + filesystem_migration_namespace::verify_marker_contents(self.root())?; + active_stage(self, FixedArtifact::Receipt)?.verify_linked(self.root()) + } + + fn remove_receipt_stage(&mut self) -> io::Result<()> { + remove_stage(self, FixedArtifact::Receipt) + } + + fn synchronize_root_after_receipt_cleanup(&mut self) -> io::Result<()> { + synchronize_root(self)?; + verify_receipt_view(self) + } +} + +fn verify_authority( + authority: &FilesystemStoreMigrationAuthority, + intent: &CanonicalStoreMigrationIntent, +) -> io::Result<()> { + authority.verify_current(intent).map_err(io::Error::other) +} + +fn write_stage( + authority: &mut FilesystemStoreMigrationAuthority, + artifact: FixedArtifact, + expected: &[u8], +) -> io::Result<()> { + if authority.fixed_stage.is_some() { + return Err(stage_state("migration fixed stage was already active")); + } + let stage = FilesystemMigrationFixedStage::create(authority.root(), artifact, expected)?; + authority.fixed_stage = Some(stage); + Ok(()) +} + +fn synchronize_stage( + authority: &FilesystemStoreMigrationAuthority, + artifact: FixedArtifact, +) -> io::Result<()> { + active_stage(authority, artifact)?.synchronize(authority.root()) +} + +fn link_stage( + authority: &FilesystemStoreMigrationAuthority, + artifact: FixedArtifact, + expected: &[u8], +) -> io::Result<()> { + active_stage(authority, artifact)?.link(authority.root(), artifact, expected) +} + +fn remove_stage( + authority: &mut FilesystemStoreMigrationAuthority, + artifact: FixedArtifact, +) -> io::Result<()> { + if published_stage(authority, artifact).is_some() { + return Err(stage_state("migration fixed record was already published")); + } + let stage = authority + .fixed_stage + .take() + .ok_or_else(|| stage_state("migration fixed stage was not active"))?; + if stage.artifact() != artifact { + authority.fixed_stage = Some(stage); + return Err(stage_state("a different migration fixed stage was active")); + } + let published = stage.remove(authority.root())?; + match artifact { + FixedArtifact::Intent => authority.published_intent = Some(published), + FixedArtifact::Marker => authority.published_marker = Some(published), + FixedArtifact::Receipt => authority.published_receipt = Some(published), + } + Ok(()) +} + +fn active_stage( + authority: &FilesystemStoreMigrationAuthority, + artifact: FixedArtifact, +) -> io::Result<&FilesystemMigrationFixedStage> { + let stage = authority + .fixed_stage + .as_ref() + .ok_or_else(|| stage_state("migration fixed stage was not active"))?; + if stage.artifact() == artifact { + Ok(stage) + } else { + Err(stage_state("a different migration fixed stage was active")) + } +} + +fn verify_published( + authority: &FilesystemStoreMigrationAuthority, + artifact: FixedArtifact, +) -> io::Result<()> { + published_stage(authority, artifact) + .ok_or_else(|| stage_state("migration fixed record was not published"))? + .verify_canonical(authority.root()) +} + +const fn published_stage( + authority: &FilesystemStoreMigrationAuthority, + artifact: FixedArtifact, +) -> Option<&FilesystemMigrationFixedStage> { + match artifact { + FixedArtifact::Intent => authority.published_intent.as_ref(), + FixedArtifact::Marker => authority.published_marker.as_ref(), + FixedArtifact::Receipt => authority.published_receipt.as_ref(), + } +} + +fn verify_namespace_view(authority: &FilesystemStoreMigrationAuthority) -> io::Result<()> { + verify_published(authority, FixedArtifact::Intent)?; + filesystem_migration_namespace::verify_namespace_prefix(authority.root()) +} + +fn verify_marker_view(authority: &FilesystemStoreMigrationAuthority) -> io::Result<()> { + verify_published(authority, FixedArtifact::Intent)?; + verify_published(authority, FixedArtifact::Marker)?; + filesystem_migration_namespace::verify_marker_view(authority.root()) +} + +fn verify_receipt_view(authority: &FilesystemStoreMigrationAuthority) -> io::Result<()> { + verify_published(authority, FixedArtifact::Intent)?; + verify_published(authority, FixedArtifact::Marker)?; + verify_published(authority, FixedArtifact::Receipt)?; + filesystem_migration_namespace::verify_receipt_view(authority.root()) +} + +fn synchronize_root(authority: &FilesystemStoreMigrationAuthority) -> io::Result<()> { + filesystem_catalog_artifact::synchronize_directory(authority.root()) +} + +fn stage_state(message: &'static str) -> io::Error { + io::Error::new(io::ErrorKind::InvalidData, message) +} diff --git a/src/adapters/store_migration/filesystem_migration_storage_tests.rs b/src/adapters/store_migration/filesystem_migration_storage_tests.rs new file mode 100644 index 0000000..cd21454 --- /dev/null +++ b/src/adapters/store_migration/filesystem_migration_storage_tests.rs @@ -0,0 +1,203 @@ +//! Filesystem migration storage laws. + +use std::collections::BTreeSet; +use std::error::Error; +use std::ffi::OsString; +use std::fs; +use std::io; +use std::path::{Path, PathBuf}; + +use super::filesystem_migration_test_fixture::open_authority; +use super::{ + AdmittedStoreFormatMarker, AdmittedStoreMigrationIntent, AdmittedStoreMigrationReceipt, + CanonicalStoreFormatMarker, StoreMigrationStorage, execute_store_migration, +}; + +#[test] +fn complete_migration_preserves_v1_bytes_and_publishes_exact_v2_prefix() +-> Result<(), Box> { + let (sandbox, mut authority) = open_authority("filesystem-migration-complete")?; + let before = version_one_witness(sandbox.path())?; + let intent = authority.observe_intent()?; + + let receipt = execute_store_migration(&mut authority, &intent)?; + + assert_eq!(version_one_witness(sandbox.path())?, before); + admit_published_records(sandbox.path(), &intent, receipt.encoded())?; + assert_complete_namespace(sandbox.path())?; + drop(authority); + sandbox.remove()?; + Ok(()) +} + +#[test] +fn existing_intent_stage_is_never_truncated() -> Result<(), Box> { + let (sandbox, mut authority) = open_authority("filesystem-migration-exclusive-stage")?; + let intent = authority.observe_intent()?; + StoreMigrationStorage::verify_current(&mut authority, &intent)?; + let stage = sandbox.path().join("migration.intent.next"); + fs::write(&stage, b"retained partial evidence")?; + + let error = StoreMigrationStorage::write_intent_stage(&mut authority, &intent) + .err() + .ok_or("existing intent stage was unexpectedly replaced")?; + + assert_eq!(error.kind(), io::ErrorKind::AlreadyExists); + assert_eq!(fs::read(&stage)?, b"retained partial evidence"); + drop(authority); + sandbox.remove()?; + Ok(()) +} + +#[test] +fn byte_equal_substituted_canonical_intent_is_refused() -> Result<(), Box> { + let (sandbox, mut authority) = open_authority("filesystem-migration-substituted-target")?; + let intent = authority.observe_intent()?; + StoreMigrationStorage::verify_current(&mut authority, &intent)?; + StoreMigrationStorage::write_intent_stage(&mut authority, &intent)?; + StoreMigrationStorage::synchronize_intent_stage(&mut authority)?; + fs::write(sandbox.path().join("migration.intent"), intent.encoded())?; + + let error = StoreMigrationStorage::link_intent(&mut authority, &intent) + .err() + .ok_or("substituted canonical intent was unexpectedly admitted")?; + + assert_eq!(error.kind(), io::ErrorKind::InvalidData); + drop(authority); + sandbox.remove()?; + Ok(()) +} + +#[test] +fn out_of_order_namespace_refuses_before_creating_its_predecessor() -> Result<(), Box> { + let (sandbox, mut authority) = open_authority("filesystem-migration-prefix-order")?; + let intent = authority.observe_intent()?; + publish_intent_and_reader(&mut authority, &intent)?; + fs::create_dir(sandbox.path().join("gc"))?; + + let error = StoreMigrationStorage::admit_namespace_prefix(&mut authority) + .err() + .ok_or("out-of-order gc namespace was unexpectedly admitted")?; + + assert_eq!(error.kind(), io::ErrorKind::InvalidData); + assert!(!sandbox.path().join("retention").exists()); + drop(authority); + sandbox.remove()?; + Ok(()) +} + +#[test] +fn canonical_intent_drift_refuses_before_marker_stage_creation() -> Result<(), Box> { + let (sandbox, mut authority) = open_authority("filesystem-migration-intent-drift")?; + let intent = authority.observe_intent()?; + publish_intent_and_reader(&mut authority, &intent)?; + StoreMigrationStorage::admit_namespace_prefix(&mut authority)?; + StoreMigrationStorage::synchronize_root_after_namespace(&mut authority)?; + fs::write( + sandbox.path().join("migration.intent"), + vec![0_u8; intent.encoded().len()], + )?; + let marker = CanonicalStoreFormatMarker::version_two(); + + let error = StoreMigrationStorage::write_marker_stage(&mut authority, &marker) + .err() + .ok_or("changed canonical intent unexpectedly authorized marker publication")?; + + assert_eq!(error.kind(), io::ErrorKind::InvalidData); + assert!(!sandbox.path().join("FORMAT.next").exists()); + drop(authority); + sandbox.remove()?; + Ok(()) +} + +fn publish_intent_and_reader( + authority: &mut super::FilesystemStoreMigrationAuthority, + intent: &super::CanonicalStoreMigrationIntent, +) -> io::Result<()> { + StoreMigrationStorage::verify_current(authority, intent)?; + StoreMigrationStorage::write_intent_stage(authority, intent)?; + StoreMigrationStorage::synchronize_intent_stage(authority)?; + StoreMigrationStorage::link_intent(authority, intent)?; + StoreMigrationStorage::synchronize_root_after_intent(authority)?; + StoreMigrationStorage::remove_intent_stage(authority)?; + StoreMigrationStorage::synchronize_root_after_intent_cleanup(authority)?; + StoreMigrationStorage::admit_reader_fence(authority) +} + +fn admit_published_records( + root: &Path, + expected_intent: &super::CanonicalStoreMigrationIntent, + expected_receipt: &[u8], +) -> Result<(), Box> { + let intent_bytes = fs::read(root.join("migration.intent"))?; + let marker_bytes = fs::read(root.join("FORMAT"))?; + let receipt_bytes = fs::read(root.join("migration.receipt"))?; + let intent = AdmittedStoreMigrationIntent::decode(&intent_bytes)?; + let marker = AdmittedStoreFormatMarker::decode(&marker_bytes)?; + let receipt = AdmittedStoreMigrationReceipt::decode(&receipt_bytes, &intent, &marker)?; + assert_eq!(intent.encoded(), expected_intent.encoded()); + assert_eq!(receipt.encoded(), expected_receipt); + Ok(()) +} + +fn assert_complete_namespace(root: &Path) -> Result<(), Box> { + let expected = BTreeSet::from([ + OsString::from("FORMAT"), + OsString::from("HEAD"), + OsString::from("catalogs"), + OsString::from("gc"), + OsString::from("migration.intent"), + OsString::from("migration.receipt"), + OsString::from("reader.lock"), + OsString::from("recovery"), + OsString::from("retention"), + OsString::from("segments"), + OsString::from("staging"), + OsString::from("writer.lock"), + ]); + assert_eq!(directory_names(root)?, expected); + assert_eq!(fs::metadata(root.join("reader.lock"))?.len(), 0); + assert_eq!( + directory_names(&root.join("retention"))?, + BTreeSet::from([OsString::from("manifests"), OsString::from("roots")]) + ); + assert_eq!( + directory_names(&root.join("recovery"))?, + BTreeSet::from([OsString::from("dispositions")]) + ); + for relative in [ + "gc", + "retention/roots", + "retention/manifests", + "recovery/dispositions", + ] { + assert!(directory_names(&root.join(relative))?.is_empty()); + } + for stage in [ + "migration.intent.next", + "FORMAT.next", + "migration.receipt.next", + ] { + assert!(!root.join(stage).exists()); + } + Ok(()) +} + +fn version_one_witness(root: &Path) -> io::Result)>> { + let mut witness = Vec::new(); + witness.push((PathBuf::from("HEAD"), fs::read(root.join("HEAD"))?)); + for pool in ["segments", "catalogs"] { + for entry in fs::read_dir(root.join(pool))? { + let path = PathBuf::from(pool).join(entry?.file_name()); + witness.push((path.clone(), fs::read(root.join(path))?)); + } + } + witness.sort_by(|left, right| left.0.cmp(&right.0)); + Ok(witness) +} + +fn directory_names(path: &Path) -> io::Result> { + fs::read_dir(path)? + .map(|entry| entry.map(|entry| entry.file_name())) + .collect() +} diff --git a/src/adapters/store_migration/filesystem_migration_test_fixture.rs b/src/adapters/store_migration/filesystem_migration_test_fixture.rs new file mode 100644 index 0000000..3463c6f --- /dev/null +++ b/src/adapters/store_migration/filesystem_migration_test_fixture.rs @@ -0,0 +1,40 @@ +//! This test module owns one exact published version-1 migration fixture. + +use std::error::Error; +use std::fs; + +use super::filesystem_migration_authority::FilesystemStoreMigrationAuthority; +use crate::adapters::FilesystemPlatformAdmission; +use crate::adapters::filesystem_test_sandbox::TestDirectory; +use crate::adapters::test_support::decode_hex; + +const SEGMENT_HEX: &str = + include_str!("../../../conformance/segment-store/v1/one-zero-segment.hex"); +const CATALOG_HEX: &str = + include_str!("../../../conformance/segment-store/v1/one-zero-catalog.hex"); +const HEAD_HEX: &str = include_str!("../../../conformance/segment-store/v1/one-zero-head.hex"); +const CATALOG_NAME: &str = + "0000000000000001-04b82519b0399baefd0b9c0f32a871052e4c47e3a00226ab03b21661470f7320.cat"; +const SEGMENT_NAME: &str = "b7542dced2ab770894a14d1d04b066e3a899942602c5986d35ba6df6c1a35cfc.seg"; + +pub(super) fn open_authority( + name: &str, +) -> Result<(TestDirectory, FilesystemStoreMigrationAuthority), Box> { + let sandbox = TestDirectory::create(name)?; + let admission = FilesystemPlatformAdmission::initialize_unchecked_for_tests(sandbox.path())?; + fs::write( + sandbox.path().join("segments").join(SEGMENT_NAME), + decode_hex(SEGMENT_HEX.trim())?, + )?; + fs::write( + sandbox.path().join("catalogs").join(CATALOG_NAME), + decode_hex(CATALOG_HEX.trim())?, + )?; + fs::write(sandbox.path().join("HEAD"), decode_hex(HEAD_HEX.trim())?)?; + let authority = FilesystemStoreMigrationAuthority::open(admission, maximum_policy())?; + Ok((sandbox, authority)) +} + +pub(super) const fn maximum_policy() -> crate::adapters::SegmentReadPolicy { + super::filesystem_inventory_catalogs_test_fixture::maximum_policy() +} diff --git a/src/lib.rs b/src/lib.rs index 18cd737..83570f1 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -35,9 +35,12 @@ //! framing, checksum, catalog and predecessor grammar, registered definition, //! deterministic store identity, and typed recovery coordinates. Completion //! receipts bind an admitted intent and marker, registered empty-state digests, -//! and the complete synchronization mask. Live inventory and root revalidation, -//! filesystem migration, retention execution, recovery, and garbage collection -//! remain intentionally absent. +//! and the complete synchronization mask. Writer-locked filesystem authority +//! now executes one fresh forward migration through exact fixed-record and +//! namespace transitions while retaining version-1 immutable bytes. +//! Partial-prefix migration recovery, filesystem retention execution, +//! immutable reader snapshots, and garbage collection remain intentionally +//! absent. #[cfg(test)] extern crate self as keep; From ece67946c2216b8521c37053f81625f539eed6b5 Mon Sep 17 00:00:00 2001 From: James Ross Date: Sun, 23 Aug 2026 04:01:18 -0700 Subject: [PATCH 052/111] Fix migration recovery contract wording --- docs/formats/segment-store-v2/recovery.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/formats/segment-store-v2/recovery.md b/docs/formats/segment-store-v2/recovery.md index 3a37fa5..444a758 100644 --- a/docs/formats/segment-store-v2/recovery.md +++ b/docs/formats/segment-store-v2/recovery.md @@ -295,6 +295,6 @@ Each point requires before, during, and after process-death evidence. Restart must establish exact catalog visibility, retention head, namespace generation, orphan classification, stage disposition, and recovery report. -GC and recovery-disposition grammar and transitions are owned by the -[GC specification](gc.md). Until issue #21 implements them, any such artifact -is unsupported mandatory state and refuses without mutation. +`GcRetirementIntent`, `GcRetirementReceipt`, and +`RecoveryDispositionReceipt` are owned by the [GC specification](gc.md). Until +issue #21 implements them, any such artifact is unsupported and refuses. From c262a3fb3ed64684ab09dfe8d06a159aaf4d0a52 Mon Sep 17 00:00:00 2001 From: James Ross Date: Sun, 6 Sep 2026 19:15:58 -0700 Subject: [PATCH 053/111] Add fresh filesystem retention publication Implement the production filesystem adapter for the 17-phase retention publication protocol, mirroring the version-2 migration writer. `FilesystemRetentionPublicationAuthority` pins `retention`, `retention/roots`, and `retention/manifests` under retained writer authority, then executes only the forward protocol from an exactly admitted current state. Each of the three records is staged exclusively, verified by device and inode identity at every transition, and committed without replacement: the root and manifest hard-link into their immutable pools, and the retention head is renamed atomically over `retention/HEAD`. Retained stages are removed only after their canonical target reverifies. An exact already-committed candidate returns its receipt with zero retention mutation. Any retained stage refuses as recovery-required rather than being continued, so partial-prefix recovery remains an explicit nonclaim. Version-1 reopen refuses a completely migrated root, so version-2 admission gets its own `admit_version_two` namespace boundary and `reopen_version_two` entry point. Five filesystem laws cover complete publication with a byte-identical migrated witness, exclusive staging, byte-equal inode substitution, retained-stage recovery refusal, and exact committed retry. Evidence recorded against KEEP-RETENTION-004, KEEP-RETENTION-006, KEEP-RETENTION-009, and KEEP-MIGRATION-008. Refs #19 --- .gitignore | 3 + CHANGELOG.md | 10 + docs/formats/segment-store-v2/recovery.md | 6 +- docs/formats/segment-store-v2/requirements.md | 12 +- docs/formats/segment-store-v2/retention.md | 4 +- .../filesystem_initialization_namespace.rs | 42 ++++ src/adapters/filesystem_store_initializer.rs | 42 +++- src/adapters/retention.rs | 13 + .../filesystem_retention_authority.rs | 124 ++++++++++ .../filesystem_retention_authority_error.rs | 63 +++++ .../filesystem_retention_pool_name.rs | 47 ++++ .../retention/filesystem_retention_stage.rs | 158 ++++++++++++ .../retention/filesystem_retention_storage.rs | 213 ++++++++++++++++ .../filesystem_retention_storage_tests.rs | 232 ++++++++++++++++++ .../filesystem_retention_test_fixture.rs | 100 ++++++++ src/lib.rs | 20 +- 16 files changed, 1069 insertions(+), 20 deletions(-) create mode 100644 src/adapters/retention/filesystem_retention_authority.rs create mode 100644 src/adapters/retention/filesystem_retention_authority_error.rs create mode 100644 src/adapters/retention/filesystem_retention_pool_name.rs create mode 100644 src/adapters/retention/filesystem_retention_stage.rs create mode 100644 src/adapters/retention/filesystem_retention_storage.rs create mode 100644 src/adapters/retention/filesystem_retention_storage_tests.rs create mode 100644 src/adapters/retention/filesystem_retention_test_fixture.rs diff --git a/.gitignore b/.gitignore index f34925c..0dcad44 100644 --- a/.gitignore +++ b/.gitignore @@ -29,3 +29,6 @@ *.swp *.swo *~ + +# Local mktxt repository snapshots (exceed the documentation corpus byte budget) +/keep.txt diff --git a/CHANGELOG.md b/CHANGELOG.md index 72be8af..a3df28a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -21,6 +21,16 @@ after its public API and format compatibility policies are established. verification-first execution. Retention preflight combines expected-generation planning with deterministic closure verification; authority-revalidated 17-phase orchestration returns its receipt after durable cleanup. + `FilesystemRetentionPublicationAuthority` executes those 17 phases against a + completely migrated version-2 root: it stages `root.next`, `manifest.next`, + and `head.next` exclusively, verifies device and inode identity at every + transition, hard-links both immutable pool entries without replacement, + atomically replaces `retention/HEAD`, and removes retained stages only after + its canonical target verifies. An exact already-committed candidate returns + its receipt with zero retention mutation, and any retained stage refuses as + recovery-required rather than being continued. Version-1 reopen now refuses a + migrated root, and `admit_version_two` owns the separate version-2 namespace + boundary. - Repository crash-matrix execution now terminates isolated writer process groups at all 105 canonical before/during/after coordinates, retains open writer and stage authority until termination, executes production diff --git a/docs/formats/segment-store-v2/recovery.md b/docs/formats/segment-store-v2/recovery.md index 444a758..0b48e83 100644 --- a/docs/formats/segment-store-v2/recovery.md +++ b/docs/formats/segment-store-v2/recovery.md @@ -59,9 +59,9 @@ corpus `definition.tsv` bytes. The format-marker digest is BLAKE3-256 of `CanonicalStoreFormatMarker` produces the registered marker; `AdmittedStoreFormatMarker` admits its framing, checksum, definition, and namespace bound. `CanonicalStoreMigrationIntent` retains typed intent coordinates; `CanonicalStoreMigrationReceipt` binds completion; admitted record types verify both. `StoreMigrationStorage` names all 21 durability capabilities; `execute_store_migration` verifies current authority first and returns only after final synchronization. -`FilesystemStoreMigrationInventoryReader` inventories immutable version-1 bytes -under retained writer authority and pinned pools. The fresh filesystem writer -executes once; partial-prefix restart admission and recovery remain absent. +`FilesystemStoreMigrationInventoryReader` inventories version-1 bytes under +retained writer authority. The fresh writer executes once; partial-prefix +recovery is absent and version-1 reopen refuses a migrated root. ## Reader fence diff --git a/docs/formats/segment-store-v2/requirements.md b/docs/formats/segment-store-v2/requirements.md index cdb7009..f624d5f 100644 --- a/docs/formats/segment-store-v2/requirements.md +++ b/docs/formats/segment-store-v2/requirements.md @@ -12,12 +12,12 @@ case is not evidence. | `KEEP-RETENTION-001` | `RetentionNamespace`, `RootGeneration`, `LivenessGeneration`, profile coordinates, limits, anchors, and digests are validated typed values | `tests/retention_values.rs`, `tests/retention_root_encoding.rs`, and typed verified anchor-set evidence in `tests/retention_root_decoding.rs` | Implemented | | `KEEP-RETENTION-002` | Root, manifest, and head codecs implement the exact canonical grammars and fixed bounds | independent golden corpus plus `tests/retention_root_encoding.rs`, `tests/retention_root_decoding.rs`, `tests/retention_manifest_codec.rs`, and `tests/retention_head_codec.rs` | Implemented | | `KEEP-RETENTION-003` | Every structural field, truncation boundary, ordering law, duplicate, overflow, flag, reserved byte, digest, checksum, and trailing byte has a precise refusal | seeded `retention_format` fuzz target plus the root, manifest, and head corruption matrix; mutation coverage remains | In progress in #19 | -| `KEEP-RETENTION-004` | Retain and release compare expected and observed generations and publish exact successors only | unforgeable readiness and preflight proofs in `tests/retention_transition.rs` and `tests/retention_preflight.rs`; exact successor preparation and complete receipt evidence in `tests/retention_publication_preparation.rs` and `tests/retention_publication_execution.rs`; filesystem evidence remains | In progress in #19 | +| `KEEP-RETENTION-004` | Retain and release compare expected and observed generations and publish exact successors only | unforgeable readiness and preflight proofs in `tests/retention_transition.rs` and `tests/retention_preflight.rs`; exact successor preparation and complete receipt evidence in `tests/retention_publication_preparation.rs` and `tests/retention_publication_execution.rs`; writer-locked forward filesystem publication in `filesystem_retention_storage_tests`; successor-generation filesystem evidence remains | In progress in #19 | | `KEEP-RETENTION-005` | Closure derivation is deterministic, bounded, cycle-safe, fail-closed, and verifies complete blob reconstruction | exact accounting, reconstruction, adversarial-catalog, and exhaustive model laws in `tests/retention_closure.rs`; corrupt members refuse through the inherited segment-record admission laws and seeded `segment_format` fuzz target routed by `closure-corruption.md` | Implemented | -| `KEEP-RETENTION-006` | Publication follows the exact ordered durability protocol, including new namespace-directory admission and retention of fixed-stage evidence until head commit, and returns only after cleanup synchronization | typed vocabulary and blocking port in `tests/retention_publication_phase.rs` and `tests/retention_publication_storage.rs`; ordered execution, conditional namespace sync, and all 17 exact storage-fault boundaries in `tests/retention_publication_execution.rs`; crash injection remains | In progress in #19 | +| `KEEP-RETENTION-006` | Publication follows the exact ordered durability protocol, including new namespace-directory admission and retention of fixed-stage evidence until head commit, and returns only after cleanup synchronization | typed vocabulary and blocking port in `tests/retention_publication_phase.rs` and `tests/retention_publication_storage.rs`; ordered execution, conditional namespace sync, and all 17 exact storage-fault boundaries in `tests/retention_publication_execution.rs`; production 17-phase forward filesystem execution, exclusive staging, byte-equal inode-substitution refusal, and retained-stage recovery refusal in `filesystem_retention_storage_tests`; crash injection remains | In progress in #19 | | `KEEP-RETENTION-007` | Restart resolves every fixed-stage crash prefix to one documented lawful state or typed ambiguity | debug and release crash matrix | Planned in #19 | | `KEEP-RETENTION-008` | Readers double-collect catalog and retention heads and bind one complete catalog, manifest, and root-generation view under a `ReaderFence` | immutable snapshot and concurrency tests | Planned in #19 | -| `KEEP-RETENTION-009` | Exact already-committed retry is idempotent only while its successor remains current | byte-identical planning in `tests/retention_transition.rs`; authority-revalidated zero-mutation retry receipt in `tests/retention_publication_execution.rs`; filesystem retry remains | In progress in #19 | +| `KEEP-RETENTION-009` | Exact already-committed retry is idempotent only while its successor remains current | byte-identical planning in `tests/retention_transition.rs`; authority-revalidated zero-mutation retry receipt in `tests/retention_publication_execution.rs`; exact already-committed filesystem retry with a byte-identical retention witness in `filesystem_retention_storage_tests`; superseded-successor filesystem refusal remains | In progress in #19 | | `KEEP-RETENTION-010` | Model operation sequences agree with a deterministic namespace-to-anchor-set map and never admit caller identity, paths, clocks, or application policy | model-based and source-architecture tests | Planned in #19 | @@ -35,7 +35,7 @@ case is not evidence. | `KEEP-MIGRATION-005` | Unknown, out-of-order, substituted, corrupt, conflicting, or changed evidence is unrecoverable ambiguity | forward-execution stage preservation, byte-equal inode-substitution, out-of-order-prefix, and post-publication drift laws in `filesystem_migration_storage_tests`; restart corruption and mutation matrix remains | In progress in #19 | | `KEEP-MIGRATION-006` | Migration never rewrites or deletes admitted version-1 immutable bytes | exact segment, catalog, and head before/after witness in `filesystem_migration_storage_tests`; restart-path evidence remains | In progress in #19 | | `KEEP-MIGRATION-007` | Process death around every intent stage, canonical link, namespace prefix, marker stage, receipt stage, cleanup, and synchronization boundary reaches a documented lawful state | ordered phases and capabilities in `tests/store_migration_phase.rs` and `tests/store_migration_storage.rs`; exact phase-failure execution in `tests/store_migration_execution.rs`; production 21-phase forward execution in `filesystem_migration_storage_tests`; `KEEP-CRASH-053..=073` process-death matrix remains | In progress in #19 | -| `KEEP-MIGRATION-008` | Version-1 admission refuses every version-2 or partial-migration artifact after migration begins | `FORMAT` refusal before mutation in `filesystem_migration_authority_tests`; remaining compatibility and fuzz matrix | In progress in #19 | +| `KEEP-MIGRATION-008` | Version-1 admission refuses every version-2 or partial-migration artifact after migration begins | `FORMAT` refusal before mutation in `filesystem_migration_authority_tests`; exact version-1 reopen refusal of a migrated root and separate version-2 namespace admission in `filesystem_initialization_namespace`, exercised by `filesystem_retention_storage_tests`; remaining compatibility and fuzz matrix | In progress in #19 | @@ -59,6 +59,8 @@ case is not evidence. application meaning, causal ownership, future policy, or secure erasure. - A fresh forward writer is not proof that version 2 is restart-safe or production-admitted; partial-prefix recovery and crash evidence remain - mandatory. + mandatory. This applies to retention publication exactly as it applies to + migration: the filesystem publication writer refuses every retained stage + instead of continuing it. - Benchmarks are required before performance-sensitive retention or migration optimization. diff --git a/docs/formats/segment-store-v2/retention.md b/docs/formats/segment-store-v2/retention.md index e1d192c..745f265 100644 --- a/docs/formats/segment-store-v2/retention.md +++ b/docs/formats/segment-store-v2/retention.md @@ -163,8 +163,8 @@ Names with alternate width, case, suffix, generation, or digest refuse. Keep implements root, manifest, and head codecs with a typed verified anchor-set digest, expected-state transition planning, deterministic closure verification, a blocking publication storage capability port, and ordered storage-port -orchestration. Production filesystem execution, recovery, and garbage -collection remain absent. +orchestration. `FilesystemRetentionPublicationAuthority` executes that protocol +once; retained-stage recovery, reader fencing, and collection remain absent. ## Global retention manifest diff --git a/src/adapters/filesystem_initialization_namespace.rs b/src/adapters/filesystem_initialization_namespace.rs index 2ec01d1..2f927ac 100644 --- a/src/adapters/filesystem_initialization_namespace.rs +++ b/src/adapters/filesystem_initialization_namespace.rs @@ -18,6 +18,27 @@ const PUBLISHED_NAMES: [&str; 5] = [ CATALOGS_NAME, HEAD_NAME, ]; +const READER_LOCK_NAME: &str = "reader.lock"; +const MARKER_NAME: &str = "FORMAT"; +const INTENT_NAME: &str = "migration.intent"; +const RECEIPT_NAME: &str = "migration.receipt"; +const RETENTION_NAME: &str = "retention"; +const GC_NAME: &str = "gc"; +const RECOVERY_NAME: &str = "recovery"; +const VERSION_TWO_NAMES: [&str; 12] = [ + LOCK_NAME, + STAGING_NAME, + SEGMENTS_NAME, + CATALOGS_NAME, + HEAD_NAME, + READER_LOCK_NAME, + MARKER_NAME, + INTENT_NAME, + RECEIPT_NAME, + RETENTION_NAME, + GC_NAME, + RECOVERY_NAME, +]; pub(super) fn admit(directory: &Dir) -> io::Result<()> { admit_optional_file(directory, LOCK_NAME)?; @@ -36,6 +57,27 @@ pub(super) fn admit_published(directory: &Dir) -> io::Result<()> { admit_membership(directory, &PUBLISHED_NAMES) } +/// Admits the exact completely migrated version-2 root namespace. +/// +/// Every version-1 published entry, the persistent reader fence, the format +/// marker, both migration records, and all three protocol directories must be +/// present. Any other entry is unrecoverable ambiguity. +pub(super) fn admit_version_two(directory: &Dir) -> io::Result<()> { + admit_required_file(directory, LOCK_NAME)?; + admit_required_directory(directory, STAGING_NAME)?; + admit_required_directory(directory, SEGMENTS_NAME)?; + admit_required_directory(directory, CATALOGS_NAME)?; + admit_required_file(directory, HEAD_NAME)?; + admit_required_file(directory, READER_LOCK_NAME)?; + admit_required_file(directory, MARKER_NAME)?; + admit_required_file(directory, INTENT_NAME)?; + admit_required_file(directory, RECEIPT_NAME)?; + admit_required_directory(directory, RETENTION_NAME)?; + admit_required_directory(directory, GC_NAME)?; + admit_required_directory(directory, RECOVERY_NAME)?; + admit_membership(directory, &VERSION_TWO_NAMES) +} + fn admit_optional_file(directory: &Dir, name: &str) -> io::Result<()> { admit_optional_kind(directory, name, cap_std::fs::FileType::is_file) } diff --git a/src/adapters/filesystem_store_initializer.rs b/src/adapters/filesystem_store_initializer.rs index 80456bb..1e4f7ef 100644 --- a/src/adapters/filesystem_store_initializer.rs +++ b/src/adapters/filesystem_store_initializer.rs @@ -66,6 +66,33 @@ impl FilesystemPlatformAdmission { initialize_storage(storage) } + /// Reacquires writer authority over one completely migrated version-2 store. + /// + /// The call mutates no protocol state. It admits the production platform, + /// acquires the existing writer lock, and requires the exact version-2 root + /// namespace. Retention and recovery adapters perform content-level + /// validation under the returned authority. The synchronous call may block + /// on filesystem I/O. + /// + /// # Errors + /// + /// Returns [`FilesystemPlatformAdmissionError`] with the exact platform, + /// writer-lock, or namespace boundary and preserved source. + pub fn reopen_version_two(store_root: &Path) -> Result { + let root = filesystem_platform_profile::open(store_root) + .map_err(|source| FilesystemPlatformAdmissionError::Platform { source })?; + reopen_version_two_root(root) + } + + #[cfg(test)] + pub(super) fn reopen_version_two_unchecked_for_tests( + store_root: &Path, + ) -> Result { + let root = Dir::open_ambient_dir(store_root, ambient_authority()) + .map_err(|source| FilesystemPlatformAdmissionError::Platform { source })?; + reopen_version_two_root(root) + } + #[cfg(test)] pub(super) fn reopen_unchecked_for_tests( store_root: &Path, @@ -96,15 +123,28 @@ fn initialize_storage( )) } +fn reopen_version_two_root( + root: cap_std::fs::Dir, +) -> Result { + admit_reopened(root, filesystem_initialization_namespace::admit_version_two) +} + fn reopen_root( root: cap_std::fs::Dir, +) -> Result { + admit_reopened(root, filesystem_initialization_namespace::admit_published) +} + +fn admit_reopened( + root: cap_std::fs::Dir, + admit_namespace: fn(&cap_std::fs::Dir) -> std::io::Result<()>, ) -> Result { let lock = FilesystemWriterLock::try_acquire_in(root) .map_err(|source| FilesystemPlatformAdmissionError::WriterLock { source })?; let directory = lock .clone_directory() .map_err(|source| FilesystemPlatformAdmissionError::Namespace { source })?; - filesystem_initialization_namespace::admit_published(&directory) + admit_namespace(&directory) .map_err(|source| FilesystemPlatformAdmissionError::Namespace { source })?; let root_identity = filesystem_platform_profile::root_identity(&directory) .map_err(|source| FilesystemPlatformAdmissionError::Platform { source })?; diff --git a/src/adapters/retention.rs b/src/adapters/retention.rs index 10943c0..cda32d4 100644 --- a/src/adapters/retention.rs +++ b/src/adapters/retention.rs @@ -13,6 +13,15 @@ mod closure_error_display; mod closure_member; mod closure_profile_error; mod closure_verifier; +mod filesystem_retention_authority; +mod filesystem_retention_authority_error; +mod filesystem_retention_pool_name; +mod filesystem_retention_stage; +mod filesystem_retention_storage; +#[cfg(test)] +mod filesystem_retention_storage_tests; +#[cfg(test)] +mod filesystem_retention_test_fixture; mod head_decode_error; mod head_decode_error_display; mod head_decoder; @@ -65,6 +74,10 @@ pub use canonical_root::CanonicalRetentionRoot; pub use checksummed_head::ChecksummedRetentionHead; pub use closure_error::RetentionClosureVerificationError; pub use closure_verifier::verify_retention_closure; +pub use filesystem_retention_authority::FilesystemRetentionPublicationAuthority; +pub use filesystem_retention_authority_error::{ + FilesystemRetentionAuthorityError, RetentionAuthorityDirectory, +}; pub use head_decode_error::RetentionHeadDecodeError; pub use manifest_decode_error::RetentionManifestDecodeError; pub use manifest_encode_error::RetentionManifestEncodeError; diff --git a/src/adapters/retention/filesystem_retention_authority.rs b/src/adapters/retention/filesystem_retention_authority.rs new file mode 100644 index 0000000..2469f2f --- /dev/null +++ b/src/adapters/retention/filesystem_retention_authority.rs @@ -0,0 +1,124 @@ +//! This module owns exact writer-locked filesystem retention authority. + +use std::io; + +use cap_fs_ext::DirExt; +use cap_std::fs::Dir; + +use super::filesystem_retention_authority_error::{ + FilesystemRetentionAuthorityError as Error, RetentionAuthorityDirectory as Directory, +}; +use super::filesystem_retention_pool_name as pool_name; +use super::filesystem_retention_stage::{FilesystemRetentionStage, invalid_data}; +use crate::adapters::{FilesystemPlatformAdmission, FilesystemWriterLock}; + +/// Exclusive authority to publish retention transitions on one pinned root. +/// +/// The authority retains the admitted writer lock and pinned `retention`, +/// `retention/roots`, and `retention/manifests` capabilities for its entire +/// lifetime. When passed to +/// [`execute_retention_publication`](crate::execute_retention_publication) its +/// [`RetentionPublicationStorage`](super::RetentionPublicationStorage) +/// implementation executes only the forward publication protocol from an +/// exactly admitted current state. It retains opened stage handles through +/// final verification, performs synchronous capability-relative I/O, and uses +/// neither a network nor an asynchronous runtime. Reopening a retained stage +/// prefix remains a separate recovery boundary. +#[must_use] +pub struct FilesystemRetentionPublicationAuthority { + pub(super) retention: Dir, + pub(super) roots: Dir, + pub(super) manifests: Dir, + pub(super) namespace: Option, + pub(super) liveness_generation: Option, + pub(super) retained_root: Option, + pub(super) retained_manifest: Option, + pub(super) root_stage: Option, + pub(super) manifest_stage: Option, + pub(super) head_stage: Option, + _lock: FilesystemWriterLock, +} + +impl FilesystemRetentionPublicationAuthority { + /// Pins one migrated version-2 root for retention publication. + /// + /// This synchronous constructor opens pinned directory capabilities but + /// materializes no record bodies and performs no protocol mutation. + /// + /// # Errors + /// + /// Returns [`FilesystemRetentionAuthorityError`](super::FilesystemRetentionAuthorityError) + /// when the root capability cannot be cloned or the retention namespace and + /// either immutable pool cannot be pinned without following links. + pub fn open(admission: FilesystemPlatformAdmission) -> Result { + let lock = admission.into_lock(); + let root = lock.clone_directory().map_err(|source| Error::Directory { + directory: Directory::Root, + source, + })?; + let retention = open_directory(&root, pool_name::RETENTION, Directory::Retention)?; + let roots = open_directory(&retention, pool_name::ROOTS, Directory::Roots)?; + let manifests = open_directory(&retention, pool_name::MANIFESTS, Directory::Manifests)?; + Ok(Self { + retention, + roots, + manifests, + namespace: None, + liveness_generation: None, + retained_root: None, + retained_manifest: None, + root_stage: None, + manifest_stage: None, + head_stage: None, + _lock: lock, + }) + } + + pub(super) fn namespace(&self) -> io::Result<&Dir> { + self.namespace + .as_ref() + .ok_or_else(|| invalid_data("retention root namespace was not admitted")) + } + + pub(super) fn take_root_stage(&mut self) -> io::Result { + self.root_stage + .take() + .ok_or_else(|| invalid_data("retention root stage was not retained")) + } + + pub(super) fn take_manifest_stage(&mut self) -> io::Result { + self.manifest_stage + .take() + .ok_or_else(|| invalid_data("retention manifest stage was not retained")) + } + + pub(super) fn take_head_stage(&mut self) -> io::Result { + self.head_stage + .take() + .ok_or_else(|| invalid_data("retention head stage was not retained")) + } + + pub(super) fn root_stage(&self) -> io::Result<&FilesystemRetentionStage> { + self.root_stage + .as_ref() + .ok_or_else(|| invalid_data("retention root stage was not retained")) + } + + pub(super) fn manifest_stage(&self) -> io::Result<&FilesystemRetentionStage> { + self.manifest_stage + .as_ref() + .ok_or_else(|| invalid_data("retention manifest stage was not retained")) + } + + pub(super) fn head_stage(&self) -> io::Result<&FilesystemRetentionStage> { + self.head_stage + .as_ref() + .ok_or_else(|| invalid_data("retention head stage was not retained")) + } +} + +fn open_directory(parent: &Dir, name: &str, directory: Directory) -> Result { + parent + .open_dir_nofollow(name) + .map_err(|source| Error::Directory { directory, source }) +} diff --git a/src/adapters/retention/filesystem_retention_authority_error.rs b/src/adapters/retention/filesystem_retention_authority_error.rs new file mode 100644 index 0000000..78690cb --- /dev/null +++ b/src/adapters/retention/filesystem_retention_authority_error.rs @@ -0,0 +1,63 @@ +//! This boundary module owns filesystem retention authority refusals. + +use std::error::Error; +use std::fmt; +use std::io; + +/// Protocol directory a retention authority failed to pin. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum RetentionAuthorityDirectory { + /// The pinned store root. + Root, + /// The `retention` protocol directory. + Retention, + /// The `retention/roots` immutable pool. + Roots, + /// The `retention/manifests` immutable pool. + Manifests, +} + +impl fmt::Display for RetentionAuthorityDirectory { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter.write_str(match self { + Self::Root => "store root", + Self::Retention => "retention", + Self::Roots => "retention/roots", + Self::Manifests => "retention/manifests", + }) + } +} + +/// Exact refusal opening one writer-locked filesystem retention authority. +#[derive(Debug)] +#[non_exhaustive] +pub enum FilesystemRetentionAuthorityError { + /// A required protocol directory could not be pinned without following links. + Directory { + /// The directory that refused. + directory: RetentionAuthorityDirectory, + /// The exact underlying failure. + source: io::Error, + }, +} + +impl fmt::Display for FilesystemRetentionAuthorityError { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::Directory { directory, .. } => { + write!( + formatter, + "could not pin {directory} for retention publication" + ) + } + } + } +} + +impl Error for FilesystemRetentionAuthorityError { + fn source(&self) -> Option<&(dyn Error + 'static)> { + match self { + Self::Directory { source, .. } => Some(source), + } + } +} diff --git a/src/adapters/retention/filesystem_retention_pool_name.rs b/src/adapters/retention/filesystem_retention_pool_name.rs new file mode 100644 index 0000000..f50805a --- /dev/null +++ b/src/adapters/retention/filesystem_retention_pool_name.rs @@ -0,0 +1,47 @@ +//! Exact retention immutable-pool and namespace filename emission. + +use std::fmt; + +use crate::{ + LivenessGeneration, RetentionManifestDigest, RetentionNamespaceDigest, RetentionRootDigest, + RootGeneration, +}; + +pub(super) const RETENTION: &str = "retention"; +pub(super) const ROOTS: &str = "roots"; +pub(super) const MANIFESTS: &str = "manifests"; +pub(super) const HEAD: &str = "HEAD"; +pub(super) const ROOT_STAGE: &str = "root.next"; +pub(super) const MANIFEST_STAGE: &str = "manifest.next"; +pub(super) const HEAD_STAGE: &str = "head.next"; + +pub(super) fn namespace(digest: RetentionNamespaceDigest) -> String { + DigestHex(digest.as_bytes()).to_string() +} + +pub(super) fn root(generation: RootGeneration, digest: RetentionRootDigest) -> String { + format!( + "{:016x}-{}.root", + generation.get(), + DigestHex(digest.as_bytes()) + ) +} + +pub(super) fn manifest(generation: LivenessGeneration, digest: RetentionManifestDigest) -> String { + format!( + "{:016x}-{}.manifest", + generation.get(), + DigestHex(digest.as_bytes()) + ) +} + +struct DigestHex<'a>(&'a [u8; 32]); + +impl fmt::Display for DigestHex<'_> { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + for byte in self.0 { + write!(formatter, "{byte:02x}")?; + } + Ok(()) + } +} diff --git a/src/adapters/retention/filesystem_retention_stage.rs b/src/adapters/retention/filesystem_retention_stage.rs new file mode 100644 index 0000000..d882f56 --- /dev/null +++ b/src/adapters/retention/filesystem_retention_stage.rs @@ -0,0 +1,158 @@ +//! This module owns exact variable-length retention stage publication. + +use std::io::{self, Read, Write}; + +use cap_fs_ext::{FollowSymlinks, MetadataExt, OpenOptionsFollowExt}; +use cap_std::fs::{Dir, File, Metadata, OpenOptions}; + +use crate::adapters::filesystem_catalog_artifact; + +/// One exclusively created, verified, and retained retention stage file. +/// +/// The stage retains its opened handle and recorded device and inode identity +/// for its whole lifetime. Every transition reverifies both the handle and the +/// named entry, so a replaced or byte-equal substituted file refuses instead of +/// being admitted. +pub(super) struct FilesystemRetentionStage { + name: &'static str, + expected: Box<[u8]>, + identity: StageIdentity, + file: File, +} + +impl FilesystemRetentionStage { + /// Exclusively creates the named stage and writes its complete bytes. + pub(super) fn create(root: &Dir, name: &'static str, expected: &[u8]) -> io::Result { + let mut file = filesystem_catalog_artifact::create_exclusive(root, name)?; + let identity = StageIdentity::read_file(&file)?; + file.write_all(expected)?; + file.flush()?; + Ok(Self { + name, + expected: Box::from(expected), + identity, + file, + }) + } + + /// Synchronizes the complete stage and reverifies its exact bytes. + pub(super) fn synchronize(&self, root: &Dir) -> io::Result<()> { + self.require_handle()?; + self.file.sync_all()?; + self.verify_stage(root) + } + + /// Links the verified stage into `target` under `name` without replacement. + pub(super) fn link(&self, root: &Dir, target: &Dir, name: &str) -> io::Result<()> { + self.verify_stage(root)?; + match root.hard_link(self.name, target, name) { + Ok(()) => {} + Err(source) if source.kind() == io::ErrorKind::AlreadyExists => {} + Err(source) => return Err(source), + } + self.verify_stage(root)?; + verify_name(target, name, &self.expected, self.identity) + } + + /// Removes only the retained stage after confirming its linked target. + pub(super) fn remove(self, root: &Dir, target: &Dir, name: &str) -> io::Result<()> { + verify_name(target, name, &self.expected, self.identity)?; + root.remove_file(self.name)?; + require_absent(root, self.name)?; + verify_name(target, name, &self.expected, self.identity) + } + + /// Renames the verified stage onto `name`, replacing it atomically. + pub(super) fn replace(self, root: &Dir, name: &str) -> io::Result<()> { + self.verify_stage(root)?; + root.rename(self.name, root, name)?; + require_absent(root, self.name)?; + verify_name(root, name, &self.expected, self.identity) + } + + fn require_handle(&self) -> io::Result<()> { + if StageIdentity::read_file(&self.file)? == self.identity { + Ok(()) + } else { + Err(invalid_data("retention stage handle changed identity")) + } + } + + fn verify_stage(&self, root: &Dir) -> io::Result<()> { + self.require_handle()?; + verify_name(root, self.name, &self.expected, self.identity) + } +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +struct StageIdentity { + device: u64, + inode: u64, +} + +impl StageIdentity { + fn read_file(file: &File) -> io::Result { + file.metadata().map(|metadata| Self::from(&metadata)) + } +} + +impl From<&Metadata> for StageIdentity { + fn from(metadata: &Metadata) -> Self { + Self { + device: metadata.dev(), + inode: metadata.ino(), + } + } +} + +fn verify_name( + directory: &Dir, + name: &str, + expected: &[u8], + identity: StageIdentity, +) -> io::Result<()> { + let mut options = OpenOptions::new(); + options.read(true).follow(FollowSymlinks::No); + let mut file = directory.open_with(name, &options)?; + require_metadata(&file.metadata()?, expected.len(), identity)?; + require_metadata(&directory.symlink_metadata(name)?, expected.len(), identity)?; + let mut observed = vec![0_u8; expected.len()]; + file.read_exact(&mut observed)?; + let mut trailing = [0_u8; 1]; + if observed != expected || file.read(&mut trailing)? != 0 { + return Err(invalid_data("retention record bytes disagreed")); + } + require_metadata(&file.metadata()?, expected.len(), identity)?; + require_metadata(&directory.symlink_metadata(name)?, expected.len(), identity) +} + +fn require_metadata( + metadata: &Metadata, + expected_length: usize, + expected_identity: StageIdentity, +) -> io::Result<()> { + let expected_length = u64::try_from(expected_length) + .map_err(|_source| invalid_data("retention record length exceeded u64"))?; + if metadata.is_file() + && metadata.len() == expected_length + && StageIdentity::from(metadata) == expected_identity + { + Ok(()) + } else { + Err(invalid_data( + "retention record kind, length, or identity disagreed", + )) + } +} + +fn require_absent(directory: &Dir, name: &str) -> io::Result<()> { + match directory.symlink_metadata(name) { + Err(source) if source.kind() == io::ErrorKind::NotFound => Ok(()), + Ok(_) => Err(invalid_data("removed retention stage remained visible")), + Err(source) => Err(source), + } +} + +pub(super) fn invalid_data(message: &'static str) -> io::Error { + io::Error::new(io::ErrorKind::InvalidData, message) +} diff --git a/src/adapters/retention/filesystem_retention_storage.rs b/src/adapters/retention/filesystem_retention_storage.rs new file mode 100644 index 0000000..1cdf456 --- /dev/null +++ b/src/adapters/retention/filesystem_retention_storage.rs @@ -0,0 +1,213 @@ +//! This module owns forward filesystem retention publication execution. + +use std::io::{self, Read}; + +use cap_fs_ext::{DirExt, FollowSymlinks, OpenOptionsFollowExt}; +use cap_std::fs::{Dir, OpenOptions}; + +use super::filesystem_retention_authority::FilesystemRetentionPublicationAuthority; +use super::filesystem_retention_pool_name as pool_name; +use super::filesystem_retention_stage::{FilesystemRetentionStage, invalid_data}; +use super::{ + AdmittedRetentionRoot, CanonicalRetentionHead, CanonicalRetentionManifest, + ChecksummedRetentionHead, RetentionNamespaceAdmission, RetentionPublicationPreparation, + RetentionPublicationStorage, RetentionTransitionDisposition, +}; +use crate::adapters::filesystem_catalog_artifact::synchronize_directory; + +const HEAD_LENGTH: usize = 144; + +impl RetentionPublicationStorage for FilesystemRetentionPublicationAuthority { + fn verify_current( + &mut self, + preparation: &RetentionPublicationPreparation<'_>, + ) -> io::Result { + self.liveness_generation = Some(preparation.liveness_generation()); + require_no_retained_stage(&self.retention)?; + let Some(head_bytes) = read_head(&self.retention)? else { + return Ok(RetentionTransitionDisposition::Publish); + }; + let head = ChecksummedRetentionHead::decode(&head_bytes) + .map_err(|_source| invalid_data("current retention head refused admission"))?; + if head.head().generation() == preparation.liveness_generation() + && head.head().manifest_digest() == preparation.manifest_digest() + { + return Ok(RetentionTransitionDisposition::AlreadyCommitted); + } + Err(invalid_data( + "current retention head is not the prepared predecessor; recovery is required", + )) + } + + fn write_root_stage(&mut self, root: &AdmittedRetentionRoot<'_>) -> io::Result<()> { + self.root_stage = Some(FilesystemRetentionStage::create( + &self.retention, + pool_name::ROOT_STAGE, + root.encoded(), + )?); + Ok(()) + } + + fn synchronize_root_stage(&mut self) -> io::Result<()> { + self.root_stage()?.synchronize(&self.retention) + } + + fn admit_root_namespace( + &mut self, + root: &AdmittedRetentionRoot<'_>, + ) -> io::Result { + let name = pool_name::namespace(root.root().namespace().digest()); + let admission = match self.roots.create_dir(&name) { + Ok(()) => RetentionNamespaceAdmission::Created, + Err(source) if source.kind() == io::ErrorKind::AlreadyExists => { + RetentionNamespaceAdmission::Existing + } + Err(source) => return Err(source), + }; + self.namespace = Some(self.roots.open_dir_nofollow(&name)?); + Ok(admission) + } + + fn synchronize_roots_after_namespace(&mut self) -> io::Result<()> { + synchronize_directory(&self.roots) + } + + fn link_root(&mut self, root: &AdmittedRetentionRoot<'_>) -> io::Result<()> { + let name = pool_name::root(root.root().generation(), root.digest()); + let namespace = self.namespace()?; + self.root_stage()?.link(&self.retention, namespace, &name)?; + self.retained_root = Some(name); + Ok(()) + } + + fn synchronize_root_namespace(&mut self, _root: &AdmittedRetentionRoot<'_>) -> io::Result<()> { + synchronize_directory(self.namespace()?) + } + + fn write_manifest_stage(&mut self, manifest: &CanonicalRetentionManifest) -> io::Result<()> { + self.manifest_stage = Some(FilesystemRetentionStage::create( + &self.retention, + pool_name::MANIFEST_STAGE, + manifest.encoded(), + )?); + Ok(()) + } + + fn synchronize_manifest_stage(&mut self) -> io::Result<()> { + self.manifest_stage()?.synchronize(&self.retention) + } + + fn link_manifest(&mut self, manifest: &CanonicalRetentionManifest) -> io::Result<()> { + let name = self.manifest_name(manifest)?; + self.manifest_stage()? + .link(&self.retention, &self.manifests, &name)?; + self.retained_manifest = Some(name); + Ok(()) + } + + fn synchronize_manifest_pool(&mut self) -> io::Result<()> { + synchronize_directory(&self.manifests) + } + + fn write_head_stage(&mut self, head: &CanonicalRetentionHead) -> io::Result<()> { + self.head_stage = Some(FilesystemRetentionStage::create( + &self.retention, + pool_name::HEAD_STAGE, + head.encoded(), + )?); + Ok(()) + } + + fn synchronize_head_stage(&mut self) -> io::Result<()> { + self.head_stage()?.synchronize(&self.retention) + } + + fn replace_head(&mut self) -> io::Result<()> { + self.take_head_stage()? + .replace(&self.retention, pool_name::HEAD) + } + + fn synchronize_retention_namespace(&mut self) -> io::Result<()> { + synchronize_directory(&self.retention) + } + + fn remove_root_stage(&mut self) -> io::Result<()> { + let stage = self.take_root_stage()?; + let name = self.retained_root_name()?; + let namespace = self.namespace()?; + stage.remove(&self.retention, namespace, &name) + } + + fn remove_manifest_stage(&mut self) -> io::Result<()> { + let stage = self.take_manifest_stage()?; + let name = self.retained_manifest_name()?; + stage.remove(&self.retention, &self.manifests, &name) + } + + fn synchronize_cleanup(&mut self) -> io::Result<()> { + synchronize_directory(&self.retention) + } +} + +fn require_no_retained_stage(retention: &Dir) -> io::Result<()> { + for stage in [ + pool_name::ROOT_STAGE, + pool_name::MANIFEST_STAGE, + pool_name::HEAD_STAGE, + ] { + match retention.symlink_metadata(stage) { + Err(source) if source.kind() == io::ErrorKind::NotFound => {} + Ok(_) => { + return Err(invalid_data( + "retained retention stage requires recovery before publication", + )); + } + Err(source) => return Err(source), + } + } + Ok(()) +} + +fn read_head(retention: &Dir) -> io::Result>> { + let mut options = OpenOptions::new(); + options.read(true).follow(FollowSymlinks::No); + let mut file = match retention.open_with(pool_name::HEAD, &options) { + Ok(file) => file, + Err(source) if source.kind() == io::ErrorKind::NotFound => return Ok(None), + Err(source) => return Err(source), + }; + let expected_length = u64::try_from(HEAD_LENGTH) + .map_err(|_source| invalid_data("retention head length exceeded u64"))?; + let metadata = file.metadata()?; + if !metadata.is_file() || metadata.len() != expected_length { + return Err(invalid_data("retention head kind or length disagreed")); + } + let mut bytes = vec![0_u8; HEAD_LENGTH]; + file.read_exact(&mut bytes)?; + let mut trailing = [0_u8; 1]; + if file.read(&mut trailing)? != 0 { + return Err(invalid_data("retention head carried trailing bytes")); + } + Ok(Some(bytes)) +} + +impl FilesystemRetentionPublicationAuthority { + fn manifest_name(&self, manifest: &CanonicalRetentionManifest) -> io::Result { + let generation = self + .liveness_generation + .ok_or_else(|| invalid_data("selected liveness generation was not retained"))?; + Ok(pool_name::manifest(generation, manifest.digest())) + } + + fn retained_root_name(&self) -> io::Result { + self.retained_root + .clone() + .ok_or_else(|| invalid_data("retention root pool coordinate was not retained")) + } + + fn retained_manifest_name(&self) -> io::Result { + self.retained_manifest + .clone() + .ok_or_else(|| invalid_data("retention manifest pool coordinate was not retained")) + } +} diff --git a/src/adapters/retention/filesystem_retention_storage_tests.rs b/src/adapters/retention/filesystem_retention_storage_tests.rs new file mode 100644 index 0000000..5332cbd --- /dev/null +++ b/src/adapters/retention/filesystem_retention_storage_tests.rs @@ -0,0 +1,232 @@ +//! Filesystem retention publication storage laws. + +use std::collections::BTreeSet; +use std::error::Error; +use std::ffi::OsString; +use std::fs; +use std::io; +use std::path::{Path, PathBuf}; + +use super::filesystem_retention_test_fixture::{ + HEAD_HEX, MANIFEST_HEX, ROOT_HEX, fixture, open_authority, with_snapshot, +}; +use super::{ + AdmittedRetentionRoot, RetentionPublicationError, RetentionPublicationOutcome, + RetentionPublicationPreparation, RetentionPublicationStorage, RetentionTransitionDisposition, +}; +use crate::{ + RetentionGenerationExpectation, execute_retention_publication, preflight_retention_transition, + prepare_retention_publication, +}; + +#[test] +fn complete_publication_preserves_migrated_bytes_and_publishes_exact_retention_prefix() +-> Result<(), Box> { + let (sandbox, mut authority) = open_authority("filesystem-retention-complete")?; + let before = migrated_witness(sandbox.path())?; + let root_bytes = fixture(ROOT_HEX)?; + let preparation = publish_preparation(&root_bytes)?; + + let receipt = execute_retention_publication(&mut authority, &preparation)?; + + assert_eq!(receipt.outcome(), RetentionPublicationOutcome::Published); + assert_eq!(migrated_witness(sandbox.path())?, before); + assert_eq!(fs::read(head_path(sandbox.path()))?, fixture(HEAD_HEX)?); + assert_eq!( + fs::read(root_pool_path(sandbox.path(), &preparation))?, + root_bytes + ); + assert_eq!( + fs::read(manifest_pool_path(sandbox.path(), &preparation))?, + fixture(MANIFEST_HEX)? + ); + assert_stages_absent(sandbox.path())?; + drop(authority); + sandbox.remove()?; + Ok(()) +} + +#[test] +fn existing_root_stage_is_never_truncated() -> Result<(), Box> { + let (sandbox, mut authority) = open_authority("filesystem-retention-exclusive-stage")?; + let root_bytes = fixture(ROOT_HEX)?; + let preparation = publish_preparation(&root_bytes)?; + let stage = sandbox.path().join("retention").join("root.next"); + fs::write(&stage, b"retained partial evidence")?; + + let error = + RetentionPublicationStorage::write_root_stage(&mut authority, preparation.candidate()) + .err() + .ok_or("existing root stage was unexpectedly replaced")?; + + assert_eq!(error.kind(), io::ErrorKind::AlreadyExists); + assert_eq!(fs::read(&stage)?, b"retained partial evidence"); + drop(authority); + sandbox.remove()?; + Ok(()) +} + +#[test] +fn retained_stage_refuses_publication_before_recovery() -> Result<(), Box> { + let (sandbox, mut authority) = open_authority("filesystem-retention-recovery-required")?; + let root_bytes = fixture(ROOT_HEX)?; + let preparation = publish_preparation(&root_bytes)?; + fs::write( + sandbox.path().join("retention").join("head.next"), + fixture(HEAD_HEX)?, + )?; + + let error = execute_retention_publication(&mut authority, &preparation) + .err() + .ok_or("retained head stage was unexpectedly published over")?; + + let RetentionPublicationError::CurrentVerification { source } = error else { + return Err("retained stage refused outside current-state verification".into()); + }; + assert_eq!(source.kind(), io::ErrorKind::InvalidData); + assert!(!head_path(sandbox.path()).exists()); + drop(authority); + sandbox.remove()?; + Ok(()) +} + +#[test] +fn byte_equal_substituted_canonical_root_is_refused() -> Result<(), Box> { + let (sandbox, mut authority) = open_authority("filesystem-retention-substituted-target")?; + let root_bytes = fixture(ROOT_HEX)?; + let preparation = publish_preparation(&root_bytes)?; + let candidate = preparation.candidate(); + let _disposition = RetentionPublicationStorage::verify_current(&mut authority, &preparation)?; + RetentionPublicationStorage::write_root_stage(&mut authority, candidate)?; + RetentionPublicationStorage::synchronize_root_stage(&mut authority)?; + let _admission = RetentionPublicationStorage::admit_root_namespace(&mut authority, candidate)?; + RetentionPublicationStorage::synchronize_roots_after_namespace(&mut authority)?; + fs::write(root_pool_path(sandbox.path(), &preparation), &root_bytes)?; + + let error = RetentionPublicationStorage::link_root(&mut authority, candidate) + .err() + .ok_or("byte-equal substituted root was unexpectedly admitted")?; + + assert_eq!(error.kind(), io::ErrorKind::InvalidData); + drop(authority); + sandbox.remove()?; + Ok(()) +} + +#[test] +fn exact_committed_retry_mutates_nothing() -> Result<(), Box> { + let (sandbox, mut authority) = open_authority("filesystem-retention-exact-retry")?; + let root_bytes = fixture(ROOT_HEX)?; + let preparation = publish_preparation(&root_bytes)?; + let _published = execute_retention_publication(&mut authority, &preparation)?; + let after_publication = retention_witness(sandbox.path())?; + let retry = publish_preparation(&root_bytes)?; + + let receipt = execute_retention_publication(&mut authority, &retry)?; + + assert_eq!( + receipt.outcome(), + RetentionPublicationOutcome::AlreadyCommitted + ); + assert_eq!(retention_witness(sandbox.path())?, after_publication); + drop(authority); + sandbox.remove()?; + Ok(()) +} + +fn publish_preparation( + root_bytes: &[u8], +) -> Result, Box> { + let candidate = AdmittedRetentionRoot::decode(root_bytes)?; + let preflight = with_snapshot(|snapshot| { + preflight_retention_transition( + RetentionGenerationExpectation::Absent, + None, + candidate, + snapshot, + ) + })??; + let preparation = prepare_retention_publication(preflight, None)?; + assert_eq!( + preparation.disposition(), + RetentionTransitionDisposition::Publish + ); + Ok(preparation) +} + +fn head_path(root: &Path) -> PathBuf { + root.join("retention").join("HEAD") +} + +fn root_pool_path(root: &Path, preparation: &RetentionPublicationPreparation<'_>) -> PathBuf { + let candidate = preparation.candidate(); + root.join("retention") + .join("roots") + .join(hex(candidate.root().namespace().digest().as_bytes())) + .join(format!( + "{:016x}-{}.root", + candidate.root().generation().get(), + hex(candidate.digest().as_bytes()) + )) +} + +fn manifest_pool_path(root: &Path, preparation: &RetentionPublicationPreparation<'_>) -> PathBuf { + root.join("retention").join("manifests").join(format!( + "{:016x}-{}.manifest", + preparation.liveness_generation().get(), + hex(preparation.manifest_digest().as_bytes()) + )) +} + +fn assert_stages_absent(root: &Path) -> Result<(), Box> { + for stage in ["root.next", "manifest.next", "head.next"] { + let path = root.join("retention").join(stage); + if path.exists() { + return Err(format!("retained publication stage {stage} remained visible").into()); + } + } + Ok(()) +} + +fn migrated_witness(root: &Path) -> io::Result)>> { + let mut witness = Vec::new(); + for name in ["HEAD", "FORMAT", "migration.intent", "migration.receipt"] { + witness.push((PathBuf::from(name), fs::read(root.join(name))?)); + } + for pool in ["segments", "catalogs"] { + for entry in fs::read_dir(root.join(pool))? { + let path = entry?.path(); + let bytes = fs::read(&path)?; + witness.push((path, bytes)); + } + } + witness.sort(); + Ok(witness) +} + +fn retention_witness(root: &Path) -> io::Result)>> { + let mut witness = BTreeSet::new(); + collect(&root.join("retention"), &mut witness)?; + Ok(witness) +} + +fn collect(directory: &Path, witness: &mut BTreeSet<(OsString, Vec)>) -> io::Result<()> { + for entry in fs::read_dir(directory)? { + let entry = entry?; + let path = entry.path(); + if entry.file_type()?.is_dir() { + collect(&path, witness)?; + } else { + witness.insert((path.into_os_string(), fs::read(entry.path())?)); + } + } + Ok(()) +} + +fn hex(bytes: &[u8; 32]) -> String { + use std::fmt::Write as _; + bytes.iter().fold(String::new(), |mut rendered, byte| { + let _ = write!(rendered, "{byte:02x}"); + rendered + }) +} diff --git a/src/adapters/retention/filesystem_retention_test_fixture.rs b/src/adapters/retention/filesystem_retention_test_fixture.rs new file mode 100644 index 0000000..6120339 --- /dev/null +++ b/src/adapters/retention/filesystem_retention_test_fixture.rs @@ -0,0 +1,100 @@ +//! This test module owns one migrated version-2 retention publication fixture. + +use std::error::Error; +use std::fs; + +use super::filesystem_retention_authority::FilesystemRetentionPublicationAuthority; +use crate::LayoutEntryLimit; +use crate::adapters::filesystem_test_sandbox::TestDirectory; +use crate::adapters::test_support::decode_hex; +use crate::adapters::{ + AdmittedCatalog, AdmittedSegment, CatalogSnapshot, ChecksummedCatalog, + ChecksummedPublicationHead, FilesystemPlatformAdmission, FilesystemStoreMigrationAuthority, + SegmentReadPolicy, SegmentRecordLimit, +}; +use crate::execute_store_migration; + +/// Frozen canonical generation-one root. +pub(super) const ROOT_HEX: &str = + include_str!("../../../conformance/segment-store/v2/one-anchor-root.hex"); +/// Frozen canonical generation-one manifest. +pub(super) const MANIFEST_HEX: &str = + include_str!("../../../conformance/segment-store/v2/one-root-manifest.hex"); +/// Frozen canonical generation-one retention head. +pub(super) const HEAD_HEX: &str = + include_str!("../../../conformance/segment-store/v2/one-root-head.hex"); + +const SEGMENT_HEX: &str = + include_str!("../../../conformance/segment-store/v1/one-zero-bundle-segment.hex"); +const CATALOG_HEX: &str = + include_str!("../../../conformance/segment-store/v1/one-zero-bundle-catalog.hex"); +const CATALOG_HEAD_HEX: &str = + include_str!("../../../conformance/segment-store/v1/one-zero-bundle-head.hex"); + +const SEGMENT_NAME: &str = "221f6745cd8a5221c9a87c3707593608479282b54a4a74d0e753fd76f70e8db2.seg"; +const CATALOG_NAME: &str = + "0000000000000001-0b7cad1b6de663d34beacbc214db7497f2e36ab6b08dfbd5febbc8d06a418811.cat"; + +/// Builds one migrated version-2 store and pins its retention authority. +/// +/// The fixture publishes the exact bundle version-1 corpus, executes the +/// complete forward migration, releases writer authority, then reopens the +/// admitted root for retention publication. +pub(super) fn open_authority( + name: &str, +) -> Result<(TestDirectory, FilesystemRetentionPublicationAuthority), Box> { + let sandbox = migrated_store(name)?; + let admission = + FilesystemPlatformAdmission::reopen_version_two_unchecked_for_tests(sandbox.path())?; + let authority = FilesystemRetentionPublicationAuthority::open(admission)?; + Ok((sandbox, authority)) +} + +/// Decodes one LF-terminated lowercase hexadecimal conformance fixture. +pub(super) fn fixture(hex: &str) -> Result, Box> { + decode_hex(hex.strip_suffix('\n').ok_or("fixture must end in one LF")?).map_err(Into::into) +} + +/// Runs one operation against the frozen bundle catalog snapshot. +pub(super) fn with_snapshot( + operation: impl FnOnce(&CatalogSnapshot<'_, '_, '_>) -> T, +) -> Result> { + let segment_bytes = fixture(SEGMENT_HEX)?; + let catalog_bytes = fixture(CATALOG_HEX)?; + let head_bytes = fixture(CATALOG_HEAD_HEX)?; + let segment = AdmittedSegment::decode(&segment_bytes, maximum_policy())?; + let segments = [segment]; + let catalog: AdmittedCatalog<'_, '_> = + ChecksummedCatalog::decode(&catalog_bytes)?.admit(&segments)?; + let head = ChecksummedPublicationHead::decode(&head_bytes)?; + let snapshot = head.admit(catalog)?; + Ok(operation(&snapshot)) +} + +fn migrated_store(name: &str) -> Result> { + let sandbox = TestDirectory::create(name)?; + let admission = FilesystemPlatformAdmission::initialize_unchecked_for_tests(sandbox.path())?; + write_version_one(&sandbox)?; + let mut authority = FilesystemStoreMigrationAuthority::open(admission, maximum_policy())?; + let intent = authority.observe_intent()?; + let _receipt = execute_store_migration(&mut authority, &intent)?; + drop(authority); + Ok(sandbox) +} + +fn write_version_one(sandbox: &TestDirectory) -> Result<(), Box> { + fs::write( + sandbox.path().join("segments").join(SEGMENT_NAME), + fixture(SEGMENT_HEX)?, + )?; + fs::write( + sandbox.path().join("catalogs").join(CATALOG_NAME), + fixture(CATALOG_HEX)?, + )?; + fs::write(sandbox.path().join("HEAD"), fixture(CATALOG_HEAD_HEX)?)?; + Ok(()) +} + +const fn maximum_policy() -> SegmentReadPolicy { + SegmentReadPolicy::new(SegmentRecordLimit::MAXIMUM, LayoutEntryLimit::MAXIMUM) +} diff --git a/src/lib.rs b/src/lib.rs index 83570f1..b04c9ed 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -135,15 +135,17 @@ pub use adapters::{ pub use adapters::{ AdmittedRetentionManifest, AdmittedRetentionRoot, CanonicalRetentionHead, CanonicalRetentionManifest, CanonicalRetentionRoot, ChecksummedRetentionHead, - PreparedRetentionPublication, RetentionClosureVerificationError, RetentionHeadDecodeError, - RetentionManifestDecodeError, RetentionManifestEncodeError, RetentionNamespaceAdmission, - RetentionPublicationError, RetentionPublicationOutcome, RetentionPublicationPhase, - RetentionPublicationPreparation, RetentionPublicationPreparationError, - RetentionPublicationReceipt, RetentionPublicationStorage, RetentionRootDecodeError, - RetentionRootEncodeError, RetentionTransitionDisposition, RetentionTransitionError, - RetentionTransitionPreflight, RetentionTransitionPreflightError, RetentionTransitionReadiness, - VerifiedRetentionClosure, execute_retention_publication, plan_retention_transition, - preflight_retention_transition, prepare_retention_publication, verify_retention_closure, + FilesystemRetentionAuthorityError, FilesystemRetentionPublicationAuthority, + PreparedRetentionPublication, RetentionAuthorityDirectory, RetentionClosureVerificationError, + RetentionHeadDecodeError, RetentionManifestDecodeError, RetentionManifestEncodeError, + RetentionNamespaceAdmission, RetentionPublicationError, RetentionPublicationOutcome, + RetentionPublicationPhase, RetentionPublicationPreparation, + RetentionPublicationPreparationError, RetentionPublicationReceipt, RetentionPublicationStorage, + RetentionRootDecodeError, RetentionRootEncodeError, RetentionTransitionDisposition, + RetentionTransitionError, RetentionTransitionPreflight, RetentionTransitionPreflightError, + RetentionTransitionReadiness, VerifiedRetentionClosure, execute_retention_publication, + plan_retention_transition, preflight_retention_transition, prepare_retention_publication, + verify_retention_closure, }; pub use blob::{ BlobHashError, BlobHasher, BlobId, BlobLength, BlobReadError, ByteLength, ByteOffset, From 855db515b430f730f465e6d8311a6d69f810d7e3 Mon Sep 17 00:00:00 2001 From: James Ross Date: Sun, 6 Sep 2026 22:29:12 -0700 Subject: [PATCH 054/111] Rewrite README as a front door Replace the status-ledger prose with a README that does the job the Documentation Standard assigns it: what Keep is, its core law, its design boundary, current status, and links to deeper docs. The previous file carried 105 lines of commit-by-commit status inside a 73-line paragraph, introduced roughly forty type names without scaffolding, and contradicted itself: lines 221-224 described catalog publication and restart recovery as future milestones that lines 54-120 had already documented as implemented. It also still listed filesystem retention execution as planned after c262a3fb implemented it. The new file adds a why, a plain list of what is guaranteed today, a table of what is not yet done with issue links, a four-layer picture, a store-and-read example whose calls are copied from passing tests, and a where-to-go-next table. Requirement status now routes to the segment-store-v2 ledger instead of being restated here. Every phrase asserted by tests/range_read_contract.rs, tests/reference_store_contract.rs, and the xtask documentation contracts is preserved. Refs #69 --- README.md | 408 +++++++++++++++++++++++------------------------------- 1 file changed, 176 insertions(+), 232 deletions(-) diff --git a/README.md b/README.md index d8bc65e..defd12d 100644 --- a/README.md +++ b/README.md @@ -3,253 +3,197 @@ **Correctness-first content-addressed storage.** > For a given content identity, Keep must return exactly the bytes named by -> that identity—or refuse. - -The [authenticated reconstruction contract](docs/invariants/authenticated-reconstruction/README.md) -defines the proof scopes, output-failure rule, receipt posture, and precise -limits of that promise. - -Keep is a standalone Rust library for durable, content-addressed storage. It is -intended to provide streaming ingestion, content-defined chunking, physical -deduplication, exact range reads, explicit retention, integrity verification, -crash recovery, and garbage collection without relying on Git or subprocesses -in the storage path. - -## Status - -Keep exposes strict, versioned `BlobId`, `ChunkId`, `StorageProfileId`, and -`LayoutId` coordinates. It implements the frozen `fastcdc-64k-v1` detector and -the canonical `keep.flat-chunks/v1` layout codec with language-neutral golden -and mutation corpora. - -The public -[non-durable reference CAS](docs/architecture/reference-store/README.md) -provides capacity-bounded blocking ingestion, identity-based chunk -deduplication, an explicit staged-to-visible transition, authenticated -whole-blob reconstruction, and authenticated exact byte-range reads. -Reconstruction verifies every chunk, replays the registered storage profile, -and verifies the complete named `BlobId` before writing any bytes. Range reads -load only the minimal overlapping chunks and state their narrower verification -claim explicitly. - -The public `keep.segment-store/v1` boundary provides exact segment, record, -seal, catalog, and publication-head codecs plus explicit immutable-segment and -catalog-generation transitions. `StagedSegment` writes only content-admitted -chunk or layout records, while `AdmittedSegment` exposes payloads only after -complete framing, checksum, logical-identity, duplicate, and physical-digest -verification. A platform-admitted `FilesystemCatalogPublisher` exclusively -creates the fixed `current.seg` stage without truncating existing evidence, -and the `FilesystemSegmentStage` lifetime keeps that writer authority borrowed -until the writable stage closes. Publisher construction consumes an -unforgeable `FilesystemPlatformAdmission`. On Linux, its public initializer -admits only one writable, non-casefolded ext4 store profile, requires every -existing protocol directory to share the root's filesystem and mount identity, -refuses unknown, aliased, or foreign namespace entries before mutation, creates -or verifies the canonical `writer.lock`, `staging`, `segments`, and `catalogs` -shape, and returns only after root synchronization with the writer lock -retained. After publication, `FilesystemPlatformAdmission::reopen` reacquires -the existing writer lock without mutation and requires that exact initialized -shape plus a regular `HEAD` before returning new publisher authority. - -`FilesystemCatalogPublisher` retains one kernel-managed writer lock and pinned -root, staging, segment-pool, and catalog-pool capabilities for the complete -blocking publication. It reopens and verifies synchronized stages, uses -no-replacement immutable-pool links, synchronizes every required file and -directory, verifies the complete `head.next` view, atomically replaces `HEAD`, -and returns a receipt only after root synchronization. New filesystem segment -publication requires `FilesystemCatalogPublisher::select_segment` to consume -the sealed writable stage, prove that this publisher created it, and bind its -synchronized metadata to exact admitted bytes. A storage-agnostic -`ClosedSegment` receipt alone cannot authorize a retained filesystem stage. -`FilesystemCatalogSnapshot` follows only the exact checksummed head, catalog, -and segment coordinates and retains caller-bounded immutable bytes for pinned -logical reads. - -The reference CAS is executable evidence for M2 storage laws, not a durable -backend. Its committed state is process memory; process death loses it all. -The durable boundary can initialize or reopen and platform-admit a store only -under the documented Linux ext4 contract. Acquiring `FilesystemWriterLock` -alone cannot construct a filesystem publisher. Ambiguous crash states remain -explicit recovery work. An absent `HEAD` is admitted for first publication -only when both immutable pools are empty. The public storage-independent -recovery inventory counts all four protocol namespaces before retaining names, -applies a configurable ceiling no greater than 2,097,152 entries, and returns -duplicate-free deterministic raw name order. -`FilesystemRecoveryInventoryReader` implements that contract with pinned, -no-follow namespace capabilities and pre/post identity verification on the -admitted Linux ext4 profile. Its bounded stage-fingerprint operation opens -fixed stages relative to those capabilities, refuses links and nonregular -files, and verifies entry identity and length after reading. Complete -caller-supplied segment-stage bytes can be classified as a reusable prefix, -complete admitted segment, or exact truncation only while every available -fixed-framing byte remains canonical. Catalog and next-head stages apply the -same prefix rule before distinguishing truncation from complete canonical -bytes. -Materialized bytes enter read-only semantic assessment only after their stage, -length, and recomputed fingerprint match prior observation evidence. -An exact reusable segment assessment can authorize storage-independent -continuation: the executor consumes writer authority, re-admits the complete -bounded prefix, rebuilds digest and duplicate-identity state, and returns the -ordinary append-only stage without rewriting admitted bytes. -`FilesystemRecoverySegmentResumer` implements that contract with pinned -namespaces and writer authority, no-follow read-write reopening, exact bounded -materialization, final streamed fingerprint plus entry and namespace -revalidation, and an append position equal to the admitted prefix length. - -Exact truncation assessments can authorize durable, evidence-bound discard. -Complete segment and catalog assessments can authorize verified immutable-pool -completion through `FilesystemRecoveryStageCompleter`; its receipt proves a -valid orphan, not reachability. A complete `head.next` and its transitive -`CatalogSnapshot` can authorize storage-independent finalization only when the -candidate is generation one over an uninitialized root or the exact successor -of the expected current snapshot. The executor distinguishes first -finalization from an already-finalized retry and returns only after root -synchronization. `FilesystemRecoveryNextHeadFinalizer` retains pinned writer -authority, reconstructs the complete current and candidate views without -following links, verifies namespace and stage identity, synchronizes and -reverifies the exact candidate, atomically replaces `HEAD`, and synchronizes -the root. An already-finalized retry requires `head.next` to be absent. -The repository-owned process-death matrix executes all 105 -`KEEP-CRASH-001`–`KEEP-CRASH-035` before/during/after coordinates in isolated -process groups. Crash children execute the production initialization, -segment-writing, catalog-publication, and recovery-discard protocols through -fault-injecting port decorators; they do not synthesize the target namespace. -Restart verification compares the exact Golden File Worldline namespace and -bytes, checks hard-link identity and writer-lock release, runs the production -recovery classifiers and immutable-artifact admission, and reconstructs the -exact published generation and visible one-zero chunk when `HEAD` exists. This -matrix proves application process-death behavior; it does not simulate host -power loss. Canonical version-2 store-format marker encoding and exact -migration-intent and completion-receipt admission are implemented. Version-2 -retention values; canonical in-memory root, global manifest, and retention-head -codecs; storage-independent expected-state transition planning; deterministic -bounded closure verification against a pinned catalog; a combined transition -preflight proof; and the exact 17-phase publication vocabulary with a blocking -storage capability port are implemented. Private-field proofs retain every -receipt coordinate. Ordered storage-port orchestration revalidates current -authority, executes all 17 durability phases, and returns a consequential -complete-coordinate receipt. Writer-locked filesystem authority now implements -the 21-phase fresh migration storage protocol: it exclusively publishes all -three fixed records, admits and synchronizes the exact version-2 namespace, -reopens every canonical view, and retains byte-and-inode evidence through final -verification without changing version-1 immutable bytes. Partial-prefix -migration recovery, filesystem retention execution, immutable reader -snapshots, compaction, and garbage collection remain planned; version 2 is not -yet an admitted restart-safe production store. -Presence in the reference CAS does not claim durable retention or crash -recovery. - -Run the complete debug-profile matrix: - -```bash -cargo xtask durability-crash-matrix +> that identity — or refuse. + +Everything else in this repository exists to make that sentence true under +power loss, process death, corrupted disks, and byte-identical files swapped +in underneath it. The +[authenticated reconstruction contract](docs/invariants/authenticated-reconstruction/README.md) +states the promise precisely, including its limits. + +Keep is a standalone Rust library. It is the storage layer beneath +[Graft](https://github.com/flyingrobots/graft) and +[Echo](https://github.com/flyingrobots/echo), and it is built so that neither +of them — nor anything else — can weaken its guarantee by leaning on it. + +## Why it exists + +Most storage answers *"did you save my bytes?"* with a return code and a +shrug. The write returned zero; the file is probably on disk; if the machine +lost power between the write and the flush, you find out later. + +Keep refuses to shrug. If it cannot prove it holds the exact bytes a name +refers to, it fails loudly instead of returning a plausible approximation. +That posture is called **fail-closed**, and it is much harder than it sounds: + +- a disk that returns a corrupted block does not announce itself; +- a process killed mid-update leaves state that *looks* finished; +- a byte-for-byte identical file substituted at the same path reads as the + original. + +Keep is required to refuse all three, before mutating anything. + +## What it guarantees today + +- **Exact identity.** `BlobId`, `ChunkId`, `LayoutId`, and + `StorageProfileId` are strict, versioned, and canonically encoded, with + language-neutral golden and mutation corpora. +- **Deterministic chunking.** The frozen `fastcdc-64k-v1` profile splits + input by content, so an insertion near the front of a file leaves the + chunks after it untouched and deduplicated. +- **Authenticated reads.** Whole-blob reconstruction verifies every chunk, + replays the storage profile, and verifies the complete `BlobId` before a + single byte reaches the caller. It also provides + authenticated exact byte-range reads that load only the overlapping chunks + and state their narrower claim explicitly. +- **Durable version-1 segment store.** `StagedSegment` writes only + content-admitted records; `AdmittedSegment` exposes payloads only after + complete framing, checksum, and identity verification. Immutable segments, + generation-versioned catalogs, and a fixed-width `HEAD` are published + through an ordered protocol whose every step is a named crash point. + Platform admission is Linux ext4, non-casefolded, one writer. +- **Proven restart recovery for version 1.** The crash matrix kills real + writer processes at 105 before/during/after coordinates + (`KEEP-CRASH-001`–`035`) and verifies the store lands in exactly one + documented lawful state each time. +- **Version-2 retention and migration, forward path.** Explicit retention + roots, deterministic closure verification, a one-way 21-phase migration, + and a 17-phase retention publication — all with production filesystem + writers, all preserving every version-1 byte. + +## What it does not do yet + +Version 2 writes correctly from a clean start. It cannot yet pick up the +pieces if it dies partway through. Until it can, **version 1 is the only +store admitted for production.** + +| Gap | Tracked | +| --- | --- | +| Restart recovery for retention publication and migration | [#19](https://github.com/flyingrobots/keep/issues/19) | +| Reader fence binding one consistent catalog + retention snapshot | [#19](https://github.com/flyingrobots/keep/issues/19) | +| Precise verification reports at explicit depths | [#20](https://github.com/flyingrobots/keep/issues/20) | +| Garbage collection and identity-preserving compaction | [#21](https://github.com/flyingrobots/keep/issues/21) | +| Bounded production ingestion through the durable store | [#82](https://github.com/flyingrobots/keep/issues/82) | +| Encrypted representations | [#86](https://github.com/flyingrobots/keep/issues/86) | + +Keep also does not claim secure deletion. Releasing a retention root +publishes a successor generation; it does not assert that bytes were +destroyed. + +The authoritative status of every requirement, with the test that proves it, +is the ledger in +[`docs/formats/segment-store-v2/requirements.md`](docs/formats/segment-store-v2/requirements.md). +Its first rule: *a planned case is not evidence.* + +## How it works + +Four layers, each named separately so the physical layer can change without +the logical name moving: + +```text + identity BlobId · ChunkId · LayoutId what the bytes ARE + chunks fastcdc-64k-v1 · flat-chunks/v1 how they are split and reassembled + segments immutable segments · catalogs · HEAD where they physically live + retention namespaces · roots · manifests what must remain reconstructible ``` -CI also runs the command through an optimized `xtask` build. +The core protocol logic knows nothing about filesystems. It is written +against capability traits — `RetentionPublicationStorage`, for example, names +seventeen durability capabilities and nothing more. Filesystem behaviour lives +in adapters that implement those traits. The ordering laws are proved +exhaustively against fault-injecting fakes, and separately against real disks. + +Every durable change runs as a numbered phase sequence. Files are staged, +synchronised, hard-linked into place without replacement, and only then is a +fixed-width head replaced atomically. Cleanup happens after the commit, never +before. Staged files are verified by device and inode identity at every +transition, so a substituted byte-identical file refuses. + +The core holds no clock, no caller identity, no paths, and no application +policy. Retention proves a *physical reconstruction* claim only — never what +the content means, who owns it, or whether deleting it is legally safe. + +## Try it + +Keep is `0.0.0` and unpublished; build from source. The in-memory +[non-durable reference CAS](docs/architecture/reference-store/README.md) is +executable evidence for the storage laws, not a durable backend — process +death loses everything in it. ```rust -use keep::BlobId; +use std::io::Cursor; +use keep::{LayoutEntryLimit, ReferenceStore, ReferenceStoreCapacity}; + +let mut store = ReferenceStore::new(ReferenceStoreCapacity::new(1_048_576)); -let identity = BlobId::hash_bytes(b"exact bytes")?; -let canonical = identity.to_string(); -assert_eq!(canonical.parse::()?, identity); +// Stage: chunk, hash, and hold the bytes without making them visible. +let mut source = Cursor::new(b"exact bytes, or nothing"); +let staged = store.stage(&mut source, LayoutEntryLimit::MAXIMUM)?; + +// Commit: the explicit staged-to-visible transition. +let published = staged.commit(&mut store)?; + +// Read back: every chunk verified, the complete BlobId verified, then bytes. +let mut output = Vec::new(); +store.reconstruct(published.target(), &mut output)?; +assert_eq!(output, b"exact bytes, or nothing"); # Ok::<(), Box>(()) ``` -## Design boundary +Run the full gate suite the way CI does: -Keep owns physical content storage: +```bash +cargo test --workspace --all-features --locked +cargo xtask durability-crash-matrix # kills real writer processes +cargo xtask golden-file-worldline-check +cargo xtask conformance-check +``` -- exact byte identity; -- chunking and physical representation; -- streaming and range reads; -- retention roots and storage generations; -- verification, recovery, compaction, and garbage collection; -- optional storage encryption. +## Design boundary -Keep does not own application semantics. In particular, the core library must -remain independent of Echo, Git, Graft, WARP, command-line interfaces, and -application policy. +Keep owns physical content storage: exact byte identity; chunking and +physical representation; streaming and range reads; retention roots and +storage generations; verification, recovery, compaction, and garbage +collection; optional storage encryption. -An application may give stored bytes causal meaning, authority, provenance, or -publication status. Keep reports only what its physical evidence can support. +Keep does not own application semantics. The core stays independent of Echo, +Git, Graft, WARP, command-line interfaces, and application policy. An +application may give stored bytes causal meaning, authority, or provenance; +Keep reports only what its physical evidence supports. ## Engineering standard -Development is governed by the normative -[Keep Rust Engineering Standard](docs/Rust%20Standards.md). Correctness, +Development follows the normative +[Keep Rust Engineering Standard](docs/Rust%20Standards.md): correctness, recoverability, auditability, and maintainability outrank performance and -convenience. - -Documentation is governed by the -[Keep Documentation Standard](docs/Documentation%20Standards.md), which maps -reader tasks onto Keep's architecture, invariant, format, and recovery -corpus. - -The initial implementation will use: - -- stable Rust 1.96.0; -- Rust edition 2024; -- one writer and many readers unless a stronger concurrency model is designed; -- synchronous core APIs until a demonstrated consumer requires otherwise; -- versioned, canonical, independently testable durable formats; -- no unsafe Rust in Keep-owned version-1 crates; dependency unsafe requires an - explicit review and cannot alter canonical identity. - -## Golden File Worldline - -The first executable vertical is split into deliberately narrow milestones. M1 -proves that exact finite logical bytes have one canonical versioned identity, -that calculation is invariant to tested input partitioning, that malformed or -unsupported identity encodings are refused precisely, and that a bounded -reference model returns exactly the bytes named by an admitted identity or -refuses. - -The complete Golden File Worldline is planned to demonstrate that Keep can: - -1. ingest exact logical bytes; -2. retain and recover multiple nearby versions; -3. reuse stable chunks after an early insertion; -4. read an exact byte range without materializing the whole blob; -5. refuse corrupted or ambiguous storage; -6. recover to a documented lawful state after interruption. - -Items 1 through 6 describe the multi-milestone destination. M1 establishes the -canonical identity boundary. M2 now provides deterministic chunk detection, -canonical layouts, capacity-bounded ingestion, and authenticated whole-blob -reconstruction and minimal exact range reads through the non-durable reference -CAS. M3 now provides exact immutable-segment construction and verified -admission. Catalog publication, durable retention and recovery, nearby-version -workflows, complete namespace verification, and restart recovery remain future -milestones. - -See the [M1 conformance contract](docs/conformance/golden-file-worldline.md), -the [CDC profile corpus](conformance/cdc-profile/v1/README.md), the -[chunk identity invariant](docs/invariants/chunk-identity/README.md), the -[Flat Chunk Layout v1 specification](docs/formats/flat-chunk-layout-v1/README.md), -the [layout corpus](conformance/layout/v1/README.md), and the -[reference CAS contract](docs/architecture/reference-store/README.md) for the -implemented proof boundaries and explicit nonclaims. The -[Durable Segment Store v1 specification](docs/formats/segment-store-v1/README.md) -and [segment-store corpus](conformance/segment-store/v1/README.md) define the -implemented segment boundary and the still-planned publication and recovery -work. The -[streaming CAS baseline protocol](docs/benchmarks/streaming-cas-baseline-v1/README.md) -defines reproducible performance evidence without treating measurements as -correctness proof or weakening verification. - -## Contributing - -See [CONTRIBUTING.md](CONTRIBUTING.md). Every change must preserve Keep's core -law and satisfy the repository's formatting, linting, testing, and review -standards. - -## Security - -Please report vulnerabilities using the process in [SECURITY.md](SECURITY.md). -Do not include plaintext content, keys, or other sensitive material in a public -issue. - -## License +convenience. Stable Rust 1.96, edition 2024, `#![forbid(unsafe_code)]`, +one writer and many readers, synchronous core APIs, versioned canonical +formats. Every pedantic lint is an error. Modules are capped at 500 lines +and functions at 60. + +Documentation follows the +[Keep Documentation Standard](docs/Documentation%20Standards.md). Each page +has one job; this one is the front door. + +## Where to go next + +| You want to… | Read | +| --- | --- | +| Understand what is proved and what is not | [`docs/invariants/`](docs/invariants/) | +| Read the byte-level formats | [`docs/formats/`](docs/formats/) | +| See the architecture and port boundaries | [`docs/architecture/`](docs/architecture/) | +| Follow the crash and recovery rules | [`segment-store-v1/recovery.md`](docs/formats/segment-store-v1/recovery.md) · [`segment-store-v2/recovery.md`](docs/formats/segment-store-v2/recovery.md) | +| Check reproducible performance evidence | [`docs/benchmarks/`](docs/benchmarks/) | +| Run the language-neutral corpora | [`conformance/`](conformance/) | +| See what changed | [`CHANGELOG.md`](CHANGELOG.md) | + +## Contributing, security, license + +See [CONTRIBUTING.md](CONTRIBUTING.md). Every change must preserve the core +law and pass the repository's formatting, linting, testing, and documentation +gates. + +Report vulnerabilities through [SECURITY.md](SECURITY.md). Do not include +plaintext content, keys, or sensitive paths in a public issue. Licensed under the [Apache License 2.0](LICENSE). From d7518a8edd8e88c0e4a1bd8e975f5729055c461f Mon Sep 17 00:00:00 2001 From: James Ross Date: Sun, 6 Sep 2026 22:33:25 -0700 Subject: [PATCH 055/111] Replace README architecture block with a Mermaid diagram The text block was a three-column table with no flow and misaligned columns. The Mermaid flowchart shows the three layers, marks down-arrows as 'stored as' and the dashed up-arrow as the closure walk that verifies retention, and highlights HEAD and retention/HEAD as the only two files Keep ever replaces in place. Validated with mermaid-cli before commit. --- README.md | 47 +++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 39 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index defd12d..72c6084 100644 --- a/README.md +++ b/README.md @@ -87,14 +87,45 @@ Its first rule: *a planned case is not evidence.* ## How it works -Four layers, each named separately so the physical layer can change without -the logical name moving: - -```text - identity BlobId · ChunkId · LayoutId what the bytes ARE - chunks fastcdc-64k-v1 · flat-chunks/v1 how they are split and reassembled - segments immutable segments · catalogs · HEAD where they physically live - retention namespaces · roots · manifests what must remain reconstructible +Three layers. Names point down into storage; proofs point back up. Every +physical thing is named by a hash of what it contains, and every retention +claim is verified by walking down to the bytes. Only two files are ever +replaced in place: + +```mermaid +flowchart TB + IN([bytes in]) --> CHUNK + + subgraph LOGICAL["Logical — names, never locations"] + direction LR + CHUNK["chunk
fastcdc-64k-v1"] --> CID["ChunkId"] + CID --> ASM["assemble
flat-chunks/v1"] --> LID["LayoutId"] + LID --> BID["BlobId
the whole payload"] + end + + subgraph PHYSICAL["Physical — where bytes live"] + direction LR + HEAD["HEAD · 128 B
the only file v1 ever replaces"] + CAT["catalog @ generation N
identity → location"] + SEG["immutable segments
sealed, never edited"] + HEAD --> CAT --> SEG + end + + subgraph RETENTION["Retention — what must survive"] + direction LR + RHEAD["retention/HEAD · 144 B
the only file v2 adds to that list"] + MAN["manifest
namespace → root generation"] + ROOT["root
anchors are BlobIds, generation-checked"] + RHEAD --> MAN --> ROOT + end + + CID -- "stored as records in" --> SEG + LID -- "stored as records in" --> SEG + ROOT -. "closure walk proves every anchor reconstructs" .-> BID + BID --> OUT([exact bytes out — or a refusal]) + + classDef mutable stroke-width:3px + class HEAD,RHEAD mutable ``` The core protocol logic knows nothing about filesystems. It is written From 06072f44ef4f514826c76240a3c89afba49b4d41 Mon Sep 17 00:00:00 2001 From: James Ross Date: Sun, 6 Sep 2026 22:36:49 -0700 Subject: [PATCH 056/111] Tighten README diagram titles Single-word subgraph titles: the longer ones clipped in mermaid-cli, and the taglines already live in the surrounding prose. Removed the inner direction hints that cross-subgraph edges override anyway. --- README.md | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 72c6084..94a0550 100644 --- a/README.md +++ b/README.md @@ -96,23 +96,20 @@ replaced in place: flowchart TB IN([bytes in]) --> CHUNK - subgraph LOGICAL["Logical — names, never locations"] - direction LR + subgraph LOGICAL["Logical"] CHUNK["chunk
fastcdc-64k-v1"] --> CID["ChunkId"] CID --> ASM["assemble
flat-chunks/v1"] --> LID["LayoutId"] LID --> BID["BlobId
the whole payload"] end - subgraph PHYSICAL["Physical — where bytes live"] - direction LR + subgraph PHYSICAL["Physical"] HEAD["HEAD · 128 B
the only file v1 ever replaces"] CAT["catalog @ generation N
identity → location"] SEG["immutable segments
sealed, never edited"] HEAD --> CAT --> SEG end - subgraph RETENTION["Retention — what must survive"] - direction LR + subgraph RETENTION["Retention"] RHEAD["retention/HEAD · 144 B
the only file v2 adds to that list"] MAN["manifest
namespace → root generation"] ROOT["root
anchors are BlobIds, generation-checked"] From a55ad659a6a182ea7df5ee6b8661cca1d707062d Mon Sep 17 00:00:00 2001 From: James Ross Date: Sun, 6 Sep 2026 23:32:23 -0700 Subject: [PATCH 057/111] Publish retention successors against the observed head Complete plan item 5 of #78: production filesystem retention publication now covers successor generations and the superseded-candidate refusal that makes exact retry idempotent only while its successor remains current. `FilesystemRetentionPublicationAuthority::observe_current` reopens `retention/HEAD` without following links, reads the manifest it selects from the immutable pool at the head's declared length, decodes both, and refuses unless the manifest's canonical digest and generation match the head. Callers decode the returned bytes to plan the next transition. Current-state verification now has four lawful outcomes. An absent head admits only an `Absent` expectation. A head equal to the prepared successor is `AlreadyCommitted` with zero mutation. A head that the prepared successor names as its exact predecessor at the next liveness generation admits `Publish`. Any other head means the candidate is superseded and refuses before mutation. Three new filesystem laws: successor publication over an existing head lands the exact successor in the existing namespace and leaves every generation-one pool entry byte-identical; a stale generation-one candidate refuses with an unchanged retention witness once generation two is current; and a successor prepared against an absent head refuses. KEEP-RETENTION-004 and KEEP-RETENTION-009 move to Implemented. Refs #19 --- CHANGELOG.md | 5 +- docs/formats/segment-store-v2/requirements.md | 4 +- docs/formats/segment-store-v2/retention.md | 5 +- src/adapters/retention.rs | 4 + .../filesystem_retention_authority.rs | 15 ++ .../retention/filesystem_retention_current.rs | 149 ++++++++++++++++++ .../retention/filesystem_retention_storage.rs | 51 +----- .../filesystem_retention_storage_tests.rs | 102 ++---------- .../filesystem_retention_successor_tests.rs | 120 ++++++++++++++ .../filesystem_retention_test_fixture.rs | 125 ++++++++++++++- src/lib.rs | 8 +- 11 files changed, 445 insertions(+), 143 deletions(-) create mode 100644 src/adapters/retention/filesystem_retention_current.rs create mode 100644 src/adapters/retention/filesystem_retention_successor_tests.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index a3df28a..2799e76 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -30,7 +30,10 @@ after its public API and format compatibility policies are established. its receipt with zero retention mutation, and any retained stage refuses as recovery-required rather than being continued. Version-1 reopen now refuses a migrated root, and `admit_version_two` owns the separate version-2 namespace - boundary. + boundary. `observe_current` returns the published head and its cross-verified + pool manifest, and current-state verification admits a successor only when + the prepared head names the observed manifest as its exact predecessor at the + next liveness generation; a superseded candidate refuses with zero mutation. - Repository crash-matrix execution now terminates isolated writer process groups at all 105 canonical before/during/after coordinates, retains open writer and stage authority until termination, executes production diff --git a/docs/formats/segment-store-v2/requirements.md b/docs/formats/segment-store-v2/requirements.md index f624d5f..36ba382 100644 --- a/docs/formats/segment-store-v2/requirements.md +++ b/docs/formats/segment-store-v2/requirements.md @@ -12,12 +12,12 @@ case is not evidence. | `KEEP-RETENTION-001` | `RetentionNamespace`, `RootGeneration`, `LivenessGeneration`, profile coordinates, limits, anchors, and digests are validated typed values | `tests/retention_values.rs`, `tests/retention_root_encoding.rs`, and typed verified anchor-set evidence in `tests/retention_root_decoding.rs` | Implemented | | `KEEP-RETENTION-002` | Root, manifest, and head codecs implement the exact canonical grammars and fixed bounds | independent golden corpus plus `tests/retention_root_encoding.rs`, `tests/retention_root_decoding.rs`, `tests/retention_manifest_codec.rs`, and `tests/retention_head_codec.rs` | Implemented | | `KEEP-RETENTION-003` | Every structural field, truncation boundary, ordering law, duplicate, overflow, flag, reserved byte, digest, checksum, and trailing byte has a precise refusal | seeded `retention_format` fuzz target plus the root, manifest, and head corruption matrix; mutation coverage remains | In progress in #19 | -| `KEEP-RETENTION-004` | Retain and release compare expected and observed generations and publish exact successors only | unforgeable readiness and preflight proofs in `tests/retention_transition.rs` and `tests/retention_preflight.rs`; exact successor preparation and complete receipt evidence in `tests/retention_publication_preparation.rs` and `tests/retention_publication_execution.rs`; writer-locked forward filesystem publication in `filesystem_retention_storage_tests`; successor-generation filesystem evidence remains | In progress in #19 | +| `KEEP-RETENTION-004` | Retain and release compare expected and observed generations and publish exact successors only | unforgeable readiness and preflight proofs in `tests/retention_transition.rs` and `tests/retention_preflight.rs`; exact successor preparation and complete receipt evidence in `tests/retention_publication_preparation.rs` and `tests/retention_publication_execution.rs`; writer-locked initial filesystem publication in `filesystem_retention_storage_tests`; observed-head successor publication, exact predecessor binding, and absent-head refusal in `filesystem_retention_successor_tests` | Implemented | | `KEEP-RETENTION-005` | Closure derivation is deterministic, bounded, cycle-safe, fail-closed, and verifies complete blob reconstruction | exact accounting, reconstruction, adversarial-catalog, and exhaustive model laws in `tests/retention_closure.rs`; corrupt members refuse through the inherited segment-record admission laws and seeded `segment_format` fuzz target routed by `closure-corruption.md` | Implemented | | `KEEP-RETENTION-006` | Publication follows the exact ordered durability protocol, including new namespace-directory admission and retention of fixed-stage evidence until head commit, and returns only after cleanup synchronization | typed vocabulary and blocking port in `tests/retention_publication_phase.rs` and `tests/retention_publication_storage.rs`; ordered execution, conditional namespace sync, and all 17 exact storage-fault boundaries in `tests/retention_publication_execution.rs`; production 17-phase forward filesystem execution, exclusive staging, byte-equal inode-substitution refusal, and retained-stage recovery refusal in `filesystem_retention_storage_tests`; crash injection remains | In progress in #19 | | `KEEP-RETENTION-007` | Restart resolves every fixed-stage crash prefix to one documented lawful state or typed ambiguity | debug and release crash matrix | Planned in #19 | | `KEEP-RETENTION-008` | Readers double-collect catalog and retention heads and bind one complete catalog, manifest, and root-generation view under a `ReaderFence` | immutable snapshot and concurrency tests | Planned in #19 | -| `KEEP-RETENTION-009` | Exact already-committed retry is idempotent only while its successor remains current | byte-identical planning in `tests/retention_transition.rs`; authority-revalidated zero-mutation retry receipt in `tests/retention_publication_execution.rs`; exact already-committed filesystem retry with a byte-identical retention witness in `filesystem_retention_storage_tests`; superseded-successor filesystem refusal remains | In progress in #19 | +| `KEEP-RETENTION-009` | Exact already-committed retry is idempotent only while its successor remains current | byte-identical planning in `tests/retention_transition.rs`; authority-revalidated zero-mutation retry receipt in `tests/retention_publication_execution.rs`; exact already-committed filesystem retry with a byte-identical retention witness in `filesystem_retention_storage_tests`; superseded-candidate filesystem refusal with zero mutation in `filesystem_retention_successor_tests` | Implemented | | `KEEP-RETENTION-010` | Model operation sequences agree with a deterministic namespace-to-anchor-set map and never admit caller identity, paths, clocks, or application policy | model-based and source-architecture tests | Planned in #19 | diff --git a/docs/formats/segment-store-v2/retention.md b/docs/formats/segment-store-v2/retention.md index 745f265..238fc82 100644 --- a/docs/formats/segment-store-v2/retention.md +++ b/docs/formats/segment-store-v2/retention.md @@ -163,8 +163,9 @@ Names with alternate width, case, suffix, generation, or digest refuse. Keep implements root, manifest, and head codecs with a typed verified anchor-set digest, expected-state transition planning, deterministic closure verification, a blocking publication storage capability port, and ordered storage-port -orchestration. `FilesystemRetentionPublicationAuthority` executes that protocol -once; retained-stage recovery, reader fencing, and collection remain absent. +orchestration. `FilesystemRetentionPublicationAuthority` publishes initial and +successor generations against its observed head and refuses superseded +candidates and retained stages; recovery, fencing, and collection remain absent. ## Global retention manifest diff --git a/src/adapters/retention.rs b/src/adapters/retention.rs index cda32d4..d8441b2 100644 --- a/src/adapters/retention.rs +++ b/src/adapters/retention.rs @@ -15,12 +15,15 @@ mod closure_profile_error; mod closure_verifier; mod filesystem_retention_authority; mod filesystem_retention_authority_error; +mod filesystem_retention_current; mod filesystem_retention_pool_name; mod filesystem_retention_stage; mod filesystem_retention_storage; #[cfg(test)] mod filesystem_retention_storage_tests; #[cfg(test)] +mod filesystem_retention_successor_tests; +#[cfg(test)] mod filesystem_retention_test_fixture; mod head_decode_error; mod head_decode_error_display; @@ -78,6 +81,7 @@ pub use filesystem_retention_authority::FilesystemRetentionPublicationAuthority; pub use filesystem_retention_authority_error::{ FilesystemRetentionAuthorityError, RetentionAuthorityDirectory, }; +pub use filesystem_retention_current::ObservedRetentionState; pub use head_decode_error::RetentionHeadDecodeError; pub use manifest_decode_error::RetentionManifestDecodeError; pub use manifest_encode_error::RetentionManifestEncodeError; diff --git a/src/adapters/retention/filesystem_retention_authority.rs b/src/adapters/retention/filesystem_retention_authority.rs index 2469f2f..dca5a38 100644 --- a/src/adapters/retention/filesystem_retention_authority.rs +++ b/src/adapters/retention/filesystem_retention_authority.rs @@ -8,6 +8,7 @@ use cap_std::fs::Dir; use super::filesystem_retention_authority_error::{ FilesystemRetentionAuthorityError as Error, RetentionAuthorityDirectory as Directory, }; +use super::filesystem_retention_current::{self, ObservedRetentionState}; use super::filesystem_retention_pool_name as pool_name; use super::filesystem_retention_stage::{FilesystemRetentionStage, invalid_data}; use crate::adapters::{FilesystemPlatformAdmission, FilesystemWriterLock}; @@ -74,6 +75,20 @@ impl FilesystemRetentionPublicationAuthority { }) } + /// Observes the published retention head and the manifest it selects. + /// + /// Returns `None` when no retention head has been published. This + /// synchronous read performs no protocol mutation and does not consult + /// retained stages; callers decode the returned bytes to plan the next + /// transition, then let publication revalidate them under authority. + /// + /// # Errors + /// + /// Returns the exact open, kind, length, decode, or cross-check refusal. + pub fn observe_current(&self) -> io::Result> { + filesystem_retention_current::observe(&self.retention, &self.manifests) + } + pub(super) fn namespace(&self) -> io::Result<&Dir> { self.namespace .as_ref() diff --git a/src/adapters/retention/filesystem_retention_current.rs b/src/adapters/retention/filesystem_retention_current.rs new file mode 100644 index 0000000..db48205 --- /dev/null +++ b/src/adapters/retention/filesystem_retention_current.rs @@ -0,0 +1,149 @@ +//! This module owns exact observation of the current filesystem retention state. + +use std::io::{self, Read}; + +use cap_fs_ext::{FollowSymlinks, OpenOptionsFollowExt}; +use cap_std::fs::{Dir, OpenOptions}; + +use super::filesystem_retention_pool_name as pool_name; +use super::filesystem_retention_stage::invalid_data; +use super::{ + AdmittedRetentionManifest, ChecksummedRetentionHead, RetentionPublicationPreparation, + RetentionTransitionDisposition, +}; +use crate::RetentionGenerationExpectation; + +const HEAD_LENGTH: usize = 144; + +/// Exact bytes of one published retention head and the manifest it selects. +/// +/// Both records were reopened without following links, bounded by their +/// declared lengths, decoded, and cross-checked: the manifest's canonical +/// digest equals the digest the head names. Callers decode the bytes with +/// [`ChecksummedRetentionHead`] and [`AdmittedRetentionManifest`] to plan the +/// next transition. +#[must_use] +#[derive(Debug)] +pub struct ObservedRetentionState { + head: Box<[u8]>, + manifest: Box<[u8]>, +} + +impl ObservedRetentionState { + /// Returns the exact 144 published head bytes. + pub const fn head_bytes(&self) -> &[u8] { + &self.head + } + + /// Returns the exact bytes of the manifest the head selects. + pub const fn manifest_bytes(&self) -> &[u8] { + &self.manifest + } +} + +/// Reads and cross-verifies `retention/HEAD` and its selected manifest. +/// +/// Returns `None` only when no head is published. A present head that does +/// not decode, or whose manifest is missing, wrong-length, or names another +/// digest, refuses. +pub(super) fn observe( + retention: &Dir, + manifests: &Dir, +) -> io::Result> { + let Some(head) = read_exact_optional(retention, pool_name::HEAD, HEAD_LENGTH)? else { + return Ok(None); + }; + let decoded = ChecksummedRetentionHead::decode(&head) + .map_err(|_source| invalid_data("current retention head refused admission"))?; + let selected = decoded.head(); + let length = usize::try_from(selected.manifest_length().get()) + .map_err(|_source| invalid_data("current manifest length exceeded usize"))?; + let name = pool_name::manifest(selected.generation(), selected.manifest_digest()); + let manifest = read_exact_optional(manifests, &name, length)? + .ok_or_else(|| invalid_data("current retention head names an absent manifest"))?; + let admitted = AdmittedRetentionManifest::decode(&manifest) + .map_err(|_source| invalid_data("current retention manifest refused admission"))?; + if admitted.digest() != selected.manifest_digest() + || admitted.manifest().generation() != selected.generation() + { + return Err(invalid_data( + "current retention manifest disagreed with its head", + )); + } + Ok(Some(ObservedRetentionState { head, manifest })) +} + +/// Compares one preparation against the observed current state. +/// +/// An absent head admits only an `Absent` expectation. A present head that +/// equals the prepared successor is `AlreadyCommitted`. Otherwise the head +/// must be the exact predecessor the prepared successor names, or the +/// candidate is superseded and refuses. +pub(super) fn disposition( + preparation: &RetentionPublicationPreparation<'_>, + current: Option<&ObservedRetentionState>, +) -> io::Result { + let Some(current) = current else { + return match preparation.expected() { + RetentionGenerationExpectation::Absent => Ok(RetentionTransitionDisposition::Publish), + RetentionGenerationExpectation::Current(_) => Err(invalid_data( + "expected a current retention generation but no head is published", + )), + }; + }; + let head = ChecksummedRetentionHead::decode(current.head_bytes()) + .map_err(|_source| invalid_data("observed retention head refused admission"))?; + let head = head.head(); + let committed = ( + preparation.liveness_generation(), + preparation.manifest_digest(), + ); + if (head.generation(), head.manifest_digest()) == committed { + return Ok(RetentionTransitionDisposition::AlreadyCommitted); + } + let publication = preparation.publication().ok_or_else(|| { + invalid_data("already-committed retry is stale: another successor is current") + })?; + let prepared = ChecksummedRetentionHead::decode(publication.head().encoded()) + .map_err(|_source| invalid_data("prepared retention head refused admission"))?; + let expected_generation = head + .generation() + .successor() + .map_err(|_source| invalid_data("current liveness generation cannot advance"))?; + if prepared.head().predecessor() == Some(head.manifest_digest()) + && prepared.head().generation() == expected_generation + { + Ok(RetentionTransitionDisposition::Publish) + } else { + Err(invalid_data( + "current retention head is not the prepared predecessor; the candidate is superseded", + )) + } +} + +fn read_exact_optional( + directory: &Dir, + name: &str, + length: usize, +) -> io::Result>> { + let mut options = OpenOptions::new(); + options.read(true).follow(FollowSymlinks::No); + let mut file = match directory.open_with(name, &options) { + Ok(file) => file, + Err(source) if source.kind() == io::ErrorKind::NotFound => return Ok(None), + Err(source) => return Err(source), + }; + let expected_length = u64::try_from(length) + .map_err(|_source| invalid_data("retention record length exceeded u64"))?; + let metadata = file.metadata()?; + if !metadata.is_file() || metadata.len() != expected_length { + return Err(invalid_data("retention record kind or length disagreed")); + } + let mut bytes = vec![0_u8; length]; + file.read_exact(&mut bytes)?; + let mut trailing = [0_u8; 1]; + if file.read(&mut trailing)? != 0 { + return Err(invalid_data("retention record carried trailing bytes")); + } + Ok(Some(bytes.into_boxed_slice())) +} diff --git a/src/adapters/retention/filesystem_retention_storage.rs b/src/adapters/retention/filesystem_retention_storage.rs index 1cdf456..9492b65 100644 --- a/src/adapters/retention/filesystem_retention_storage.rs +++ b/src/adapters/retention/filesystem_retention_storage.rs @@ -1,22 +1,21 @@ //! This module owns forward filesystem retention publication execution. -use std::io::{self, Read}; +use std::io; -use cap_fs_ext::{DirExt, FollowSymlinks, OpenOptionsFollowExt}; -use cap_std::fs::{Dir, OpenOptions}; +use cap_fs_ext::DirExt; +use cap_std::fs::Dir; use super::filesystem_retention_authority::FilesystemRetentionPublicationAuthority; +use super::filesystem_retention_current; use super::filesystem_retention_pool_name as pool_name; use super::filesystem_retention_stage::{FilesystemRetentionStage, invalid_data}; use super::{ AdmittedRetentionRoot, CanonicalRetentionHead, CanonicalRetentionManifest, - ChecksummedRetentionHead, RetentionNamespaceAdmission, RetentionPublicationPreparation, - RetentionPublicationStorage, RetentionTransitionDisposition, + RetentionNamespaceAdmission, RetentionPublicationPreparation, RetentionPublicationStorage, + RetentionTransitionDisposition, }; use crate::adapters::filesystem_catalog_artifact::synchronize_directory; -const HEAD_LENGTH: usize = 144; - impl RetentionPublicationStorage for FilesystemRetentionPublicationAuthority { fn verify_current( &mut self, @@ -24,19 +23,8 @@ impl RetentionPublicationStorage for FilesystemRetentionPublicationAuthority { ) -> io::Result { self.liveness_generation = Some(preparation.liveness_generation()); require_no_retained_stage(&self.retention)?; - let Some(head_bytes) = read_head(&self.retention)? else { - return Ok(RetentionTransitionDisposition::Publish); - }; - let head = ChecksummedRetentionHead::decode(&head_bytes) - .map_err(|_source| invalid_data("current retention head refused admission"))?; - if head.head().generation() == preparation.liveness_generation() - && head.head().manifest_digest() == preparation.manifest_digest() - { - return Ok(RetentionTransitionDisposition::AlreadyCommitted); - } - Err(invalid_data( - "current retention head is not the prepared predecessor; recovery is required", - )) + let current = filesystem_retention_current::observe(&self.retention, &self.manifests)?; + filesystem_retention_current::disposition(preparation, current.as_ref()) } fn write_root_stage(&mut self, root: &AdmittedRetentionRoot<'_>) -> io::Result<()> { @@ -168,29 +156,6 @@ fn require_no_retained_stage(retention: &Dir) -> io::Result<()> { Ok(()) } -fn read_head(retention: &Dir) -> io::Result>> { - let mut options = OpenOptions::new(); - options.read(true).follow(FollowSymlinks::No); - let mut file = match retention.open_with(pool_name::HEAD, &options) { - Ok(file) => file, - Err(source) if source.kind() == io::ErrorKind::NotFound => return Ok(None), - Err(source) => return Err(source), - }; - let expected_length = u64::try_from(HEAD_LENGTH) - .map_err(|_source| invalid_data("retention head length exceeded u64"))?; - let metadata = file.metadata()?; - if !metadata.is_file() || metadata.len() != expected_length { - return Err(invalid_data("retention head kind or length disagreed")); - } - let mut bytes = vec![0_u8; HEAD_LENGTH]; - file.read_exact(&mut bytes)?; - let mut trailing = [0_u8; 1]; - if file.read(&mut trailing)? != 0 { - return Err(invalid_data("retention head carried trailing bytes")); - } - Ok(Some(bytes)) -} - impl FilesystemRetentionPublicationAuthority { fn manifest_name(&self, manifest: &CanonicalRetentionManifest) -> io::Result { let generation = self diff --git a/src/adapters/retention/filesystem_retention_storage_tests.rs b/src/adapters/retention/filesystem_retention_storage_tests.rs index 5332cbd..67b24fc 100644 --- a/src/adapters/retention/filesystem_retention_storage_tests.rs +++ b/src/adapters/retention/filesystem_retention_storage_tests.rs @@ -1,23 +1,16 @@ //! Filesystem retention publication storage laws. -use std::collections::BTreeSet; use std::error::Error; -use std::ffi::OsString; use std::fs; use std::io; use std::path::{Path, PathBuf}; use super::filesystem_retention_test_fixture::{ - HEAD_HEX, MANIFEST_HEX, ROOT_HEX, fixture, open_authority, with_snapshot, -}; -use super::{ - AdmittedRetentionRoot, RetentionPublicationError, RetentionPublicationOutcome, - RetentionPublicationPreparation, RetentionPublicationStorage, RetentionTransitionDisposition, -}; -use crate::{ - RetentionGenerationExpectation, execute_retention_publication, preflight_retention_transition, - prepare_retention_publication, + HEAD_HEX, MANIFEST_HEX, ROOT_HEX, fixture, head_path, initial_preparation, manifest_pool_path, + open_authority, retention_witness, root_pool_path, }; +use super::{RetentionPublicationError, RetentionPublicationOutcome, RetentionPublicationStorage}; +use crate::execute_retention_publication; #[test] fn complete_publication_preserves_migrated_bytes_and_publishes_exact_retention_prefix() @@ -25,7 +18,7 @@ fn complete_publication_preserves_migrated_bytes_and_publishes_exact_retention_p let (sandbox, mut authority) = open_authority("filesystem-retention-complete")?; let before = migrated_witness(sandbox.path())?; let root_bytes = fixture(ROOT_HEX)?; - let preparation = publish_preparation(&root_bytes)?; + let preparation = initial_preparation(&root_bytes)?; let receipt = execute_retention_publication(&mut authority, &preparation)?; @@ -33,7 +26,7 @@ fn complete_publication_preserves_migrated_bytes_and_publishes_exact_retention_p assert_eq!(migrated_witness(sandbox.path())?, before); assert_eq!(fs::read(head_path(sandbox.path()))?, fixture(HEAD_HEX)?); assert_eq!( - fs::read(root_pool_path(sandbox.path(), &preparation))?, + fs::read(root_pool_path(sandbox.path(), preparation.candidate()))?, root_bytes ); assert_eq!( @@ -50,7 +43,7 @@ fn complete_publication_preserves_migrated_bytes_and_publishes_exact_retention_p fn existing_root_stage_is_never_truncated() -> Result<(), Box> { let (sandbox, mut authority) = open_authority("filesystem-retention-exclusive-stage")?; let root_bytes = fixture(ROOT_HEX)?; - let preparation = publish_preparation(&root_bytes)?; + let preparation = initial_preparation(&root_bytes)?; let stage = sandbox.path().join("retention").join("root.next"); fs::write(&stage, b"retained partial evidence")?; @@ -70,7 +63,7 @@ fn existing_root_stage_is_never_truncated() -> Result<(), Box> { fn retained_stage_refuses_publication_before_recovery() -> Result<(), Box> { let (sandbox, mut authority) = open_authority("filesystem-retention-recovery-required")?; let root_bytes = fixture(ROOT_HEX)?; - let preparation = publish_preparation(&root_bytes)?; + let preparation = initial_preparation(&root_bytes)?; fs::write( sandbox.path().join("retention").join("head.next"), fixture(HEAD_HEX)?, @@ -94,14 +87,14 @@ fn retained_stage_refuses_publication_before_recovery() -> Result<(), Box Result<(), Box> { let (sandbox, mut authority) = open_authority("filesystem-retention-substituted-target")?; let root_bytes = fixture(ROOT_HEX)?; - let preparation = publish_preparation(&root_bytes)?; + let preparation = initial_preparation(&root_bytes)?; let candidate = preparation.candidate(); let _disposition = RetentionPublicationStorage::verify_current(&mut authority, &preparation)?; RetentionPublicationStorage::write_root_stage(&mut authority, candidate)?; RetentionPublicationStorage::synchronize_root_stage(&mut authority)?; let _admission = RetentionPublicationStorage::admit_root_namespace(&mut authority, candidate)?; RetentionPublicationStorage::synchronize_roots_after_namespace(&mut authority)?; - fs::write(root_pool_path(sandbox.path(), &preparation), &root_bytes)?; + fs::write(root_pool_path(sandbox.path(), candidate), &root_bytes)?; let error = RetentionPublicationStorage::link_root(&mut authority, candidate) .err() @@ -117,10 +110,10 @@ fn byte_equal_substituted_canonical_root_is_refused() -> Result<(), Box Result<(), Box> { let (sandbox, mut authority) = open_authority("filesystem-retention-exact-retry")?; let root_bytes = fixture(ROOT_HEX)?; - let preparation = publish_preparation(&root_bytes)?; + let preparation = initial_preparation(&root_bytes)?; let _published = execute_retention_publication(&mut authority, &preparation)?; let after_publication = retention_witness(sandbox.path())?; - let retry = publish_preparation(&root_bytes)?; + let retry = initial_preparation(&root_bytes)?; let receipt = execute_retention_publication(&mut authority, &retry)?; @@ -134,50 +127,6 @@ fn exact_committed_retry_mutates_nothing() -> Result<(), Box> { Ok(()) } -fn publish_preparation( - root_bytes: &[u8], -) -> Result, Box> { - let candidate = AdmittedRetentionRoot::decode(root_bytes)?; - let preflight = with_snapshot(|snapshot| { - preflight_retention_transition( - RetentionGenerationExpectation::Absent, - None, - candidate, - snapshot, - ) - })??; - let preparation = prepare_retention_publication(preflight, None)?; - assert_eq!( - preparation.disposition(), - RetentionTransitionDisposition::Publish - ); - Ok(preparation) -} - -fn head_path(root: &Path) -> PathBuf { - root.join("retention").join("HEAD") -} - -fn root_pool_path(root: &Path, preparation: &RetentionPublicationPreparation<'_>) -> PathBuf { - let candidate = preparation.candidate(); - root.join("retention") - .join("roots") - .join(hex(candidate.root().namespace().digest().as_bytes())) - .join(format!( - "{:016x}-{}.root", - candidate.root().generation().get(), - hex(candidate.digest().as_bytes()) - )) -} - -fn manifest_pool_path(root: &Path, preparation: &RetentionPublicationPreparation<'_>) -> PathBuf { - root.join("retention").join("manifests").join(format!( - "{:016x}-{}.manifest", - preparation.liveness_generation().get(), - hex(preparation.manifest_digest().as_bytes()) - )) -} - fn assert_stages_absent(root: &Path) -> Result<(), Box> { for stage in ["root.next", "manifest.next", "head.next"] { let path = root.join("retention").join(stage); @@ -203,30 +152,3 @@ fn migrated_witness(root: &Path) -> io::Result)>> { witness.sort(); Ok(witness) } - -fn retention_witness(root: &Path) -> io::Result)>> { - let mut witness = BTreeSet::new(); - collect(&root.join("retention"), &mut witness)?; - Ok(witness) -} - -fn collect(directory: &Path, witness: &mut BTreeSet<(OsString, Vec)>) -> io::Result<()> { - for entry in fs::read_dir(directory)? { - let entry = entry?; - let path = entry.path(); - if entry.file_type()?.is_dir() { - collect(&path, witness)?; - } else { - witness.insert((path.into_os_string(), fs::read(entry.path())?)); - } - } - Ok(()) -} - -fn hex(bytes: &[u8; 32]) -> String { - use std::fmt::Write as _; - bytes.iter().fold(String::new(), |mut rendered, byte| { - let _ = write!(rendered, "{byte:02x}"); - rendered - }) -} diff --git a/src/adapters/retention/filesystem_retention_successor_tests.rs b/src/adapters/retention/filesystem_retention_successor_tests.rs new file mode 100644 index 0000000..56d91f6 --- /dev/null +++ b/src/adapters/retention/filesystem_retention_successor_tests.rs @@ -0,0 +1,120 @@ +//! Filesystem retention successor-generation and superseded-retry laws. + +use std::error::Error; +use std::fs; +use std::io; + +use super::filesystem_retention_test_fixture::{ + ROOT_HEX, fixture, head_path, initial_generation, initial_preparation, manifest_pool_path, + open_authority, retention_witness, root_pool_path, successor_preparation, successor_root, +}; +use super::{ + AdmittedRetentionManifest, AdmittedRetentionRoot, ChecksummedRetentionHead, + RetentionNamespaceAdmission, RetentionPublicationError, RetentionPublicationOutcome, + RetentionPublicationStorage, +}; +use crate::execute_retention_publication; + +#[test] +fn successor_publication_over_existing_head_publishes_exact_successor() -> Result<(), Box> +{ + let (sandbox, mut authority) = open_authority("filesystem-retention-successor")?; + let root_bytes = fixture(ROOT_HEX)?; + let initial = initial_preparation(&root_bytes)?; + let _published = execute_retention_publication(&mut authority, &initial)?; + let generation_one = retention_witness(sandbox.path())?; + let current = authority + .observe_current()? + .ok_or("published retention head was not observed")?; + let current_root = AdmittedRetentionRoot::decode(&root_bytes)?; + let current_manifest = AdmittedRetentionManifest::decode(current.manifest_bytes())?; + let candidate = successor_root(¤t_root)?; + let preparation = successor_preparation(¤t_root, ¤t_manifest, candidate.encoded())?; + + let receipt = execute_retention_publication(&mut authority, &preparation)?; + + assert_eq!(receipt.outcome(), RetentionPublicationOutcome::Published); + assert_eq!( + receipt.namespace_admission(), + Some(RetentionNamespaceAdmission::Existing) + ); + let head_bytes = fs::read(head_path(sandbox.path()))?; + let head = ChecksummedRetentionHead::decode(&head_bytes)?; + assert_eq!( + head.head().generation(), + current_manifest.manifest().generation().successor()? + ); + assert_eq!(head.head().predecessor(), Some(current_manifest.digest())); + assert_eq!(head.head().manifest_digest(), preparation.manifest_digest()); + assert_eq!( + fs::read(root_pool_path(sandbox.path(), preparation.candidate()))?, + candidate.encoded() + ); + assert!(manifest_pool_path(sandbox.path(), &preparation).exists()); + let generation_two = retention_witness(sandbox.path())?; + for entry in &generation_one { + if !entry.0.to_string_lossy().ends_with("HEAD") { + assert!( + generation_two.contains(entry), + "generation-one evidence changed" + ); + } + } + drop(authority); + sandbox.remove()?; + Ok(()) +} + +#[test] +fn superseded_candidate_refuses_once_a_successor_is_current() -> Result<(), Box> { + let (sandbox, mut authority) = open_authority("filesystem-retention-superseded")?; + let root_bytes = fixture(ROOT_HEX)?; + let _published = + execute_retention_publication(&mut authority, &initial_preparation(&root_bytes)?)?; + let current = authority + .observe_current()? + .ok_or("published retention head was not observed")?; + let current_root = AdmittedRetentionRoot::decode(&root_bytes)?; + let current_manifest = AdmittedRetentionManifest::decode(current.manifest_bytes())?; + let candidate = successor_root(¤t_root)?; + let successor = successor_preparation(¤t_root, ¤t_manifest, candidate.encoded())?; + let _advanced = execute_retention_publication(&mut authority, &successor)?; + let after_successor = retention_witness(sandbox.path())?; + let stale_retry = initial_preparation(&root_bytes)?; + + let error = execute_retention_publication(&mut authority, &stale_retry) + .err() + .ok_or("superseded generation-one candidate was unexpectedly accepted")?; + + let RetentionPublicationError::CurrentVerification { source } = error else { + return Err("superseded candidate refused outside current-state verification".into()); + }; + assert_eq!(source.kind(), io::ErrorKind::InvalidData); + assert_eq!(retention_witness(sandbox.path())?, after_successor); + drop(authority); + sandbox.remove()?; + Ok(()) +} + +#[test] +fn expected_current_generation_refuses_when_no_head_is_published() -> Result<(), Box> { + let (sandbox, mut authority) = open_authority("filesystem-retention-absent-predecessor")?; + let root_bytes = fixture(ROOT_HEX)?; + let current_root = AdmittedRetentionRoot::decode(&root_bytes)?; + let manifest_bytes = fixture(super::filesystem_retention_test_fixture::MANIFEST_HEX)?; + let current_manifest = AdmittedRetentionManifest::decode(&manifest_bytes)?; + let candidate = successor_root(¤t_root)?; + let preparation = successor_preparation(¤t_root, ¤t_manifest, candidate.encoded())?; + assert_eq!(preparation.observed(), Some(initial_generation())); + + let error = RetentionPublicationStorage::verify_current(&mut authority, &preparation) + .err() + .ok_or("successor over an absent head was unexpectedly admitted")?; + + assert_eq!(error.kind(), io::ErrorKind::InvalidData); + assert!(authority.observe_current()?.is_none()); + assert!(!head_path(sandbox.path()).exists()); + drop(authority); + sandbox.remove()?; + Ok(()) +} diff --git a/src/adapters/retention/filesystem_retention_test_fixture.rs b/src/adapters/retention/filesystem_retention_test_fixture.rs index 6120339..d2d326f 100644 --- a/src/adapters/retention/filesystem_retention_test_fixture.rs +++ b/src/adapters/retention/filesystem_retention_test_fixture.rs @@ -1,9 +1,18 @@ //! This test module owns one migrated version-2 retention publication fixture. +use std::collections::BTreeSet; use std::error::Error; +use std::ffi::OsString; +use std::fmt::Write as _; use std::fs; +use std::io; +use std::path::{Path, PathBuf}; use super::filesystem_retention_authority::FilesystemRetentionPublicationAuthority; +use super::{ + AdmittedRetentionManifest, AdmittedRetentionRoot, CanonicalRetentionRoot, + RetentionPublicationPreparation, RetentionTransitionDisposition, +}; use crate::LayoutEntryLimit; use crate::adapters::filesystem_test_sandbox::TestDirectory; use crate::adapters::test_support::decode_hex; @@ -12,7 +21,10 @@ use crate::adapters::{ ChecksummedPublicationHead, FilesystemPlatformAdmission, FilesystemStoreMigrationAuthority, SegmentReadPolicy, SegmentRecordLimit, }; -use crate::execute_store_migration; +use crate::{ + RetentionGenerationExpectation, RetentionPolicy, RetentionRoot, RootGeneration, + execute_store_migration, preflight_retention_transition, prepare_retention_publication, +}; /// Frozen canonical generation-one root. pub(super) const ROOT_HEX: &str = @@ -71,6 +83,117 @@ pub(super) fn with_snapshot( Ok(operation(&snapshot)) } +/// Prepares the frozen generation-one root as an initial `Publish` transition. +pub(super) fn initial_preparation( + root_bytes: &[u8], +) -> Result, Box> { + let candidate = AdmittedRetentionRoot::decode(root_bytes)?; + let preflight = with_snapshot(|snapshot| { + preflight_retention_transition( + RetentionGenerationExpectation::Absent, + None, + candidate, + snapshot, + ) + })??; + let preparation = prepare_retention_publication(preflight, None)?; + assert_eq!( + preparation.disposition(), + RetentionTransitionDisposition::Publish + ); + Ok(preparation) +} + +/// Prepares `candidate_bytes` as the exact successor of `current` under +/// `current_manifest`. +pub(super) fn successor_preparation<'encoded>( + current: &AdmittedRetentionRoot<'_>, + current_manifest: &AdmittedRetentionManifest<'_>, + candidate_bytes: &'encoded [u8], +) -> Result, Box> { + let candidate = AdmittedRetentionRoot::decode(candidate_bytes)?; + let preflight = with_snapshot(|snapshot| { + preflight_retention_transition( + RetentionGenerationExpectation::Current(current.root().generation()), + Some(current), + candidate, + snapshot, + ) + })??; + prepare_retention_publication(preflight, Some(current_manifest)).map_err(Into::into) +} + +/// Builds the exact semantic successor of one admitted root. +pub(super) fn successor_root( + current: &AdmittedRetentionRoot<'_>, +) -> Result> { + let root = RetentionRoot::new( + current.root().namespace().clone(), + current.root().generation().successor()?, + RetentionPolicy::new(current.root().profile(), current.root().limits()), + Some(current.digest()), + current.root().anchors().to_vec(), + )?; + CanonicalRetentionRoot::from_root(&root).map_err(Into::into) +} + +pub(super) fn head_path(root: &Path) -> PathBuf { + root.join("retention").join("HEAD") +} + +pub(super) fn root_pool_path(root: &Path, candidate: &AdmittedRetentionRoot<'_>) -> PathBuf { + root.join("retention") + .join("roots") + .join(hex(candidate.root().namespace().digest().as_bytes())) + .join(format!( + "{:016x}-{}.root", + candidate.root().generation().get(), + hex(candidate.digest().as_bytes()) + )) +} + +pub(super) fn manifest_pool_path( + root: &Path, + preparation: &RetentionPublicationPreparation<'_>, +) -> PathBuf { + root.join("retention").join("manifests").join(format!( + "{:016x}-{}.manifest", + preparation.liveness_generation().get(), + hex(preparation.manifest_digest().as_bytes()) + )) +} + +/// Every regular file beneath `retention`, with its exact bytes. +pub(super) fn retention_witness(root: &Path) -> io::Result)>> { + let mut witness = BTreeSet::new(); + collect(&root.join("retention"), &mut witness)?; + Ok(witness) +} + +pub(super) const fn initial_generation() -> RootGeneration { + RootGeneration::INITIAL +} + +fn collect(directory: &Path, witness: &mut BTreeSet<(OsString, Vec)>) -> io::Result<()> { + for entry in fs::read_dir(directory)? { + let entry = entry?; + let path = entry.path(); + if entry.file_type()?.is_dir() { + collect(&path, witness)?; + } else { + witness.insert((path.into_os_string(), fs::read(entry.path())?)); + } + } + Ok(()) +} + +fn hex(bytes: &[u8; 32]) -> String { + bytes.iter().fold(String::new(), |mut rendered, byte| { + let _ = write!(rendered, "{byte:02x}"); + rendered + }) +} + fn migrated_store(name: &str) -> Result> { let sandbox = TestDirectory::create(name)?; let admission = FilesystemPlatformAdmission::initialize_unchecked_for_tests(sandbox.path())?; diff --git a/src/lib.rs b/src/lib.rs index b04c9ed..6d54af1 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -136,10 +136,10 @@ pub use adapters::{ AdmittedRetentionManifest, AdmittedRetentionRoot, CanonicalRetentionHead, CanonicalRetentionManifest, CanonicalRetentionRoot, ChecksummedRetentionHead, FilesystemRetentionAuthorityError, FilesystemRetentionPublicationAuthority, - PreparedRetentionPublication, RetentionAuthorityDirectory, RetentionClosureVerificationError, - RetentionHeadDecodeError, RetentionManifestDecodeError, RetentionManifestEncodeError, - RetentionNamespaceAdmission, RetentionPublicationError, RetentionPublicationOutcome, - RetentionPublicationPhase, RetentionPublicationPreparation, + ObservedRetentionState, PreparedRetentionPublication, RetentionAuthorityDirectory, + RetentionClosureVerificationError, RetentionHeadDecodeError, RetentionManifestDecodeError, + RetentionManifestEncodeError, RetentionNamespaceAdmission, RetentionPublicationError, + RetentionPublicationOutcome, RetentionPublicationPhase, RetentionPublicationPreparation, RetentionPublicationPreparationError, RetentionPublicationReceipt, RetentionPublicationStorage, RetentionRootDecodeError, RetentionRootEncodeError, RetentionTransitionDisposition, RetentionTransitionError, RetentionTransitionPreflight, RetentionTransitionPreflightError, From 73c3ecaa899d06c3c19e92da0218a80af8cf98b6 Mon Sep 17 00:00:00 2001 From: James Ross Date: Sun, 6 Sep 2026 23:58:01 -0700 Subject: [PATCH 058/111] Fix: admit version-two migration records before granting writer authority reopen_version_two admitted a migrated root on file kinds alone, so a FORMAT, migration.intent, or migration.receipt with arbitrary bytes, wrong length, or mutually inconsistent contents still produced writer authority that FilesystemRetentionPublicationAuthority could consume. The version-two reopen path now reopens all three records without following links, bounds each to its canonical length, refuses trailing bytes, and admits the receipt only against the decoded intent and marker. Refusals surface as FilesystemPlatformAdmissionError::MigrationRecord. Regression laws: corrupt marker byte, oversized marker, and a receipt that disagrees with its intent all refuse; exact records still admit. Addresses Codex review thread on filesystem_initialization_namespace.rs:74. Refs #78 --- CHANGELOG.md | 5 ++ docs/formats/segment-store-v2/requirements.md | 2 +- .../filesystem_platform_admission_error.rs | 10 ++- src/adapters/filesystem_store_initializer.rs | 14 +++- .../filesystem_version_two_records.rs | 62 ++++++++++++++ src/adapters/mod.rs | 1 + src/adapters/retention.rs | 2 + .../filesystem_retention_test_fixture.rs | 3 +- .../filesystem_version_two_admission_tests.rs | 80 +++++++++++++++++++ 9 files changed, 174 insertions(+), 5 deletions(-) create mode 100644 src/adapters/filesystem_version_two_records.rs create mode 100644 src/adapters/retention/filesystem_version_two_admission_tests.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index 2799e76..fdb343f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -34,6 +34,11 @@ after its public API and format compatibility policies are established. pool manifest, and current-state verification admits a successor only when the prepared head names the observed manifest as its exact predecessor at the next liveness generation; a superseded candidate refuses with zero mutation. + `reopen_version_two` reopens `FORMAT`, `migration.intent`, and + `migration.receipt` without following links, bounds each to its canonical + length, and admits the receipt only against the decoded intent and marker + before returning writer authority; `FilesystemPlatformAdmissionError::MigrationRecord` + names that refusal. - Repository crash-matrix execution now terminates isolated writer process groups at all 105 canonical before/during/after coordinates, retains open writer and stage authority until termination, executes production diff --git a/docs/formats/segment-store-v2/requirements.md b/docs/formats/segment-store-v2/requirements.md index 36ba382..865cfc7 100644 --- a/docs/formats/segment-store-v2/requirements.md +++ b/docs/formats/segment-store-v2/requirements.md @@ -35,7 +35,7 @@ case is not evidence. | `KEEP-MIGRATION-005` | Unknown, out-of-order, substituted, corrupt, conflicting, or changed evidence is unrecoverable ambiguity | forward-execution stage preservation, byte-equal inode-substitution, out-of-order-prefix, and post-publication drift laws in `filesystem_migration_storage_tests`; restart corruption and mutation matrix remains | In progress in #19 | | `KEEP-MIGRATION-006` | Migration never rewrites or deletes admitted version-1 immutable bytes | exact segment, catalog, and head before/after witness in `filesystem_migration_storage_tests`; restart-path evidence remains | In progress in #19 | | `KEEP-MIGRATION-007` | Process death around every intent stage, canonical link, namespace prefix, marker stage, receipt stage, cleanup, and synchronization boundary reaches a documented lawful state | ordered phases and capabilities in `tests/store_migration_phase.rs` and `tests/store_migration_storage.rs`; exact phase-failure execution in `tests/store_migration_execution.rs`; production 21-phase forward execution in `filesystem_migration_storage_tests`; `KEEP-CRASH-053..=073` process-death matrix remains | In progress in #19 | -| `KEEP-MIGRATION-008` | Version-1 admission refuses every version-2 or partial-migration artifact after migration begins | `FORMAT` refusal before mutation in `filesystem_migration_authority_tests`; exact version-1 reopen refusal of a migrated root and separate version-2 namespace admission in `filesystem_initialization_namespace`, exercised by `filesystem_retention_storage_tests`; remaining compatibility and fuzz matrix | In progress in #19 | +| `KEEP-MIGRATION-008` | Version-1 admission refuses every version-2 or partial-migration artifact after migration begins | `FORMAT` refusal before mutation in `filesystem_migration_authority_tests`; exact version-1 reopen refusal of a migrated root and separate version-2 namespace admission in `filesystem_initialization_namespace`; version-2 reopen jointly admits the exact marker, intent, and receipt before returning writer authority, with corrupt, oversized, and mutually inconsistent record refusals in `filesystem_version_two_admission_tests`; remaining compatibility and fuzz matrix | In progress in #19 | diff --git a/src/adapters/filesystem_platform_admission_error.rs b/src/adapters/filesystem_platform_admission_error.rs index 7bee25a..035eb36 100644 --- a/src/adapters/filesystem_platform_admission_error.rs +++ b/src/adapters/filesystem_platform_admission_error.rs @@ -24,6 +24,11 @@ pub enum FilesystemPlatformAdmissionError { /// Preserved namespace-admission failure. source: io::Error, }, + /// A version-two migration record failed exact or joint admission. + MigrationRecord { + /// Preserved record-admission failure. + source: io::Error, + }, } impl fmt::Display for FilesystemPlatformAdmissionError { @@ -32,6 +37,7 @@ impl fmt::Display for FilesystemPlatformAdmissionError { Self::Platform { .. } => "published store platform admission failed", Self::WriterLock { .. } => "published store writer-lock acquisition failed", Self::Namespace { .. } => "published store namespace admission failed", + Self::MigrationRecord { .. } => "version-two migration record admission failed", }) } } @@ -39,7 +45,9 @@ impl fmt::Display for FilesystemPlatformAdmissionError { impl Error for FilesystemPlatformAdmissionError { fn source(&self) -> Option<&(dyn Error + 'static)> { match self { - Self::Platform { source } | Self::Namespace { source } => Some(source), + Self::Platform { source } + | Self::Namespace { source } + | Self::MigrationRecord { source } => Some(source), Self::WriterLock { source } => Some(source), } } diff --git a/src/adapters/filesystem_store_initializer.rs b/src/adapters/filesystem_store_initializer.rs index 1e4f7ef..dc1240f 100644 --- a/src/adapters/filesystem_store_initializer.rs +++ b/src/adapters/filesystem_store_initializer.rs @@ -11,7 +11,7 @@ use super::filesystem_initialization_storage::FilesystemInitializationStorage; use super::{ FilesystemPlatformAdmission, FilesystemPlatformAdmissionError, FilesystemWriterLock, StoreInitializationError, StoreInitializationPhase, filesystem_initialization_namespace, - filesystem_platform_profile, initialize_store, + filesystem_platform_profile, filesystem_version_two_records, initialize_store, }; impl FilesystemPlatformAdmission { @@ -126,7 +126,17 @@ fn initialize_storage( fn reopen_version_two_root( root: cap_std::fs::Dir, ) -> Result { - admit_reopened(root, filesystem_initialization_namespace::admit_version_two) + let admission = admit_reopened(root, filesystem_initialization_namespace::admit_version_two)?; + let (lock, root_identity) = admission.into_parts(); + let directory = lock + .clone_directory() + .map_err(|source| FilesystemPlatformAdmissionError::MigrationRecord { source })?; + filesystem_version_two_records::admit(&directory) + .map_err(|source| FilesystemPlatformAdmissionError::MigrationRecord { source })?; + Ok(FilesystemPlatformAdmission::initialized( + lock, + root_identity, + )) } fn reopen_root( diff --git a/src/adapters/filesystem_version_two_records.rs b/src/adapters/filesystem_version_two_records.rs new file mode 100644 index 0000000..0e2754f --- /dev/null +++ b/src/adapters/filesystem_version_two_records.rs @@ -0,0 +1,62 @@ +//! This module owns joint admission of the three fixed version-two migration records. + +use std::io::{self, Read}; + +use cap_fs_ext::{FollowSymlinks, OpenOptionsFollowExt}; +use cap_std::fs::{Dir, OpenOptions}; + +use super::{ + AdmittedStoreFormatMarker, AdmittedStoreMigrationIntent, AdmittedStoreMigrationReceipt, +}; + +const MARKER_NAME: &str = "FORMAT"; +const INTENT_NAME: &str = "migration.intent"; +const RECEIPT_NAME: &str = "migration.receipt"; +const MARKER_LENGTH: usize = 96; +const RECORD_LENGTH: usize = 256; + +/// Reads and jointly admits `FORMAT`, `migration.intent`, and `migration.receipt`. +/// +/// Each record is reopened without following links, bounded to its exact +/// canonical length, and decoded. The receipt is admitted only against the +/// decoded intent and marker, so a record set that is individually +/// well-formed but mutually inconsistent refuses. Writer authority over a +/// version-two root must not be returned before this admission succeeds. +pub(super) fn admit(root: &Dir) -> io::Result<()> { + let marker_bytes = read_exact(root, MARKER_NAME, MARKER_LENGTH)?; + let intent_bytes = read_exact(root, INTENT_NAME, RECORD_LENGTH)?; + let receipt_bytes = read_exact(root, RECEIPT_NAME, RECORD_LENGTH)?; + let marker = AdmittedStoreFormatMarker::decode(&marker_bytes) + .map_err(|source| invalid_data(MARKER_NAME, &source))?; + let intent = AdmittedStoreMigrationIntent::decode(&intent_bytes) + .map_err(|source| invalid_data(INTENT_NAME, &source))?; + let _receipt = AdmittedStoreMigrationReceipt::decode(&receipt_bytes, &intent, &marker) + .map_err(|source| invalid_data(RECEIPT_NAME, &source))?; + Ok(()) +} + +fn read_exact(root: &Dir, name: &str, length: usize) -> io::Result> { + let mut options = OpenOptions::new(); + options.read(true).follow(FollowSymlinks::No); + let mut file = root.open_with(name, &options)?; + let expected_length = u64::try_from(length) + .map_err(|_source| invalid_data(name, &"record length exceeded u64"))?; + let metadata = file.metadata()?; + if !metadata.is_file() || metadata.len() != expected_length { + return Err(invalid_data(name, &"record kind or length disagreed")); + } + let mut bytes = vec![0_u8; length]; + file.read_exact(&mut bytes)?; + let mut trailing = [0_u8; 1]; + if file.read(&mut trailing)? != 0 { + return Err(invalid_data(name, &"record carried trailing bytes")); + } + Ok(bytes) +} + +fn invalid_data(name: &str, source: &dyn std::fmt::Display) -> io::Error { + io::Error::new( + io::ErrorKind::InvalidData, + format!("version-two record {name} refused admission: {source}"), + ) +} diff --git a/src/adapters/mod.rs b/src/adapters/mod.rs index 6d69ae8..4da8d54 100644 --- a/src/adapters/mod.rs +++ b/src/adapters/mod.rs @@ -129,6 +129,7 @@ mod filesystem_store_initializer_tests; #[cfg(test)] #[path = "../../tests/segment_filesystem_stage/sandbox.rs"] mod filesystem_test_sandbox; +mod filesystem_version_two_records; mod filesystem_writer_lock; mod framed_blake3; mod layout_decode_error; diff --git a/src/adapters/retention.rs b/src/adapters/retention.rs index d8441b2..5642793 100644 --- a/src/adapters/retention.rs +++ b/src/adapters/retention.rs @@ -25,6 +25,8 @@ mod filesystem_retention_storage_tests; mod filesystem_retention_successor_tests; #[cfg(test)] mod filesystem_retention_test_fixture; +#[cfg(test)] +mod filesystem_version_two_admission_tests; mod head_decode_error; mod head_decode_error_display; mod head_decoder; diff --git a/src/adapters/retention/filesystem_retention_test_fixture.rs b/src/adapters/retention/filesystem_retention_test_fixture.rs index d2d326f..2857af4 100644 --- a/src/adapters/retention/filesystem_retention_test_fixture.rs +++ b/src/adapters/retention/filesystem_retention_test_fixture.rs @@ -194,7 +194,8 @@ fn hex(bytes: &[u8; 32]) -> String { }) } -fn migrated_store(name: &str) -> Result> { +/// Builds one completely migrated version-2 store with writer authority released. +pub(super) fn migrated_store(name: &str) -> Result> { let sandbox = TestDirectory::create(name)?; let admission = FilesystemPlatformAdmission::initialize_unchecked_for_tests(sandbox.path())?; write_version_one(&sandbox)?; diff --git a/src/adapters/retention/filesystem_version_two_admission_tests.rs b/src/adapters/retention/filesystem_version_two_admission_tests.rs new file mode 100644 index 0000000..c2411eb --- /dev/null +++ b/src/adapters/retention/filesystem_version_two_admission_tests.rs @@ -0,0 +1,80 @@ +//! Version-two reopen laws: writer authority requires jointly admitted migration records. + +use std::error::Error; +use std::fs; + +use super::filesystem_retention_test_fixture::migrated_store; +use crate::adapters::{FilesystemPlatformAdmission, FilesystemPlatformAdmissionError}; + +#[test] +fn version_two_reopen_refuses_a_corrupt_format_marker() -> Result<(), Box> { + let sandbox = migrated_store("version-two-admission-corrupt-marker")?; + let marker = sandbox.path().join("FORMAT"); + let mut bytes = fs::read(&marker)?; + *bytes + .get_mut(40) + .ok_or("format marker shorter than 41 bytes")? ^= 0x01; + fs::write(&marker, &bytes)?; + + let error = FilesystemPlatformAdmission::reopen_version_two_unchecked_for_tests(sandbox.path()) + .err() + .ok_or("corrupt format marker was unexpectedly admitted")?; + + assert!(matches!( + error, + FilesystemPlatformAdmissionError::MigrationRecord { .. } + )); + sandbox.remove()?; + Ok(()) +} + +#[test] +fn version_two_reopen_refuses_an_oversized_format_marker() -> Result<(), Box> { + let sandbox = migrated_store("version-two-admission-oversized-marker")?; + let marker = sandbox.path().join("FORMAT"); + let mut bytes = fs::read(&marker)?; + bytes.push(0); + fs::write(&marker, &bytes)?; + + let error = FilesystemPlatformAdmission::reopen_version_two_unchecked_for_tests(sandbox.path()) + .err() + .ok_or("oversized format marker was unexpectedly admitted")?; + + assert!(matches!( + error, + FilesystemPlatformAdmissionError::MigrationRecord { .. } + )); + sandbox.remove()?; + Ok(()) +} + +#[test] +fn version_two_reopen_refuses_a_receipt_that_disagrees_with_its_intent() +-> Result<(), Box> { + let sandbox = migrated_store("version-two-admission-receipt-disagrees")?; + let intent = fs::read(sandbox.path().join("migration.intent"))?; + fs::write(sandbox.path().join("migration.receipt"), &intent)?; + + let error = FilesystemPlatformAdmission::reopen_version_two_unchecked_for_tests(sandbox.path()) + .err() + .ok_or("receipt disagreeing with its intent was unexpectedly admitted")?; + + assert!(matches!( + error, + FilesystemPlatformAdmissionError::MigrationRecord { .. } + )); + sandbox.remove()?; + Ok(()) +} + +#[test] +fn version_two_reopen_admits_exact_migration_records() -> Result<(), Box> { + let sandbox = migrated_store("version-two-admission-exact")?; + + let admission = + FilesystemPlatformAdmission::reopen_version_two_unchecked_for_tests(sandbox.path())?; + + drop(admission); + sandbox.remove()?; + Ok(()) +} From 3925e7146c47ff6fc5d017c653b430fa3bb63bea Mon Sep 17 00:00:00 2001 From: James Ross Date: Mon, 7 Sep 2026 00:00:46 -0700 Subject: [PATCH 059/111] Fix: reopen committed root evidence before reporting AlreadyCommitted Current-state verification returned AlreadyCommitted whenever retention/HEAD agreed with the prepared successor coordinates, without reopening the root the manifest selects. A deleted, substituted, or corrupted root pool entry therefore produced a successful receipt for an unavailable root. verify_committed now requires the observed manifest to select the candidate namespace at exactly the candidate's generation and digest, then reopens the immutable root pool entry without following links and requires its exact bytes. Absent, changed, or corrupt evidence refuses before any mutation. Regression laws: absent root pool entry, changed root pool bytes, and a corrupt selected manifest all refuse on retry with an unchanged witness. Addresses Codex review thread on reporting committed without reopening. Refs #78 --- CHANGELOG.md | 5 +- docs/formats/segment-store-v2/requirements.md | 2 +- src/adapters/retention.rs | 2 + .../retention/filesystem_retention_current.rs | 48 +++++++++- .../filesystem_retention_current_tests.rs | 87 +++++++++++++++++++ .../retention/filesystem_retention_storage.rs | 13 ++- 6 files changed, 151 insertions(+), 6 deletions(-) create mode 100644 src/adapters/retention/filesystem_retention_current_tests.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index fdb343f..51b6587 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -38,7 +38,10 @@ after its public API and format compatibility policies are established. `migration.receipt` without following links, bounds each to its canonical length, and admits the receipt only against the decoded intent and marker before returning writer authority; `FilesystemPlatformAdmissionError::MigrationRecord` - names that refusal. + names that refusal. An already-committed retention retry now reopens the + manifest entry and the root pool bytes the head selects and refuses absent, + changed, or corrupt evidence instead of inferring the commit from head + agreement alone. - Repository crash-matrix execution now terminates isolated writer process groups at all 105 canonical before/during/after coordinates, retains open writer and stage authority until termination, executes production diff --git a/docs/formats/segment-store-v2/requirements.md b/docs/formats/segment-store-v2/requirements.md index 865cfc7..89b2f85 100644 --- a/docs/formats/segment-store-v2/requirements.md +++ b/docs/formats/segment-store-v2/requirements.md @@ -17,7 +17,7 @@ case is not evidence. | `KEEP-RETENTION-006` | Publication follows the exact ordered durability protocol, including new namespace-directory admission and retention of fixed-stage evidence until head commit, and returns only after cleanup synchronization | typed vocabulary and blocking port in `tests/retention_publication_phase.rs` and `tests/retention_publication_storage.rs`; ordered execution, conditional namespace sync, and all 17 exact storage-fault boundaries in `tests/retention_publication_execution.rs`; production 17-phase forward filesystem execution, exclusive staging, byte-equal inode-substitution refusal, and retained-stage recovery refusal in `filesystem_retention_storage_tests`; crash injection remains | In progress in #19 | | `KEEP-RETENTION-007` | Restart resolves every fixed-stage crash prefix to one documented lawful state or typed ambiguity | debug and release crash matrix | Planned in #19 | | `KEEP-RETENTION-008` | Readers double-collect catalog and retention heads and bind one complete catalog, manifest, and root-generation view under a `ReaderFence` | immutable snapshot and concurrency tests | Planned in #19 | -| `KEEP-RETENTION-009` | Exact already-committed retry is idempotent only while its successor remains current | byte-identical planning in `tests/retention_transition.rs`; authority-revalidated zero-mutation retry receipt in `tests/retention_publication_execution.rs`; exact already-committed filesystem retry with a byte-identical retention witness in `filesystem_retention_storage_tests`; superseded-candidate filesystem refusal with zero mutation in `filesystem_retention_successor_tests` | Implemented | +| `KEEP-RETENTION-009` | Exact already-committed retry is idempotent only while its successor remains current | byte-identical planning in `tests/retention_transition.rs`; authority-revalidated zero-mutation retry receipt in `tests/retention_publication_execution.rs`; exact already-committed filesystem retry with a byte-identical retention witness in `filesystem_retention_storage_tests`; superseded-candidate filesystem refusal with zero mutation in `filesystem_retention_successor_tests`; committed retry reopens the head-selected manifest entry and root pool bytes, refusing absent, changed, or corrupt evidence in `filesystem_retention_current_tests` | Implemented | | `KEEP-RETENTION-010` | Model operation sequences agree with a deterministic namespace-to-anchor-set map and never admit caller identity, paths, clocks, or application policy | model-based and source-architecture tests | Planned in #19 | diff --git a/src/adapters/retention.rs b/src/adapters/retention.rs index 5642793..2e5111e 100644 --- a/src/adapters/retention.rs +++ b/src/adapters/retention.rs @@ -16,6 +16,8 @@ mod closure_verifier; mod filesystem_retention_authority; mod filesystem_retention_authority_error; mod filesystem_retention_current; +#[cfg(test)] +mod filesystem_retention_current_tests; mod filesystem_retention_pool_name; mod filesystem_retention_stage; mod filesystem_retention_storage; diff --git a/src/adapters/retention/filesystem_retention_current.rs b/src/adapters/retention/filesystem_retention_current.rs index db48205..17e35ad 100644 --- a/src/adapters/retention/filesystem_retention_current.rs +++ b/src/adapters/retention/filesystem_retention_current.rs @@ -2,14 +2,14 @@ use std::io::{self, Read}; -use cap_fs_ext::{FollowSymlinks, OpenOptionsFollowExt}; +use cap_fs_ext::{DirExt, FollowSymlinks, OpenOptionsFollowExt}; use cap_std::fs::{Dir, OpenOptions}; use super::filesystem_retention_pool_name as pool_name; use super::filesystem_retention_stage::invalid_data; use super::{ - AdmittedRetentionManifest, ChecksummedRetentionHead, RetentionPublicationPreparation, - RetentionTransitionDisposition, + AdmittedRetentionManifest, AdmittedRetentionRoot, ChecksummedRetentionHead, + RetentionPublicationPreparation, RetentionTransitionDisposition, }; use crate::RetentionGenerationExpectation; @@ -121,6 +121,48 @@ pub(super) fn disposition( } } +/// Reopens the evidence behind an `AlreadyCommitted` disposition. +/// +/// The observed manifest must select `candidate`'s namespace at exactly its +/// generation and digest, and the immutable root-pool entry must reopen with +/// exactly `candidate`'s bytes. A head that merely agrees with its manifest is +/// not proof that the claimed root is still available. +pub(super) fn verify_committed( + roots: &Dir, + current: &ObservedRetentionState, + candidate: &AdmittedRetentionRoot<'_>, +) -> io::Result<()> { + let manifest = AdmittedRetentionManifest::decode(current.manifest_bytes()) + .map_err(|_source| invalid_data("observed retention manifest refused admission"))?; + let namespace = candidate.root().namespace().digest(); + let entries = manifest.manifest().entries(); + let entry = entries + .binary_search_by_key(&namespace, |entry| entry.namespace()) + .ok() + .and_then(|index| entries.get(index).copied()) + .ok_or_else(|| { + invalid_data("committed manifest does not select the candidate namespace") + })?; + if entry.root_generation() != candidate.root().generation() + || entry.root_digest() != candidate.digest() + { + return Err(invalid_data( + "committed manifest selects a different root for the candidate namespace", + )); + } + let directory = roots + .open_dir_nofollow(pool_name::namespace(namespace)) + .map_err(|_source| invalid_data("committed root namespace directory is unavailable"))?; + let name = pool_name::root(candidate.root().generation(), candidate.digest()); + let observed = read_exact_optional(&directory, &name, candidate.encoded().len())? + .ok_or_else(|| invalid_data("committed root pool entry is absent"))?; + if observed.as_ref() == candidate.encoded() { + Ok(()) + } else { + Err(invalid_data("committed root pool entry bytes disagreed")) + } +} + fn read_exact_optional( directory: &Dir, name: &str, diff --git a/src/adapters/retention/filesystem_retention_current_tests.rs b/src/adapters/retention/filesystem_retention_current_tests.rs new file mode 100644 index 0000000..27eac2f --- /dev/null +++ b/src/adapters/retention/filesystem_retention_current_tests.rs @@ -0,0 +1,87 @@ +//! Filesystem retention current-state laws: a committed claim is reopened, never inferred. + +use std::error::Error; +use std::fs; +use std::io; + +use super::RetentionPublicationError; +use super::filesystem_retention_test_fixture::{ + ROOT_HEX, fixture, initial_preparation, manifest_pool_path, open_authority, retention_witness, + root_pool_path, +}; +use crate::execute_retention_publication; + +#[test] +fn committed_retry_refuses_when_the_selected_root_is_absent() -> Result<(), Box> { + let (sandbox, mut authority) = open_authority("filesystem-retention-committed-root-absent")?; + let root_bytes = fixture(ROOT_HEX)?; + let published = initial_preparation(&root_bytes)?; + let _receipt = execute_retention_publication(&mut authority, &published)?; + fs::remove_file(root_pool_path(sandbox.path(), published.candidate()))?; + let before = retention_witness(sandbox.path())?; + let retry = initial_preparation(&root_bytes)?; + + let error = execute_retention_publication(&mut authority, &retry) + .err() + .ok_or("committed retry succeeded although its root pool entry is absent")?; + + let RetentionPublicationError::CurrentVerification { source } = error else { + return Err("absent root refused outside current-state verification".into()); + }; + assert_eq!(source.kind(), io::ErrorKind::InvalidData); + assert_eq!(retention_witness(sandbox.path())?, before); + drop(authority); + sandbox.remove()?; + Ok(()) +} + +#[test] +fn committed_retry_refuses_when_the_selected_root_bytes_changed() -> Result<(), Box> { + let (sandbox, mut authority) = open_authority("filesystem-retention-committed-root-changed")?; + let root_bytes = fixture(ROOT_HEX)?; + let published = initial_preparation(&root_bytes)?; + let _receipt = execute_retention_publication(&mut authority, &published)?; + let pool_entry = root_pool_path(sandbox.path(), published.candidate()); + let mut changed = fs::read(&pool_entry)?; + *changed.last_mut().ok_or("empty root pool entry")? ^= 0x01; + fs::write(&pool_entry, &changed)?; + let retry = initial_preparation(&root_bytes)?; + + let error = execute_retention_publication(&mut authority, &retry) + .err() + .ok_or("committed retry succeeded although its root pool bytes changed")?; + + let RetentionPublicationError::CurrentVerification { source } = error else { + return Err("changed root refused outside current-state verification".into()); + }; + assert_eq!(source.kind(), io::ErrorKind::InvalidData); + drop(authority); + sandbox.remove()?; + Ok(()) +} + +#[test] +fn committed_retry_refuses_when_the_selected_manifest_is_corrupt() -> Result<(), Box> { + let (sandbox, mut authority) = + open_authority("filesystem-retention-committed-manifest-corrupt")?; + let root_bytes = fixture(ROOT_HEX)?; + let published = initial_preparation(&root_bytes)?; + let _receipt = execute_retention_publication(&mut authority, &published)?; + let pool_entry = manifest_pool_path(sandbox.path(), &published); + let mut corrupt = fs::read(&pool_entry)?; + *corrupt.last_mut().ok_or("empty manifest pool entry")? ^= 0x01; + fs::write(&pool_entry, &corrupt)?; + let retry = initial_preparation(&root_bytes)?; + + let error = execute_retention_publication(&mut authority, &retry) + .err() + .ok_or("committed retry succeeded although its manifest is corrupt")?; + + assert!(matches!( + error, + RetentionPublicationError::CurrentVerification { .. } + )); + drop(authority); + sandbox.remove()?; + Ok(()) +} diff --git a/src/adapters/retention/filesystem_retention_storage.rs b/src/adapters/retention/filesystem_retention_storage.rs index 9492b65..85dde57 100644 --- a/src/adapters/retention/filesystem_retention_storage.rs +++ b/src/adapters/retention/filesystem_retention_storage.rs @@ -24,7 +24,18 @@ impl RetentionPublicationStorage for FilesystemRetentionPublicationAuthority { self.liveness_generation = Some(preparation.liveness_generation()); require_no_retained_stage(&self.retention)?; let current = filesystem_retention_current::observe(&self.retention, &self.manifests)?; - filesystem_retention_current::disposition(preparation, current.as_ref()) + let disposition = filesystem_retention_current::disposition(preparation, current.as_ref())?; + if disposition == RetentionTransitionDisposition::AlreadyCommitted { + let current = current + .as_ref() + .ok_or_else(|| invalid_data("already-committed disposition without a head"))?; + filesystem_retention_current::verify_committed( + &self.roots, + current, + preparation.candidate(), + )?; + } + Ok(disposition) } fn write_root_stage(&mut self, root: &AdmittedRetentionRoot<'_>) -> io::Result<()> { From 6b14e0b0e5065ff1f8b3fa552b4e773995f31412 Mon Sep 17 00:00:00 2001 From: James Ross Date: Mon, 7 Sep 2026 00:05:15 -0700 Subject: [PATCH 060/111] Fix: admit every version-two protocol directory under the platform profile reopen_version_two delegated to the version-1 platform opener, whose protocol directory list is staging, segments, and catalogs. A migrated store whose retention, gc, or recovery directory was a foreign mount, read-only, casefolded, or non-ext4 still received writer authority. open_version_two admits all nine version-two protocol directories, including the nested retention/roots, retention/manifests, and recovery/dispositions pools, opening each component without following links and requiring the same filesystem type, device, mount identity, and inode flags as the root. Regression laws: a structure pin over VERSION_TWO_PROTOCOL_DIRECTORIES, and two Linux-gated production-path laws (an aliased gc directory refuses; an exact migrated store admits). The Linux laws cross-compile and pass clippy here and run for real on CI. Addresses Codex review thread on filesystem_store_initializer.rs:84. Refs #78 --- CHANGELOG.md | 5 +- docs/formats/segment-store-v2/requirements.md | 2 +- src/adapters/filesystem_platform_profile.rs | 70 +++++++++++++++++-- .../filesystem_platform_profile_tests.rs | 25 ++++++- src/adapters/filesystem_store_initializer.rs | 9 +-- .../filesystem_version_two_admission_tests.rs | 34 +++++++++ 6 files changed, 133 insertions(+), 12 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 51b6587..bdd0537 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -41,7 +41,10 @@ after its public API and format compatibility policies are established. names that refusal. An already-committed retention retry now reopens the manifest entry and the root pool bytes the head selects and refuses absent, changed, or corrupt evidence instead of inferring the commit from head - agreement alone. + agreement alone. On Linux, `reopen_version_two` admits `retention`, + `retention/roots`, `retention/manifests`, `gc`, `recovery`, and + `recovery/dispositions` against the root's filesystem, mount, and inode + flags exactly as the version-1 protocol directories are admitted. - Repository crash-matrix execution now terminates isolated writer process groups at all 105 canonical before/during/after coordinates, retains open writer and stage authority until termination, executes production diff --git a/docs/formats/segment-store-v2/requirements.md b/docs/formats/segment-store-v2/requirements.md index 89b2f85..d06aeaf 100644 --- a/docs/formats/segment-store-v2/requirements.md +++ b/docs/formats/segment-store-v2/requirements.md @@ -35,7 +35,7 @@ case is not evidence. | `KEEP-MIGRATION-005` | Unknown, out-of-order, substituted, corrupt, conflicting, or changed evidence is unrecoverable ambiguity | forward-execution stage preservation, byte-equal inode-substitution, out-of-order-prefix, and post-publication drift laws in `filesystem_migration_storage_tests`; restart corruption and mutation matrix remains | In progress in #19 | | `KEEP-MIGRATION-006` | Migration never rewrites or deletes admitted version-1 immutable bytes | exact segment, catalog, and head before/after witness in `filesystem_migration_storage_tests`; restart-path evidence remains | In progress in #19 | | `KEEP-MIGRATION-007` | Process death around every intent stage, canonical link, namespace prefix, marker stage, receipt stage, cleanup, and synchronization boundary reaches a documented lawful state | ordered phases and capabilities in `tests/store_migration_phase.rs` and `tests/store_migration_storage.rs`; exact phase-failure execution in `tests/store_migration_execution.rs`; production 21-phase forward execution in `filesystem_migration_storage_tests`; `KEEP-CRASH-053..=073` process-death matrix remains | In progress in #19 | -| `KEEP-MIGRATION-008` | Version-1 admission refuses every version-2 or partial-migration artifact after migration begins | `FORMAT` refusal before mutation in `filesystem_migration_authority_tests`; exact version-1 reopen refusal of a migrated root and separate version-2 namespace admission in `filesystem_initialization_namespace`; version-2 reopen jointly admits the exact marker, intent, and receipt before returning writer authority, with corrupt, oversized, and mutually inconsistent record refusals in `filesystem_version_two_admission_tests`; remaining compatibility and fuzz matrix | In progress in #19 | +| `KEEP-MIGRATION-008` | Version-1 admission refuses every version-2 or partial-migration artifact after migration begins | `FORMAT` refusal before mutation in `filesystem_migration_authority_tests`; exact version-1 reopen refusal of a migrated root and separate version-2 namespace admission in `filesystem_initialization_namespace`; version-2 reopen admits every version-2 protocol directory under the Linux profile and jointly admits the exact marker, intent, and receipt before returning writer authority, with aliased-directory, corrupt, oversized, and mutually inconsistent record refusals in `filesystem_version_two_admission_tests` and `filesystem_platform_profile_tests`; remaining compatibility and fuzz matrix | In progress in #19 | diff --git a/src/adapters/filesystem_platform_profile.rs b/src/adapters/filesystem_platform_profile.rs index ca0423c..f0e0ad8 100644 --- a/src/adapters/filesystem_platform_profile.rs +++ b/src/adapters/filesystem_platform_profile.rs @@ -9,6 +9,23 @@ use super::filesystem_root_identity::FilesystemRootIdentity; #[cfg(target_os = "linux")] const PROTOCOL_DIRECTORIES: [&str; 3] = ["staging", "segments", "catalogs"]; +/// Every version-two protocol directory, including nested immutable pools. +/// +/// A migrated root receives writer authority only when each of these shares +/// the root's filesystem type, device, and mount identity and is not +/// casefolded or read-only. +#[cfg(target_os = "linux")] +const VERSION_TWO_PROTOCOL_DIRECTORIES: [&str; 9] = [ + "staging", + "segments", + "catalogs", + "retention", + "retention/roots", + "retention/manifests", + "gc", + "recovery", + "recovery/dispositions", +]; #[cfg(target_os = "linux")] #[derive(Clone, Copy)] @@ -35,7 +52,30 @@ pub(super) fn open(store_root: &Path) -> io::Result { ResolveFlags::NO_MAGICLINKS | ResolveFlags::NO_SYMLINKS, )?; let directory = Dir::from_std_file(File::from(descriptor)); - admit_linux_profile(&directory)?; + admit_linux_profile(&directory, &PROTOCOL_DIRECTORIES)?; + Ok(directory) +} + +/// Opens one version-two store root under the admitted Linux profile. +/// +/// Identical to [`open`], but every version-two protocol directory that +/// exists must satisfy the same filesystem, mount, and inode-flag laws as the +/// root; absence is left to namespace admission. +#[cfg(target_os = "linux")] +pub(super) fn open_version_two(store_root: &Path) -> io::Result { + use std::fs::File; + + use rustix::fs::{CWD, Mode, OFlags, ResolveFlags, openat2}; + + let descriptor = openat2( + CWD, + store_root, + OFlags::RDONLY | OFlags::DIRECTORY | OFlags::CLOEXEC, + Mode::empty(), + ResolveFlags::NO_MAGICLINKS | ResolveFlags::NO_SYMLINKS, + )?; + let directory = Dir::from_std_file(File::from(descriptor)); + admit_linux_profile(&directory, &VERSION_TWO_PROTOCOL_DIRECTORIES)?; Ok(directory) } @@ -47,13 +87,21 @@ pub(super) fn open(_store_root: &Path) -> io::Result { )) } +#[cfg(not(target_os = "linux"))] +pub(super) fn open_version_two(_store_root: &Path) -> io::Result { + Err(io::Error::new( + io::ErrorKind::Unsupported, + "version-two reopen currently requires the admitted Linux ext4 profile", + )) +} + #[cfg(target_os = "linux")] -fn admit_linux_profile(directory: &Dir) -> io::Result<()> { +fn admit_linux_profile(directory: &Dir, protocol_directories: &[&str]) -> io::Result<()> { let file = directory.try_clone()?.into_std_file(); let root = linux_directory_properties(&file)?; admit_linux_properties(root.filesystem_type, root.mount_flags, root.inode_flags)?; - for name in PROTOCOL_DIRECTORIES { - let child = match super::sync_capable_directory::open(directory, name) { + for name in protocol_directories { + let child = match open_protocol_directory(directory, name) { Ok(child) => child, Err(source) if source.kind() == io::ErrorKind::NotFound => continue, Err(source) => return Err(source), @@ -64,6 +112,20 @@ fn admit_linux_profile(directory: &Dir) -> io::Result<()> { file.sync_all() } +/// Opens a possibly nested protocol directory one no-follow component at a time. +#[cfg(target_os = "linux")] +fn open_protocol_directory(root: &Dir, name: &str) -> io::Result { + let mut components = name.split('/'); + let first = components + .next() + .ok_or_else(|| io::Error::new(io::ErrorKind::InvalidInput, "empty protocol name"))?; + let mut current = super::sync_capable_directory::open(root, first)?; + for component in components { + current = super::sync_capable_directory::open(¤t, component)?; + } + Ok(current) +} + #[cfg(target_os = "linux")] fn linux_directory_properties(file: &std::fs::File) -> io::Result { use rustix::fs::{AtFlags, StatxFlags, fstatfs, fstatvfs, ioctl_getflags, statx}; diff --git a/src/adapters/filesystem_platform_profile_tests.rs b/src/adapters/filesystem_platform_profile_tests.rs index 4f4776e..13f97f3 100644 --- a/src/adapters/filesystem_platform_profile_tests.rs +++ b/src/adapters/filesystem_platform_profile_tests.rs @@ -1,8 +1,8 @@ //! Linux filesystem platform-profile laws. use super::{ - LinuxDirectoryProperties, PROTOCOL_DIRECTORIES, admit_linux_child_properties, - admit_linux_properties, linux_root_identity, + LinuxDirectoryProperties, PROTOCOL_DIRECTORIES, VERSION_TWO_PROTOCOL_DIRECTORIES, + admit_linux_child_properties, admit_linux_properties, linux_root_identity, }; use rustix::fs::{NFS_SUPER_MAGIC, StatVfsMountFlags}; @@ -80,3 +80,24 @@ const fn properties( mount_id, } } + +#[test] +fn version_two_admission_covers_every_protocol_directory() { + assert_eq!( + VERSION_TWO_PROTOCOL_DIRECTORIES, + [ + "staging", + "segments", + "catalogs", + "retention", + "retention/roots", + "retention/manifests", + "gc", + "recovery", + "recovery/dispositions", + ] + ); + for name in PROTOCOL_DIRECTORIES { + assert!(VERSION_TWO_PROTOCOL_DIRECTORIES.contains(&name)); + } +} diff --git a/src/adapters/filesystem_store_initializer.rs b/src/adapters/filesystem_store_initializer.rs index dc1240f..12857c7 100644 --- a/src/adapters/filesystem_store_initializer.rs +++ b/src/adapters/filesystem_store_initializer.rs @@ -68,9 +68,10 @@ impl FilesystemPlatformAdmission { /// Reacquires writer authority over one completely migrated version-2 store. /// - /// The call mutates no protocol state. It admits the production platform, - /// acquires the existing writer lock, and requires the exact version-2 root - /// namespace. Retention and recovery adapters perform content-level + /// The call mutates no protocol state. It admits the production platform + /// for every version-2 protocol directory, acquires the existing writer + /// lock, requires the exact version-2 root namespace, and jointly admits the + /// marker, intent, and receipt records. Retention and recovery adapters perform content-level /// validation under the returned authority. The synchronous call may block /// on filesystem I/O. /// @@ -79,7 +80,7 @@ impl FilesystemPlatformAdmission { /// Returns [`FilesystemPlatformAdmissionError`] with the exact platform, /// writer-lock, or namespace boundary and preserved source. pub fn reopen_version_two(store_root: &Path) -> Result { - let root = filesystem_platform_profile::open(store_root) + let root = filesystem_platform_profile::open_version_two(store_root) .map_err(|source| FilesystemPlatformAdmissionError::Platform { source })?; reopen_version_two_root(root) } diff --git a/src/adapters/retention/filesystem_version_two_admission_tests.rs b/src/adapters/retention/filesystem_version_two_admission_tests.rs index c2411eb..5a5f55c 100644 --- a/src/adapters/retention/filesystem_version_two_admission_tests.rs +++ b/src/adapters/retention/filesystem_version_two_admission_tests.rs @@ -78,3 +78,37 @@ fn version_two_reopen_admits_exact_migration_records() -> Result<(), Box Result<(), Box> { + let sandbox = migrated_store("version-two-admission-aliased-gc")?; + let alias_target = sandbox.path().join("elsewhere"); + fs::create_dir(&alias_target)?; + fs::remove_dir(sandbox.path().join("gc"))?; + std::os::unix::fs::symlink(&alias_target, sandbox.path().join("gc"))?; + + let error = FilesystemPlatformAdmission::reopen_version_two(sandbox.path()) + .err() + .ok_or("aliased gc protocol directory was unexpectedly admitted")?; + + assert!(matches!( + error, + FilesystemPlatformAdmissionError::Platform { .. } + )); + sandbox.remove()?; + Ok(()) +} + +#[cfg(target_os = "linux")] +#[test] +fn production_version_two_reopen_admits_an_exact_migrated_store() -> Result<(), Box> { + let sandbox = migrated_store("version-two-admission-production-exact")?; + + let admission = FilesystemPlatformAdmission::reopen_version_two(sandbox.path())?; + + drop(admission); + sandbox.remove()?; + Ok(()) +} From e5cf718c47a3dbb1499fbad127e5ac8356bb08a6 Mon Sep 17 00:00:00 2001 From: James Ross Date: Mon, 7 Sep 2026 00:08:23 -0700 Subject: [PATCH 061/111] Fix: refuse unknown retention namespace entries before any forward write Current-state verification examined only the three stage names, so a foreign entry such as retention/junk, a non-digest directory under retention/roots, or a noncanonical pool filename passed straight into a forward publication from state the format declares ambiguous. filesystem_retention_namespace::admit now requires the complete retention membership to be exactly HEAD, roots, and manifests; every roots entry to be a 64-lowercase-hex directory; and every pool entry to be a regular <16-hex generation>-<64-hex digest> file with its canonical .root or .manifest suffix. Kinds are observed without following links. It runs immediately after the retained-stage check and before the head is observed, so refusal precedes every stage mutation. The census it returns counts namespace directories for the capacity check that follows. Regression laws: unknown entry, non-digest namespace, malformed manifest name, and uppercase root name all refuse with an unchanged retention witness. Addresses Codex review thread on filesystem_retention_storage.rs:145. Refs #78 --- CHANGELOG.md | 6 +- docs/formats/segment-store-v2/requirements.md | 2 +- src/adapters/retention.rs | 3 + .../filesystem_retention_namespace.rs | 97 ++++++++++++++++ .../filesystem_retention_namespace_tests.rs | 105 ++++++++++++++++++ .../retention/filesystem_retention_storage.rs | 3 + 6 files changed, 214 insertions(+), 2 deletions(-) create mode 100644 src/adapters/retention/filesystem_retention_namespace.rs create mode 100644 src/adapters/retention/filesystem_retention_namespace_tests.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index bdd0537..bcbfa37 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -44,7 +44,11 @@ after its public API and format compatibility policies are established. agreement alone. On Linux, `reopen_version_two` admits `retention`, `retention/roots`, `retention/manifests`, `gc`, `recovery`, and `recovery/dispositions` against the root's filesystem, mount, and inode - flags exactly as the version-1 protocol directories are admitted. + flags exactly as the version-1 protocol directories are admitted. Retention + publication admits the complete `retention` namespace before any forward + write: only `HEAD`, `roots`, and `manifests` may exist, every namespace + directory is 64 lowercase hex, and every pool entry is a regular + `-` file with its canonical suffix. - Repository crash-matrix execution now terminates isolated writer process groups at all 105 canonical before/during/after coordinates, retains open writer and stage authority until termination, executes production diff --git a/docs/formats/segment-store-v2/requirements.md b/docs/formats/segment-store-v2/requirements.md index d06aeaf..78bb01d 100644 --- a/docs/formats/segment-store-v2/requirements.md +++ b/docs/formats/segment-store-v2/requirements.md @@ -32,7 +32,7 @@ case is not evidence. | `KEEP-MIGRATION-002` | Format marker, intent, and receipt have complete fixed byte tables, named domains, bounds, checksums, deterministic store identity, and exact initial-state digests | exact admission in `tests/store_format_marker.rs`, `tests/store_migration_intent.rs`, and `tests/store_migration_receipt.rs`; canonical construction in `tests/store_migration_intent_encoding.rs` and `tests/store_migration_receipt_encoding.rs`; seeded `migration_format` fuzz target | Implemented | | `KEEP-MIGRATION-003` | Migration revalidates version-1 head, catalog, pools, root identity, and writer authority before mutation | bounded canonical pool inventory in `tests/store_migration_inventory.rs`; writer-locked filesystem pool admission in `filesystem_inventory_*_tests`; exact authority observation and drift refusal in `filesystem_migration_authority_tests`; verification-first execution in `tests/store_migration_execution.rs`; fresh filesystem integration and post-publication drift refusal in `filesystem_migration_storage_tests` | Implemented | | `KEEP-MIGRATION-004` | Every partial migration prefix continues idempotently under writer authority | state-machine and recovery tests | Planned in #19 | -| `KEEP-MIGRATION-005` | Unknown, out-of-order, substituted, corrupt, conflicting, or changed evidence is unrecoverable ambiguity | forward-execution stage preservation, byte-equal inode-substitution, out-of-order-prefix, and post-publication drift laws in `filesystem_migration_storage_tests`; restart corruption and mutation matrix remains | In progress in #19 | +| `KEEP-MIGRATION-005` | Unknown, out-of-order, substituted, corrupt, conflicting, or changed evidence is unrecoverable ambiguity | forward-execution stage preservation, byte-equal inode-substitution, out-of-order-prefix, and post-publication drift laws in `filesystem_migration_storage_tests`; unknown `retention` entries, non-digest namespace directories, and noncanonical pool names refuse before any retention stage is written in `filesystem_retention_namespace_tests`; restart corruption and mutation matrix remains | In progress in #19 | | `KEEP-MIGRATION-006` | Migration never rewrites or deletes admitted version-1 immutable bytes | exact segment, catalog, and head before/after witness in `filesystem_migration_storage_tests`; restart-path evidence remains | In progress in #19 | | `KEEP-MIGRATION-007` | Process death around every intent stage, canonical link, namespace prefix, marker stage, receipt stage, cleanup, and synchronization boundary reaches a documented lawful state | ordered phases and capabilities in `tests/store_migration_phase.rs` and `tests/store_migration_storage.rs`; exact phase-failure execution in `tests/store_migration_execution.rs`; production 21-phase forward execution in `filesystem_migration_storage_tests`; `KEEP-CRASH-053..=073` process-death matrix remains | In progress in #19 | | `KEEP-MIGRATION-008` | Version-1 admission refuses every version-2 or partial-migration artifact after migration begins | `FORMAT` refusal before mutation in `filesystem_migration_authority_tests`; exact version-1 reopen refusal of a migrated root and separate version-2 namespace admission in `filesystem_initialization_namespace`; version-2 reopen admits every version-2 protocol directory under the Linux profile and jointly admits the exact marker, intent, and receipt before returning writer authority, with aliased-directory, corrupt, oversized, and mutually inconsistent record refusals in `filesystem_version_two_admission_tests` and `filesystem_platform_profile_tests`; remaining compatibility and fuzz matrix | In progress in #19 | diff --git a/src/adapters/retention.rs b/src/adapters/retention.rs index 2e5111e..83c77b5 100644 --- a/src/adapters/retention.rs +++ b/src/adapters/retention.rs @@ -18,6 +18,9 @@ mod filesystem_retention_authority_error; mod filesystem_retention_current; #[cfg(test)] mod filesystem_retention_current_tests; +mod filesystem_retention_namespace; +#[cfg(test)] +mod filesystem_retention_namespace_tests; mod filesystem_retention_pool_name; mod filesystem_retention_stage; mod filesystem_retention_storage; diff --git a/src/adapters/retention/filesystem_retention_namespace.rs b/src/adapters/retention/filesystem_retention_namespace.rs new file mode 100644 index 0000000..2c1d0e8 --- /dev/null +++ b/src/adapters/retention/filesystem_retention_namespace.rs @@ -0,0 +1,97 @@ +//! This module owns exact admission of the `retention` protocol namespace. + +use std::ffi::OsStr; +use std::io; + +use cap_fs_ext::DirExt; +use cap_std::fs::Dir; + +use super::filesystem_retention_pool_name as pool_name; +use super::filesystem_retention_stage::invalid_data; + +const CANONICAL_ENTRIES: [&str; 3] = [pool_name::HEAD, pool_name::ROOTS, pool_name::MANIFESTS]; +const DIGEST_HEX: usize = 64; +const GENERATION_HEX: usize = 16; +const ROOT_SUFFIX: &str = ".root"; +const MANIFEST_SUFFIX: &str = ".manifest"; + +/// Bounded observation of the admitted retention namespace. +#[must_use] +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(super) struct RetentionNamespaceCensus { + /// Digest-named root namespace directories currently present. + pub(super) namespace_count: u32, +} + +/// Admits the complete `retention` namespace before any forward write. +/// +/// Every `retention` entry must be one of `HEAD`, `roots`, or `manifests`; +/// every `roots` entry must be a 64-lowercase-hex directory whose entries are +/// regular `-.root` files; every `manifests` entry must be +/// a regular `-.manifest` file. Kinds are observed without +/// following links. Any other entry is unrecoverable ambiguity and refuses. +pub(super) fn admit( + retention: &Dir, + roots: &Dir, + manifests: &Dir, +) -> io::Result { + for entry in retention.entries()? { + let name = entry?.file_name(); + if !CANONICAL_ENTRIES.iter().any(|canonical| name == *canonical) { + return Err(invalid_data("retention namespace carries an unknown entry")); + } + } + let mut namespace_count = 0_u32; + for entry in roots.entries()? { + let entry = entry?; + let name = entry.file_name(); + if !is_lower_hex(&name, DIGEST_HEX) || !entry.metadata()?.is_dir() { + return Err(invalid_data( + "retention roots carries a non-namespace entry", + )); + } + namespace_count = namespace_count + .checked_add(1) + .ok_or_else(|| invalid_data("retention namespace count overflowed"))?; + let namespace = roots.open_dir_nofollow(&name)?; + admit_pool(&namespace, ROOT_SUFFIX, "retention root pool")?; + } + admit_pool(manifests, MANIFEST_SUFFIX, "retention manifest pool")?; + Ok(RetentionNamespaceCensus { namespace_count }) +} + +fn admit_pool(pool: &Dir, suffix: &str, label: &'static str) -> io::Result<()> { + for entry in pool.entries()? { + let entry = entry?; + if !is_pool_name(&entry.file_name(), suffix) || !entry.metadata()?.is_file() { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + format!("{label} carries a noncanonical entry"), + )); + } + } + Ok(()) +} + +fn is_pool_name(name: &OsStr, suffix: &str) -> bool { + let Some(name) = name.to_str() else { + return false; + }; + let Some(stem) = name.strip_suffix(suffix) else { + return false; + }; + let Some((generation, digest)) = stem.split_once('-') else { + return false; + }; + is_lower_hex(OsStr::new(generation), GENERATION_HEX) + && is_lower_hex(OsStr::new(digest), DIGEST_HEX) +} + +fn is_lower_hex(name: &OsStr, length: usize) -> bool { + name.to_str().is_some_and(|text| { + text.len() == length + && text + .bytes() + .all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte)) + }) +} diff --git a/src/adapters/retention/filesystem_retention_namespace_tests.rs b/src/adapters/retention/filesystem_retention_namespace_tests.rs new file mode 100644 index 0000000..2d8d1e8 --- /dev/null +++ b/src/adapters/retention/filesystem_retention_namespace_tests.rs @@ -0,0 +1,105 @@ +//! Filesystem retention namespace laws: unknown entries are ambiguity, not noise. + +use std::error::Error; +use std::fs; +use std::io; + +use super::RetentionPublicationStorage; +use super::filesystem_retention_test_fixture::{ + ROOT_HEX, fixture, initial_preparation, open_authority, retention_witness, +}; + +#[test] +fn unknown_retention_entry_refuses_before_any_stage_is_written() -> Result<(), Box> { + let (sandbox, mut authority) = open_authority("filesystem-retention-unknown-entry")?; + fs::write( + sandbox.path().join("retention").join("junk"), + b"not protocol state", + )?; + let before = retention_witness(sandbox.path())?; + let root_bytes = fixture(ROOT_HEX)?; + let preparation = initial_preparation(&root_bytes)?; + + let error = RetentionPublicationStorage::verify_current(&mut authority, &preparation) + .err() + .ok_or("unknown retention entry was unexpectedly admitted")?; + + assert_eq!(error.kind(), io::ErrorKind::InvalidData); + assert_eq!(retention_witness(sandbox.path())?, before); + drop(authority); + sandbox.remove()?; + Ok(()) +} + +#[test] +fn non_digest_root_namespace_directory_refuses() -> Result<(), Box> { + let (sandbox, mut authority) = open_authority("filesystem-retention-bad-namespace-name")?; + fs::create_dir( + sandbox + .path() + .join("retention") + .join("roots") + .join("not-a-digest"), + )?; + let root_bytes = fixture(ROOT_HEX)?; + let preparation = initial_preparation(&root_bytes)?; + + let error = RetentionPublicationStorage::verify_current(&mut authority, &preparation) + .err() + .ok_or("non-digest namespace directory was unexpectedly admitted")?; + + assert_eq!(error.kind(), io::ErrorKind::InvalidData); + drop(authority); + sandbox.remove()?; + Ok(()) +} + +#[test] +fn malformed_manifest_pool_name_refuses() -> Result<(), Box> { + let (sandbox, mut authority) = open_authority("filesystem-retention-bad-manifest-name")?; + fs::write( + sandbox + .path() + .join("retention") + .join("manifests") + .join("bogus.manifest"), + b"", + )?; + let root_bytes = fixture(ROOT_HEX)?; + let preparation = initial_preparation(&root_bytes)?; + + let error = RetentionPublicationStorage::verify_current(&mut authority, &preparation) + .err() + .ok_or("malformed manifest pool name was unexpectedly admitted")?; + + assert_eq!(error.kind(), io::ErrorKind::InvalidData); + drop(authority); + sandbox.remove()?; + Ok(()) +} + +#[test] +fn uppercase_root_pool_name_refuses() -> Result<(), Box> { + let (sandbox, mut authority) = open_authority("filesystem-retention-uppercase-root-name")?; + let namespace = sandbox + .path() + .join("retention") + .join("roots") + .join("a".repeat(64)); + fs::create_dir(&namespace)?; + fs::write( + namespace.join(format!("{}-{}.root", "0".repeat(15) + "1", "A".repeat(64))), + b"", + )?; + let root_bytes = fixture(ROOT_HEX)?; + let preparation = initial_preparation(&root_bytes)?; + + let error = RetentionPublicationStorage::verify_current(&mut authority, &preparation) + .err() + .ok_or("uppercase root pool name was unexpectedly admitted")?; + + assert_eq!(error.kind(), io::ErrorKind::InvalidData); + drop(authority); + sandbox.remove()?; + Ok(()) +} diff --git a/src/adapters/retention/filesystem_retention_storage.rs b/src/adapters/retention/filesystem_retention_storage.rs index 85dde57..03fab44 100644 --- a/src/adapters/retention/filesystem_retention_storage.rs +++ b/src/adapters/retention/filesystem_retention_storage.rs @@ -7,6 +7,7 @@ use cap_std::fs::Dir; use super::filesystem_retention_authority::FilesystemRetentionPublicationAuthority; use super::filesystem_retention_current; +use super::filesystem_retention_namespace; use super::filesystem_retention_pool_name as pool_name; use super::filesystem_retention_stage::{FilesystemRetentionStage, invalid_data}; use super::{ @@ -23,6 +24,8 @@ impl RetentionPublicationStorage for FilesystemRetentionPublicationAuthority { ) -> io::Result { self.liveness_generation = Some(preparation.liveness_generation()); require_no_retained_stage(&self.retention)?; + let _census = + filesystem_retention_namespace::admit(&self.retention, &self.roots, &self.manifests)?; let current = filesystem_retention_current::observe(&self.retention, &self.manifests)?; let disposition = filesystem_retention_current::disposition(preparation, current.as_ref())?; if disposition == RetentionTransitionDisposition::AlreadyCommitted { From 43767ed70878778a56280a73cd20dca8a7340eef Mon Sep 17 00:00:00 2001 From: James Ross Date: Mon, 7 Sep 2026 00:12:39 -0700 Subject: [PATCH 062/111] Fix: count orphan namespace directories against the retention ceiling Preparation bounded only current manifest entries, so a store holding 4,096 recovery-protected orphan namespace directories under retention/roots could create a 4,097th during admit_root_namespace, after the root stage had already been written. admit_capacity consumes the namespace census taken during namespace admission: a candidate whose namespace directory is absent is admitted only while the observed count is below RetentionManifest::MAXIMUM_ENTRY_COUNT. It runs inside current-state verification, so capacity refusal precedes every stage mutation. Regression laws: 4,096 foreign namespaces refuse a new candidate with an unchanged witness; a candidate whose namespace already exists creates nothing and is not bounded here. Addresses Codex review thread on filesystem_retention_storage.rs:52. Refs #78 --- CHANGELOG.md | 5 +- docs/formats/segment-store-v2/requirements.md | 2 +- src/adapters/retention.rs | 2 + .../filesystem_retention_capacity_tests.rs | 89 +++++++++++++++++++ .../filesystem_retention_namespace.rs | 30 +++++++ .../retention/filesystem_retention_storage.rs | 7 +- 6 files changed, 132 insertions(+), 3 deletions(-) create mode 100644 src/adapters/retention/filesystem_retention_capacity_tests.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index bcbfa37..13caace 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -48,7 +48,10 @@ after its public API and format compatibility policies are established. publication admits the complete `retention` namespace before any forward write: only `HEAD`, `roots`, and `manifests` may exist, every namespace directory is 64 lowercase hex, and every pool entry is a regular - `-` file with its canonical suffix. + `-` file with its canonical suffix. Existing namespace + directories, including recovery-protected orphans, count against the 4,096 + namespace ceiling, and a candidate whose namespace would be the 4,097th + refuses before its root stage exists. - Repository crash-matrix execution now terminates isolated writer process groups at all 105 canonical before/during/after coordinates, retains open writer and stage authority until termination, executes production diff --git a/docs/formats/segment-store-v2/requirements.md b/docs/formats/segment-store-v2/requirements.md index 78bb01d..648f986 100644 --- a/docs/formats/segment-store-v2/requirements.md +++ b/docs/formats/segment-store-v2/requirements.md @@ -14,7 +14,7 @@ case is not evidence. | `KEEP-RETENTION-003` | Every structural field, truncation boundary, ordering law, duplicate, overflow, flag, reserved byte, digest, checksum, and trailing byte has a precise refusal | seeded `retention_format` fuzz target plus the root, manifest, and head corruption matrix; mutation coverage remains | In progress in #19 | | `KEEP-RETENTION-004` | Retain and release compare expected and observed generations and publish exact successors only | unforgeable readiness and preflight proofs in `tests/retention_transition.rs` and `tests/retention_preflight.rs`; exact successor preparation and complete receipt evidence in `tests/retention_publication_preparation.rs` and `tests/retention_publication_execution.rs`; writer-locked initial filesystem publication in `filesystem_retention_storage_tests`; observed-head successor publication, exact predecessor binding, and absent-head refusal in `filesystem_retention_successor_tests` | Implemented | | `KEEP-RETENTION-005` | Closure derivation is deterministic, bounded, cycle-safe, fail-closed, and verifies complete blob reconstruction | exact accounting, reconstruction, adversarial-catalog, and exhaustive model laws in `tests/retention_closure.rs`; corrupt members refuse through the inherited segment-record admission laws and seeded `segment_format` fuzz target routed by `closure-corruption.md` | Implemented | -| `KEEP-RETENTION-006` | Publication follows the exact ordered durability protocol, including new namespace-directory admission and retention of fixed-stage evidence until head commit, and returns only after cleanup synchronization | typed vocabulary and blocking port in `tests/retention_publication_phase.rs` and `tests/retention_publication_storage.rs`; ordered execution, conditional namespace sync, and all 17 exact storage-fault boundaries in `tests/retention_publication_execution.rs`; production 17-phase forward filesystem execution, exclusive staging, byte-equal inode-substitution refusal, and retained-stage recovery refusal in `filesystem_retention_storage_tests`; crash injection remains | In progress in #19 | +| `KEEP-RETENTION-006` | Publication follows the exact ordered durability protocol, including new namespace-directory admission and retention of fixed-stage evidence until head commit, and returns only after cleanup synchronization | typed vocabulary and blocking port in `tests/retention_publication_phase.rs` and `tests/retention_publication_storage.rs`; ordered execution, conditional namespace sync, and all 17 exact storage-fault boundaries in `tests/retention_publication_execution.rs`; production 17-phase forward filesystem execution, exclusive staging, byte-equal inode-substitution refusal, and retained-stage recovery refusal in `filesystem_retention_storage_tests`; orphan namespace directories count against the 4,096 ceiling and refuse a new namespace before any stage is written in `filesystem_retention_capacity_tests`; crash injection remains | In progress in #19 | | `KEEP-RETENTION-007` | Restart resolves every fixed-stage crash prefix to one documented lawful state or typed ambiguity | debug and release crash matrix | Planned in #19 | | `KEEP-RETENTION-008` | Readers double-collect catalog and retention heads and bind one complete catalog, manifest, and root-generation view under a `ReaderFence` | immutable snapshot and concurrency tests | Planned in #19 | | `KEEP-RETENTION-009` | Exact already-committed retry is idempotent only while its successor remains current | byte-identical planning in `tests/retention_transition.rs`; authority-revalidated zero-mutation retry receipt in `tests/retention_publication_execution.rs`; exact already-committed filesystem retry with a byte-identical retention witness in `filesystem_retention_storage_tests`; superseded-candidate filesystem refusal with zero mutation in `filesystem_retention_successor_tests`; committed retry reopens the head-selected manifest entry and root pool bytes, refusing absent, changed, or corrupt evidence in `filesystem_retention_current_tests` | Implemented | diff --git a/src/adapters/retention.rs b/src/adapters/retention.rs index 83c77b5..6194d49 100644 --- a/src/adapters/retention.rs +++ b/src/adapters/retention.rs @@ -15,6 +15,8 @@ mod closure_profile_error; mod closure_verifier; mod filesystem_retention_authority; mod filesystem_retention_authority_error; +#[cfg(test)] +mod filesystem_retention_capacity_tests; mod filesystem_retention_current; #[cfg(test)] mod filesystem_retention_current_tests; diff --git a/src/adapters/retention/filesystem_retention_capacity_tests.rs b/src/adapters/retention/filesystem_retention_capacity_tests.rs new file mode 100644 index 0000000..f84d898 --- /dev/null +++ b/src/adapters/retention/filesystem_retention_capacity_tests.rs @@ -0,0 +1,89 @@ +//! Filesystem retention capacity laws: orphan namespaces count against the ceiling. + +use std::error::Error; +use std::fmt::Write as _; +use std::fs; +use std::io; +use std::path::Path; + +use super::filesystem_retention_test_fixture::{ + ROOT_HEX, fixture, initial_preparation, open_authority, retention_witness, +}; +use super::{AdmittedRetentionRoot, RetentionPublicationStorage, RetentionTransitionDisposition}; +use crate::RetentionManifest; + +#[test] +fn a_full_namespace_pool_refuses_a_new_namespace_before_staging() -> Result<(), Box> { + let (sandbox, mut authority) = open_authority("filesystem-retention-capacity-full")?; + let root_bytes = fixture(ROOT_HEX)?; + let candidate = AdmittedRetentionRoot::decode(&root_bytes)?; + let own = namespace_hex(&candidate); + create_orphans(sandbox.path(), RetentionManifest::MAXIMUM_ENTRY_COUNT, &own)?; + let before = retention_witness(sandbox.path())?; + let preparation = initial_preparation(&root_bytes)?; + + let error = RetentionPublicationStorage::verify_current(&mut authority, &preparation) + .err() + .ok_or("a 4,097th retention namespace was unexpectedly admitted")?; + + assert_eq!(error.kind(), io::ErrorKind::InvalidData); + assert_eq!(retention_witness(sandbox.path())?, before); + drop(authority); + sandbox.remove()?; + Ok(()) +} + +#[test] +fn a_full_namespace_pool_admits_a_candidate_whose_namespace_exists() -> Result<(), Box> { + let (sandbox, mut authority) = open_authority("filesystem-retention-capacity-existing")?; + let root_bytes = fixture(ROOT_HEX)?; + let candidate = AdmittedRetentionRoot::decode(&root_bytes)?; + let own = namespace_hex(&candidate); + create_orphans( + sandbox.path(), + RetentionManifest::MAXIMUM_ENTRY_COUNT - 1, + &own, + )?; + fs::create_dir(sandbox.path().join("retention").join("roots").join(&own))?; + let preparation = initial_preparation(&root_bytes)?; + + let disposition = RetentionPublicationStorage::verify_current(&mut authority, &preparation)?; + + assert_eq!(disposition, RetentionTransitionDisposition::Publish); + drop(authority); + sandbox.remove()?; + Ok(()) +} + +fn create_orphans(root: &Path, count: u32, exclude: &str) -> Result<(), Box> { + let roots = root.join("retention").join("roots"); + let wanted = usize::try_from(count)?; + for name in (0_u64..) + .map(synthetic_namespace) + .filter(|name| name != exclude) + .take(wanted) + { + fs::create_dir(roots.join(name))?; + } + Ok(()) +} + +fn synthetic_namespace(seed: u64) -> String { + let mut name = String::with_capacity(64); + let _ = write!(name, "{seed:016x}"); + name.push_str(&"0".repeat(48)); + name +} + +fn namespace_hex(candidate: &AdmittedRetentionRoot<'_>) -> String { + candidate + .root() + .namespace() + .digest() + .as_bytes() + .iter() + .fold(String::new(), |mut rendered, byte| { + let _ = write!(rendered, "{byte:02x}"); + rendered + }) +} diff --git a/src/adapters/retention/filesystem_retention_namespace.rs b/src/adapters/retention/filesystem_retention_namespace.rs index 2c1d0e8..67a975f 100644 --- a/src/adapters/retention/filesystem_retention_namespace.rs +++ b/src/adapters/retention/filesystem_retention_namespace.rs @@ -6,8 +6,10 @@ use std::io; use cap_fs_ext::DirExt; use cap_std::fs::Dir; +use super::AdmittedRetentionRoot; use super::filesystem_retention_pool_name as pool_name; use super::filesystem_retention_stage::invalid_data; +use crate::RetentionManifest; const CANONICAL_ENTRIES: [&str; 3] = [pool_name::HEAD, pool_name::ROOTS, pool_name::MANIFESTS]; const DIGEST_HEX: usize = 64; @@ -60,6 +62,34 @@ pub(super) fn admit( Ok(RetentionNamespaceCensus { namespace_count }) } +/// Refuses a candidate that would create a namespace beyond the format ceiling. +/// +/// Orphan namespace directories protected by recovery count exactly like +/// manifest entries: a candidate whose namespace directory is absent may be +/// admitted only while the observed count is below +/// [`RetentionManifest::MAXIMUM_ENTRY_COUNT`]. A candidate whose namespace +/// already exists creates nothing and is not bounded here. +pub(super) fn admit_capacity( + census: RetentionNamespaceCensus, + roots: &Dir, + candidate: &AdmittedRetentionRoot<'_>, +) -> io::Result<()> { + let name = pool_name::namespace(candidate.root().namespace().digest()); + match roots.symlink_metadata(&name) { + Ok(_) => Ok(()), + Err(source) if source.kind() == io::ErrorKind::NotFound => { + if census.namespace_count < RetentionManifest::MAXIMUM_ENTRY_COUNT { + Ok(()) + } else { + Err(invalid_data( + "retention namespace pool is at its maximum count", + )) + } + } + Err(source) => Err(source), + } +} + fn admit_pool(pool: &Dir, suffix: &str, label: &'static str) -> io::Result<()> { for entry in pool.entries()? { let entry = entry?; diff --git a/src/adapters/retention/filesystem_retention_storage.rs b/src/adapters/retention/filesystem_retention_storage.rs index 03fab44..9c0037a 100644 --- a/src/adapters/retention/filesystem_retention_storage.rs +++ b/src/adapters/retention/filesystem_retention_storage.rs @@ -24,8 +24,13 @@ impl RetentionPublicationStorage for FilesystemRetentionPublicationAuthority { ) -> io::Result { self.liveness_generation = Some(preparation.liveness_generation()); require_no_retained_stage(&self.retention)?; - let _census = + let census = filesystem_retention_namespace::admit(&self.retention, &self.roots, &self.manifests)?; + filesystem_retention_namespace::admit_capacity( + census, + &self.roots, + preparation.candidate(), + )?; let current = filesystem_retention_current::observe(&self.retention, &self.manifests)?; let disposition = filesystem_retention_current::disposition(preparation, current.as_ref())?; if disposition == RetentionTransitionDisposition::AlreadyCommitted { From 149adc7ce677f888de8e0afa3c9969b648809882 Mon Sep 17 00:00:00 2001 From: James Ross Date: Mon, 7 Sep 2026 00:20:18 -0700 Subject: [PATCH 063/111] Fix: bind absent-head and namespace-directory state to the claimed expectation Two self-audit findings on current-state verification, fixed together because both live in the same preflight and express one invariant: on-disk retention state must match the expectation the transition was planned against. Absent head. verify_current treated a missing retention/HEAD as the empty state on the Absent expectation alone. recovery.md declares absence the canonical empty state only while all retention stages and pools are empty; any artifact routes through recovery. The namespace census now also counts manifest pool entries, an absent head over any artifact refuses as recovery-required, and an absent head admits only a prepared head at the initial liveness generation with no predecessor, so a stale manifest cannot revive a dead chain. Namespace directory. admit_root_namespace mapped AlreadyExists to Existing for every expectation, so an orphan or substituted namespace directory was admitted as idempotent for a namespace expected absent. admit_expectation requires the directory to be absent for Absent and present for Current. It runs, with the capacity check, only on the Publish path once the disposition is known: an exact already-committed retry legitimately finds its namespace present and is verified by reopening its evidence instead. Regression laws: absent head over populated pools; orphan directory for a new namespace beside a live manifest; absent directory for a namespace expected current. All refuse with an unchanged retention witness. The capacity law that relied on a pre-created directory under Absent was rewritten as a successor in an existing namespace, which is the state it meant to describe. KEEP-RETENTION-007 moves from Planned to In progress. Refs #78 --- CHANGELOG.md | 7 +- docs/formats/segment-store-v2/requirements.md | 2 +- src/adapters/retention.rs | 2 + .../filesystem_retention_capacity_tests.rs | 26 +++-- .../retention/filesystem_retention_current.rs | 22 ++++- .../filesystem_retention_expectation_tests.rs | 95 +++++++++++++++++++ .../filesystem_retention_namespace.rs | 60 ++++++++++-- .../retention/filesystem_retention_storage.rs | 22 ++++- .../filesystem_retention_test_fixture.rs | 37 +++++++- 9 files changed, 249 insertions(+), 24 deletions(-) create mode 100644 src/adapters/retention/filesystem_retention_expectation_tests.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index 13caace..370481b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -51,7 +51,12 @@ after its public API and format compatibility policies are established. `-` file with its canonical suffix. Existing namespace directories, including recovery-protected orphans, count against the 4,096 namespace ceiling, and a candidate whose namespace would be the 4,097th - refuses before its root stage exists. + refuses before its root stage exists. Current-state verification now binds + on-disk state to the claimed expectation: an absent `retention/HEAD` is the + empty state only while both pools are empty and admits only an initial head + with no predecessor; a namespace directory must be absent for an `Absent` + expectation and present for a `Current` one. Every mismatch refuses as + recovery-required before any stage is written. - Repository crash-matrix execution now terminates isolated writer process groups at all 105 canonical before/during/after coordinates, retains open writer and stage authority until termination, executes production diff --git a/docs/formats/segment-store-v2/requirements.md b/docs/formats/segment-store-v2/requirements.md index 648f986..f89df5b 100644 --- a/docs/formats/segment-store-v2/requirements.md +++ b/docs/formats/segment-store-v2/requirements.md @@ -15,7 +15,7 @@ case is not evidence. | `KEEP-RETENTION-004` | Retain and release compare expected and observed generations and publish exact successors only | unforgeable readiness and preflight proofs in `tests/retention_transition.rs` and `tests/retention_preflight.rs`; exact successor preparation and complete receipt evidence in `tests/retention_publication_preparation.rs` and `tests/retention_publication_execution.rs`; writer-locked initial filesystem publication in `filesystem_retention_storage_tests`; observed-head successor publication, exact predecessor binding, and absent-head refusal in `filesystem_retention_successor_tests` | Implemented | | `KEEP-RETENTION-005` | Closure derivation is deterministic, bounded, cycle-safe, fail-closed, and verifies complete blob reconstruction | exact accounting, reconstruction, adversarial-catalog, and exhaustive model laws in `tests/retention_closure.rs`; corrupt members refuse through the inherited segment-record admission laws and seeded `segment_format` fuzz target routed by `closure-corruption.md` | Implemented | | `KEEP-RETENTION-006` | Publication follows the exact ordered durability protocol, including new namespace-directory admission and retention of fixed-stage evidence until head commit, and returns only after cleanup synchronization | typed vocabulary and blocking port in `tests/retention_publication_phase.rs` and `tests/retention_publication_storage.rs`; ordered execution, conditional namespace sync, and all 17 exact storage-fault boundaries in `tests/retention_publication_execution.rs`; production 17-phase forward filesystem execution, exclusive staging, byte-equal inode-substitution refusal, and retained-stage recovery refusal in `filesystem_retention_storage_tests`; orphan namespace directories count against the 4,096 ceiling and refuse a new namespace before any stage is written in `filesystem_retention_capacity_tests`; crash injection remains | In progress in #19 | -| `KEEP-RETENTION-007` | Restart resolves every fixed-stage crash prefix to one documented lawful state or typed ambiguity | debug and release crash matrix | Planned in #19 | +| `KEEP-RETENTION-007` | Restart resolves every fixed-stage crash prefix to one documented lawful state or typed ambiguity | recovery-required refusals before any mutation in `filesystem_retention_expectation_tests`: an absent head over populated pools, a non-initial head prepared against an absent head, an orphan directory for a namespace expected absent, and an absent directory for a namespace expected current; debug and release crash matrix remains | In progress in #19 | | `KEEP-RETENTION-008` | Readers double-collect catalog and retention heads and bind one complete catalog, manifest, and root-generation view under a `ReaderFence` | immutable snapshot and concurrency tests | Planned in #19 | | `KEEP-RETENTION-009` | Exact already-committed retry is idempotent only while its successor remains current | byte-identical planning in `tests/retention_transition.rs`; authority-revalidated zero-mutation retry receipt in `tests/retention_publication_execution.rs`; exact already-committed filesystem retry with a byte-identical retention witness in `filesystem_retention_storage_tests`; superseded-candidate filesystem refusal with zero mutation in `filesystem_retention_successor_tests`; committed retry reopens the head-selected manifest entry and root pool bytes, refusing absent, changed, or corrupt evidence in `filesystem_retention_current_tests` | Implemented | | `KEEP-RETENTION-010` | Model operation sequences agree with a deterministic namespace-to-anchor-set map and never admit caller identity, paths, clocks, or application policy | model-based and source-architecture tests | Planned in #19 | diff --git a/src/adapters/retention.rs b/src/adapters/retention.rs index 6194d49..faabd1d 100644 --- a/src/adapters/retention.rs +++ b/src/adapters/retention.rs @@ -20,6 +20,8 @@ mod filesystem_retention_capacity_tests; mod filesystem_retention_current; #[cfg(test)] mod filesystem_retention_current_tests; +#[cfg(test)] +mod filesystem_retention_expectation_tests; mod filesystem_retention_namespace; #[cfg(test)] mod filesystem_retention_namespace_tests; diff --git a/src/adapters/retention/filesystem_retention_capacity_tests.rs b/src/adapters/retention/filesystem_retention_capacity_tests.rs index f84d898..06b3a4b 100644 --- a/src/adapters/retention/filesystem_retention_capacity_tests.rs +++ b/src/adapters/retention/filesystem_retention_capacity_tests.rs @@ -8,9 +8,13 @@ use std::path::Path; use super::filesystem_retention_test_fixture::{ ROOT_HEX, fixture, initial_preparation, open_authority, retention_witness, + successor_preparation, successor_root, }; -use super::{AdmittedRetentionRoot, RetentionPublicationStorage, RetentionTransitionDisposition}; -use crate::RetentionManifest; +use super::{ + AdmittedRetentionManifest, AdmittedRetentionRoot, RetentionPublicationStorage, + RetentionTransitionDisposition, +}; +use crate::{RetentionManifest, execute_retention_publication}; #[test] fn a_full_namespace_pool_refuses_a_new_namespace_before_staging() -> Result<(), Box> { @@ -34,18 +38,24 @@ fn a_full_namespace_pool_refuses_a_new_namespace_before_staging() -> Result<(), } #[test] -fn a_full_namespace_pool_admits_a_candidate_whose_namespace_exists() -> Result<(), Box> { +fn a_full_namespace_pool_admits_a_successor_in_an_existing_namespace() -> Result<(), Box> +{ let (sandbox, mut authority) = open_authority("filesystem-retention-capacity-existing")?; let root_bytes = fixture(ROOT_HEX)?; - let candidate = AdmittedRetentionRoot::decode(&root_bytes)?; - let own = namespace_hex(&candidate); + let _published = + execute_retention_publication(&mut authority, &initial_preparation(&root_bytes)?)?; + let current = authority + .observe_current()? + .ok_or("published retention head was not observed")?; + let current_root = AdmittedRetentionRoot::decode(&root_bytes)?; + let current_manifest = AdmittedRetentionManifest::decode(current.manifest_bytes())?; create_orphans( sandbox.path(), RetentionManifest::MAXIMUM_ENTRY_COUNT - 1, - &own, + &namespace_hex(¤t_root), )?; - fs::create_dir(sandbox.path().join("retention").join("roots").join(&own))?; - let preparation = initial_preparation(&root_bytes)?; + let candidate = successor_root(¤t_root)?; + let preparation = successor_preparation(¤t_root, ¤t_manifest, candidate.encoded())?; let disposition = RetentionPublicationStorage::verify_current(&mut authority, &preparation)?; diff --git a/src/adapters/retention/filesystem_retention_current.rs b/src/adapters/retention/filesystem_retention_current.rs index 17e35ad..b72bbcf 100644 --- a/src/adapters/retention/filesystem_retention_current.rs +++ b/src/adapters/retention/filesystem_retention_current.rs @@ -85,7 +85,7 @@ pub(super) fn disposition( ) -> io::Result { let Some(current) = current else { return match preparation.expected() { - RetentionGenerationExpectation::Absent => Ok(RetentionTransitionDisposition::Publish), + RetentionGenerationExpectation::Absent => require_initial_publication(preparation), RetentionGenerationExpectation::Current(_) => Err(invalid_data( "expected a current retention generation but no head is published", )), @@ -163,6 +163,26 @@ pub(super) fn verify_committed( } } +/// The empty retention state admits only a generation-one head with no predecessor. +fn require_initial_publication( + preparation: &RetentionPublicationPreparation<'_>, +) -> io::Result { + let publication = preparation + .publication() + .ok_or_else(|| invalid_data("already-committed retry against an absent retention head"))?; + let prepared = ChecksummedRetentionHead::decode(publication.head().encoded()) + .map_err(|_source| invalid_data("prepared retention head refused admission"))?; + if prepared.head().generation() == crate::LivenessGeneration::INITIAL + && prepared.head().predecessor().is_none() + { + Ok(RetentionTransitionDisposition::Publish) + } else { + Err(invalid_data( + "absent retention head admits only an initial publication with no predecessor", + )) + } +} + fn read_exact_optional( directory: &Dir, name: &str, diff --git a/src/adapters/retention/filesystem_retention_expectation_tests.rs b/src/adapters/retention/filesystem_retention_expectation_tests.rs new file mode 100644 index 0000000..6e54cf6 --- /dev/null +++ b/src/adapters/retention/filesystem_retention_expectation_tests.rs @@ -0,0 +1,95 @@ +//! Filesystem retention expectation laws: the head and namespace must match the claim. + +use std::error::Error; +use std::fs; +use std::io; + +use super::filesystem_retention_test_fixture::{ + ROOT_HEX, fixture, head_path, initial_preparation, initial_root, new_namespace_preparation, + open_authority, retention_witness, root_pool_path, successor_preparation, successor_root, +}; +use super::{AdmittedRetentionManifest, AdmittedRetentionRoot, RetentionPublicationStorage}; +use crate::execute_retention_publication; + +#[test] +fn absent_head_with_retention_artifacts_refuses_as_recovery() -> Result<(), Box> { + let (sandbox, mut authority) = open_authority("filesystem-retention-absent-head-artifacts")?; + let root_bytes = fixture(ROOT_HEX)?; + let published = initial_preparation(&root_bytes)?; + let _receipt = execute_retention_publication(&mut authority, &published)?; + fs::remove_file(head_path(sandbox.path()))?; + let before = retention_witness(sandbox.path())?; + let retry = initial_preparation(&root_bytes)?; + + let error = RetentionPublicationStorage::verify_current(&mut authority, &retry) + .err() + .ok_or("absent head over populated pools was unexpectedly treated as empty")?; + + assert_eq!(error.kind(), io::ErrorKind::InvalidData); + assert_eq!(retention_witness(sandbox.path())?, before); + drop(authority); + sandbox.remove()?; + Ok(()) +} + +#[test] +fn absent_expectation_refuses_an_orphan_directory_for_a_new_namespace() -> Result<(), Box> +{ + let (sandbox, mut authority) = open_authority("filesystem-retention-orphan-new-namespace")?; + let root_bytes = fixture(ROOT_HEX)?; + let _published = + execute_retention_publication(&mut authority, &initial_preparation(&root_bytes)?)?; + let current = authority + .observe_current()? + .ok_or("published retention head was not observed")?; + let current_manifest = AdmittedRetentionManifest::decode(current.manifest_bytes())?; + let template = AdmittedRetentionRoot::decode(&root_bytes)?; + let candidate = initial_root(b"second-namespace", &template)?; + let preparation = new_namespace_preparation(¤t_manifest, candidate.encoded())?; + let namespace = root_pool_path(sandbox.path(), preparation.candidate()) + .parent() + .ok_or("root pool path has no namespace directory")? + .to_path_buf(); + fs::create_dir(&namespace)?; + let before = retention_witness(sandbox.path())?; + + let error = RetentionPublicationStorage::verify_current(&mut authority, &preparation) + .err() + .ok_or("orphan namespace directory was unexpectedly admitted for an Absent expectation")?; + + assert_eq!(error.kind(), io::ErrorKind::InvalidData); + assert_eq!(retention_witness(sandbox.path())?, before); + drop(authority); + sandbox.remove()?; + Ok(()) +} + +#[test] +fn current_expectation_refuses_when_the_namespace_directory_is_absent() -> Result<(), Box> +{ + let (sandbox, mut authority) = open_authority("filesystem-retention-current-namespace-absent")?; + let root_bytes = fixture(ROOT_HEX)?; + let _published = + execute_retention_publication(&mut authority, &initial_preparation(&root_bytes)?)?; + let current = authority + .observe_current()? + .ok_or("published retention head was not observed")?; + let current_root = AdmittedRetentionRoot::decode(&root_bytes)?; + let current_manifest = AdmittedRetentionManifest::decode(current.manifest_bytes())?; + let namespace = root_pool_path(sandbox.path(), ¤t_root) + .parent() + .ok_or("root pool path has no namespace directory")? + .to_path_buf(); + fs::remove_dir_all(&namespace)?; + let candidate = successor_root(¤t_root)?; + let preparation = successor_preparation(¤t_root, ¤t_manifest, candidate.encoded())?; + + let error = RetentionPublicationStorage::verify_current(&mut authority, &preparation) + .err() + .ok_or("successor over an absent namespace directory was unexpectedly admitted")?; + + assert_eq!(error.kind(), io::ErrorKind::InvalidData); + drop(authority); + sandbox.remove()?; + Ok(()) +} diff --git a/src/adapters/retention/filesystem_retention_namespace.rs b/src/adapters/retention/filesystem_retention_namespace.rs index 67a975f..7c5ca7a 100644 --- a/src/adapters/retention/filesystem_retention_namespace.rs +++ b/src/adapters/retention/filesystem_retention_namespace.rs @@ -9,7 +9,7 @@ use cap_std::fs::Dir; use super::AdmittedRetentionRoot; use super::filesystem_retention_pool_name as pool_name; use super::filesystem_retention_stage::invalid_data; -use crate::RetentionManifest; +use crate::{RetentionGenerationExpectation, RetentionManifest}; const CANONICAL_ENTRIES: [&str; 3] = [pool_name::HEAD, pool_name::ROOTS, pool_name::MANIFESTS]; const DIGEST_HEX: usize = 64; @@ -23,6 +23,15 @@ const MANIFEST_SUFFIX: &str = ".manifest"; pub(super) struct RetentionNamespaceCensus { /// Digest-named root namespace directories currently present. pub(super) namespace_count: u32, + /// Canonical manifest pool entries currently present. + pub(super) manifest_count: u32, +} + +impl RetentionNamespaceCensus { + /// Returns whether no retention artifact exists in either pool. + pub(super) const fn is_empty(self) -> bool { + self.namespace_count == 0 && self.manifest_count == 0 + } } /// Admits the complete `retention` namespace before any forward write. @@ -56,10 +65,13 @@ pub(super) fn admit( .checked_add(1) .ok_or_else(|| invalid_data("retention namespace count overflowed"))?; let namespace = roots.open_dir_nofollow(&name)?; - admit_pool(&namespace, ROOT_SUFFIX, "retention root pool")?; + let _roots = admit_pool(&namespace, ROOT_SUFFIX, "retention root pool")?; } - admit_pool(manifests, MANIFEST_SUFFIX, "retention manifest pool")?; - Ok(RetentionNamespaceCensus { namespace_count }) + let manifest_count = admit_pool(manifests, MANIFEST_SUFFIX, "retention manifest pool")?; + Ok(RetentionNamespaceCensus { + namespace_count, + manifest_count, + }) } /// Refuses a candidate that would create a namespace beyond the format ceiling. @@ -90,7 +102,40 @@ pub(super) fn admit_capacity( } } -fn admit_pool(pool: &Dir, suffix: &str, label: &'static str) -> io::Result<()> { +/// Requires the candidate's namespace directory to match the claimed expectation. +/// +/// An `Absent` expectation asserts the namespace has never been published, so +/// any existing directory is an orphan or a substitution and refuses. A +/// `Current` expectation asserts a published generation, so an absent +/// directory means the claimed predecessor is unavailable and refuses. +pub(super) fn admit_expectation( + roots: &Dir, + candidate: &AdmittedRetentionRoot<'_>, + expected: RetentionGenerationExpectation, +) -> io::Result<()> { + let name = pool_name::namespace(candidate.root().namespace().digest()); + let observed = match roots.symlink_metadata(&name) { + Ok(metadata) => Some(metadata.is_dir()), + Err(source) if source.kind() == io::ErrorKind::NotFound => None, + Err(source) => return Err(source), + }; + match (expected, observed) { + (RetentionGenerationExpectation::Absent, None) + | (RetentionGenerationExpectation::Current(_), Some(true)) => Ok(()), + (RetentionGenerationExpectation::Absent, Some(_)) => Err(invalid_data( + "namespace directory exists although the namespace is expected absent", + )), + (RetentionGenerationExpectation::Current(_), None) => Err(invalid_data( + "namespace directory is absent although a current generation is expected", + )), + (RetentionGenerationExpectation::Current(_), Some(false)) => { + Err(invalid_data("namespace entry is not a directory")) + } + } +} + +fn admit_pool(pool: &Dir, suffix: &str, label: &'static str) -> io::Result { + let mut count = 0_u32; for entry in pool.entries()? { let entry = entry?; if !is_pool_name(&entry.file_name(), suffix) || !entry.metadata()?.is_file() { @@ -99,8 +144,11 @@ fn admit_pool(pool: &Dir, suffix: &str, label: &'static str) -> io::Result<()> { format!("{label} carries a noncanonical entry"), )); } + count = count + .checked_add(1) + .ok_or_else(|| invalid_data("retention pool entry count overflowed"))?; } - Ok(()) + Ok(count) } fn is_pool_name(name: &OsStr, suffix: &str) -> bool { diff --git a/src/adapters/retention/filesystem_retention_storage.rs b/src/adapters/retention/filesystem_retention_storage.rs index 9c0037a..a2f14bb 100644 --- a/src/adapters/retention/filesystem_retention_storage.rs +++ b/src/adapters/retention/filesystem_retention_storage.rs @@ -26,13 +26,25 @@ impl RetentionPublicationStorage for FilesystemRetentionPublicationAuthority { require_no_retained_stage(&self.retention)?; let census = filesystem_retention_namespace::admit(&self.retention, &self.roots, &self.manifests)?; - filesystem_retention_namespace::admit_capacity( - census, - &self.roots, - preparation.candidate(), - )?; let current = filesystem_retention_current::observe(&self.retention, &self.manifests)?; + if current.is_none() && !census.is_empty() { + return Err(invalid_data( + "retention head is absent while retention pools hold artifacts; recovery is required", + )); + } let disposition = filesystem_retention_current::disposition(preparation, current.as_ref())?; + if disposition == RetentionTransitionDisposition::Publish { + filesystem_retention_namespace::admit_expectation( + &self.roots, + preparation.candidate(), + preparation.expected(), + )?; + filesystem_retention_namespace::admit_capacity( + census, + &self.roots, + preparation.candidate(), + )?; + } if disposition == RetentionTransitionDisposition::AlreadyCommitted { let current = current .as_ref() diff --git a/src/adapters/retention/filesystem_retention_test_fixture.rs b/src/adapters/retention/filesystem_retention_test_fixture.rs index 2857af4..4bb83b9 100644 --- a/src/adapters/retention/filesystem_retention_test_fixture.rs +++ b/src/adapters/retention/filesystem_retention_test_fixture.rs @@ -22,8 +22,9 @@ use crate::adapters::{ SegmentReadPolicy, SegmentRecordLimit, }; use crate::{ - RetentionGenerationExpectation, RetentionPolicy, RetentionRoot, RootGeneration, - execute_store_migration, preflight_retention_transition, prepare_retention_publication, + RetentionGenerationExpectation, RetentionNamespace, RetentionPolicy, RetentionRoot, + RootGeneration, execute_store_migration, preflight_retention_transition, + prepare_retention_publication, }; /// Frozen canonical generation-one root. @@ -123,6 +124,38 @@ pub(super) fn successor_preparation<'encoded>( prepare_retention_publication(preflight, Some(current_manifest)).map_err(Into::into) } +/// Builds a generation-one root for another namespace from `template`'s policy. +pub(super) fn initial_root( + namespace: &[u8], + template: &AdmittedRetentionRoot<'_>, +) -> Result> { + let root = RetentionRoot::new( + RetentionNamespace::try_from(namespace)?, + RootGeneration::INITIAL, + RetentionPolicy::new(template.root().profile(), template.root().limits()), + None, + template.root().anchors().to_vec(), + )?; + CanonicalRetentionRoot::from_root(&root).map_err(Into::into) +} + +/// Prepares `candidate_bytes` as a new namespace inserted into `current_manifest`. +pub(super) fn new_namespace_preparation<'encoded>( + current_manifest: &AdmittedRetentionManifest<'_>, + candidate_bytes: &'encoded [u8], +) -> Result, Box> { + let candidate = AdmittedRetentionRoot::decode(candidate_bytes)?; + let preflight = with_snapshot(|snapshot| { + preflight_retention_transition( + RetentionGenerationExpectation::Absent, + None, + candidate, + snapshot, + ) + })??; + prepare_retention_publication(preflight, Some(current_manifest)).map_err(Into::into) +} + /// Builds the exact semantic successor of one admitted root. pub(super) fn successor_root( current: &AdmittedRetentionRoot<'_>, From 87235da658e4405c2a62154444c1de0f7aa5a22c Mon Sep 17 00:00:00 2001 From: James Ross Date: Mon, 7 Sep 2026 00:24:49 -0700 Subject: [PATCH 064/111] Fix: give version-two writer authority its own admission type FilesystemPlatformAdmission carried no version marker, so the version-one FilesystemCatalogPublisher::open accepted an admission produced by the version-two reopen path and could run version-one catalog publication on a migrated root. A crash between head.next creation and HEAD replacement would then leave a thirteenth root entry that neither version-one reopen, version-two reopen, nor the recovery discarder admits, stranding the store. FilesystemVersionTwoAdmission is a distinct type produced only by its own reopen (platform profile over every version-two protocol directory, writer lock, exact version-two namespace, root identity probe, joint migration-record admission). FilesystemRetentionPublicationAuthority::open accepts only that type; FilesystemCatalogPublisher and FilesystemStoreMigrationAuthority accept only FilesystemPlatformAdmission. The mismatch is a compile error. tests/version_two_admission_contract.rs pins the boundary in the repository's source-contract style: the retention authority names only the version-two type, the version-one publishers name only the version-one type, and the version-one initializer no longer exposes a version-two constructor. Self-audit finding S1 (P0). Refs #78 --- CHANGELOG.md | 7 +- docs/formats/segment-store-v2/requirements.md | 2 +- src/adapters/filesystem_store_initializer.rs | 46 +---------- .../filesystem_version_two_admission.rs | 76 +++++++++++++++++++ src/adapters/mod.rs | 2 + .../filesystem_retention_authority.rs | 9 ++- .../filesystem_retention_test_fixture.rs | 5 +- .../filesystem_version_two_admission_tests.rs | 15 ++-- src/lib.rs | 42 +++++----- tests/version_two_admission_contract.rs | 33 ++++++++ 10 files changed, 153 insertions(+), 84 deletions(-) create mode 100644 src/adapters/filesystem_version_two_admission.rs create mode 100644 tests/version_two_admission_contract.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index 370481b..846de26 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -34,14 +34,15 @@ after its public API and format compatibility policies are established. pool manifest, and current-state verification admits a successor only when the prepared head names the observed manifest as its exact predecessor at the next liveness generation; a superseded candidate refuses with zero mutation. - `reopen_version_two` reopens `FORMAT`, `migration.intent`, and + `FilesystemVersionTwoAdmission::reopen` reopens `FORMAT`, `migration.intent`, and `migration.receipt` without following links, bounds each to its canonical length, and admits the receipt only against the decoded intent and marker before returning writer authority; `FilesystemPlatformAdmissionError::MigrationRecord` - names that refusal. An already-committed retention retry now reopens the + names that refusal. Version-two writer authority is its own type, so no + version-one publisher can consume it. An already-committed retention retry now reopens the manifest entry and the root pool bytes the head selects and refuses absent, changed, or corrupt evidence instead of inferring the commit from head - agreement alone. On Linux, `reopen_version_two` admits `retention`, + agreement alone. On Linux, `FilesystemVersionTwoAdmission::reopen` admits `retention`, `retention/roots`, `retention/manifests`, `gc`, `recovery`, and `recovery/dispositions` against the root's filesystem, mount, and inode flags exactly as the version-1 protocol directories are admitted. Retention diff --git a/docs/formats/segment-store-v2/requirements.md b/docs/formats/segment-store-v2/requirements.md index f89df5b..8a3f1bb 100644 --- a/docs/formats/segment-store-v2/requirements.md +++ b/docs/formats/segment-store-v2/requirements.md @@ -35,7 +35,7 @@ case is not evidence. | `KEEP-MIGRATION-005` | Unknown, out-of-order, substituted, corrupt, conflicting, or changed evidence is unrecoverable ambiguity | forward-execution stage preservation, byte-equal inode-substitution, out-of-order-prefix, and post-publication drift laws in `filesystem_migration_storage_tests`; unknown `retention` entries, non-digest namespace directories, and noncanonical pool names refuse before any retention stage is written in `filesystem_retention_namespace_tests`; restart corruption and mutation matrix remains | In progress in #19 | | `KEEP-MIGRATION-006` | Migration never rewrites or deletes admitted version-1 immutable bytes | exact segment, catalog, and head before/after witness in `filesystem_migration_storage_tests`; restart-path evidence remains | In progress in #19 | | `KEEP-MIGRATION-007` | Process death around every intent stage, canonical link, namespace prefix, marker stage, receipt stage, cleanup, and synchronization boundary reaches a documented lawful state | ordered phases and capabilities in `tests/store_migration_phase.rs` and `tests/store_migration_storage.rs`; exact phase-failure execution in `tests/store_migration_execution.rs`; production 21-phase forward execution in `filesystem_migration_storage_tests`; `KEEP-CRASH-053..=073` process-death matrix remains | In progress in #19 | -| `KEEP-MIGRATION-008` | Version-1 admission refuses every version-2 or partial-migration artifact after migration begins | `FORMAT` refusal before mutation in `filesystem_migration_authority_tests`; exact version-1 reopen refusal of a migrated root and separate version-2 namespace admission in `filesystem_initialization_namespace`; version-2 reopen admits every version-2 protocol directory under the Linux profile and jointly admits the exact marker, intent, and receipt before returning writer authority, with aliased-directory, corrupt, oversized, and mutually inconsistent record refusals in `filesystem_version_two_admission_tests` and `filesystem_platform_profile_tests`; remaining compatibility and fuzz matrix | In progress in #19 | +| `KEEP-MIGRATION-008` | Version-1 admission refuses every version-2 or partial-migration artifact after migration begins | `FORMAT` refusal before mutation in `filesystem_migration_authority_tests`; exact version-1 reopen refusal of a migrated root and separate version-2 namespace admission in `filesystem_initialization_namespace`; version-2 reopen returns a distinct `FilesystemVersionTwoAdmission` that no version-1 publisher can consume (pinned by `tests/version_two_admission_contract.rs`), admits every version-2 protocol directory under the Linux profile, and jointly admits the exact marker, intent, and receipt before returning writer authority, with aliased-directory, corrupt, oversized, and mutually inconsistent record refusals in `filesystem_version_two_admission_tests` and `filesystem_platform_profile_tests`; remaining compatibility and fuzz matrix | In progress in #19 | diff --git a/src/adapters/filesystem_store_initializer.rs b/src/adapters/filesystem_store_initializer.rs index 12857c7..ff2fb3a 100644 --- a/src/adapters/filesystem_store_initializer.rs +++ b/src/adapters/filesystem_store_initializer.rs @@ -11,7 +11,7 @@ use super::filesystem_initialization_storage::FilesystemInitializationStorage; use super::{ FilesystemPlatformAdmission, FilesystemPlatformAdmissionError, FilesystemWriterLock, StoreInitializationError, StoreInitializationPhase, filesystem_initialization_namespace, - filesystem_platform_profile, filesystem_version_two_records, initialize_store, + filesystem_platform_profile, initialize_store, }; impl FilesystemPlatformAdmission { @@ -66,34 +66,6 @@ impl FilesystemPlatformAdmission { initialize_storage(storage) } - /// Reacquires writer authority over one completely migrated version-2 store. - /// - /// The call mutates no protocol state. It admits the production platform - /// for every version-2 protocol directory, acquires the existing writer - /// lock, requires the exact version-2 root namespace, and jointly admits the - /// marker, intent, and receipt records. Retention and recovery adapters perform content-level - /// validation under the returned authority. The synchronous call may block - /// on filesystem I/O. - /// - /// # Errors - /// - /// Returns [`FilesystemPlatformAdmissionError`] with the exact platform, - /// writer-lock, or namespace boundary and preserved source. - pub fn reopen_version_two(store_root: &Path) -> Result { - let root = filesystem_platform_profile::open_version_two(store_root) - .map_err(|source| FilesystemPlatformAdmissionError::Platform { source })?; - reopen_version_two_root(root) - } - - #[cfg(test)] - pub(super) fn reopen_version_two_unchecked_for_tests( - store_root: &Path, - ) -> Result { - let root = Dir::open_ambient_dir(store_root, ambient_authority()) - .map_err(|source| FilesystemPlatformAdmissionError::Platform { source })?; - reopen_version_two_root(root) - } - #[cfg(test)] pub(super) fn reopen_unchecked_for_tests( store_root: &Path, @@ -124,22 +96,6 @@ fn initialize_storage( )) } -fn reopen_version_two_root( - root: cap_std::fs::Dir, -) -> Result { - let admission = admit_reopened(root, filesystem_initialization_namespace::admit_version_two)?; - let (lock, root_identity) = admission.into_parts(); - let directory = lock - .clone_directory() - .map_err(|source| FilesystemPlatformAdmissionError::MigrationRecord { source })?; - filesystem_version_two_records::admit(&directory) - .map_err(|source| FilesystemPlatformAdmissionError::MigrationRecord { source })?; - Ok(FilesystemPlatformAdmission::initialized( - lock, - root_identity, - )) -} - fn reopen_root( root: cap_std::fs::Dir, ) -> Result { diff --git a/src/adapters/filesystem_version_two_admission.rs b/src/adapters/filesystem_version_two_admission.rs new file mode 100644 index 0000000..635987b --- /dev/null +++ b/src/adapters/filesystem_version_two_admission.rs @@ -0,0 +1,76 @@ +//! This module owns proof that a filesystem root passed version-two admission. + +use std::path::Path; + +#[cfg(test)] +use cap_std::ambient_authority; +use cap_std::fs::Dir; + +use super::filesystem_root_identity::FilesystemRootIdentity; +use super::{ + FilesystemPlatformAdmissionError, FilesystemWriterLock, filesystem_initialization_namespace, + filesystem_platform_profile, filesystem_version_two_records, +}; + +/// Exclusive writer authority over a completely migrated version-two root. +/// +/// This type is deliberately distinct from +/// [`FilesystemPlatformAdmission`](super::FilesystemPlatformAdmission): a +/// version-one publisher cannot consume it, so version-one catalog publication +/// can never run against a migrated root and leave residue no adapter admits. +/// Fields are private so only version-two admission can create values. +#[must_use] +pub struct FilesystemVersionTwoAdmission { + lock: FilesystemWriterLock, +} + +impl FilesystemVersionTwoAdmission { + /// Reacquires writer authority over one completely migrated version-two store. + /// + /// The call mutates no protocol state. It admits the production platform + /// for every version-two protocol directory, acquires the existing writer + /// lock, requires the exact version-two root namespace, and jointly admits + /// the marker, intent, and receipt records. Retention adapters perform + /// content-level validation under the returned authority. The synchronous + /// call may block on filesystem I/O. + /// + /// # Errors + /// + /// Returns [`FilesystemPlatformAdmissionError`] with the exact platform, + /// writer-lock, namespace, or migration-record boundary and preserved + /// source. + pub fn reopen(store_root: &Path) -> Result { + let root = filesystem_platform_profile::open_version_two(store_root) + .map_err(|source| FilesystemPlatformAdmissionError::Platform { source })?; + Self::admit(root) + } + + #[cfg(test)] + pub(super) fn reopen_unchecked_for_tests( + store_root: &Path, + ) -> Result { + let root = Dir::open_ambient_dir(store_root, ambient_authority()) + .map_err(|source| FilesystemPlatformAdmissionError::Platform { source })?; + Self::admit(root) + } + + pub(super) fn into_lock(self) -> FilesystemWriterLock { + self.lock + } + + fn admit(root: Dir) -> Result { + let lock = FilesystemWriterLock::try_acquire_in(root) + .map_err(|source| FilesystemPlatformAdmissionError::WriterLock { source })?; + let directory = lock + .clone_directory() + .map_err(|source| FilesystemPlatformAdmissionError::Namespace { source })?; + filesystem_initialization_namespace::admit_version_two(&directory) + .map_err(|source| FilesystemPlatformAdmissionError::Namespace { source })?; + let _root_identity: FilesystemRootIdentity = + filesystem_platform_profile::root_identity(&directory) + .map_err(|source| FilesystemPlatformAdmissionError::Platform { source })?; + filesystem_version_two_records::admit(&directory) + .map_err(|source| FilesystemPlatformAdmissionError::MigrationRecord { source })?; + Ok(Self { lock }) + } +} diff --git a/src/adapters/mod.rs b/src/adapters/mod.rs index 4da8d54..cee5da7 100644 --- a/src/adapters/mod.rs +++ b/src/adapters/mod.rs @@ -129,6 +129,7 @@ mod filesystem_store_initializer_tests; #[cfg(test)] #[path = "../../tests/segment_filesystem_stage/sandbox.rs"] mod filesystem_test_sandbox; +mod filesystem_version_two_admission; mod filesystem_version_two_records; mod filesystem_writer_lock; mod framed_blake3; @@ -364,6 +365,7 @@ pub use filesystem_recovery_stage_error::{ FilesystemRecoveryStageError, RecoveryStageNamespacePhase, }; pub use filesystem_segment_stage::FilesystemSegmentStage; +pub use filesystem_version_two_admission::FilesystemVersionTwoAdmission; pub use filesystem_writer_lock::FilesystemWriterLock; pub use layout_decode_error::LayoutDecodeError; pub use layout_decode_policy::LayoutDecodePolicy; diff --git a/src/adapters/retention/filesystem_retention_authority.rs b/src/adapters/retention/filesystem_retention_authority.rs index dca5a38..3faeda1 100644 --- a/src/adapters/retention/filesystem_retention_authority.rs +++ b/src/adapters/retention/filesystem_retention_authority.rs @@ -11,7 +11,7 @@ use super::filesystem_retention_authority_error::{ use super::filesystem_retention_current::{self, ObservedRetentionState}; use super::filesystem_retention_pool_name as pool_name; use super::filesystem_retention_stage::{FilesystemRetentionStage, invalid_data}; -use crate::adapters::{FilesystemPlatformAdmission, FilesystemWriterLock}; +use crate::adapters::{FilesystemVersionTwoAdmission, FilesystemWriterLock}; /// Exclusive authority to publish retention transitions on one pinned root. /// @@ -41,7 +41,10 @@ pub struct FilesystemRetentionPublicationAuthority { } impl FilesystemRetentionPublicationAuthority { - /// Pins one migrated version-2 root for retention publication. + /// Pins one admitted version-two root for retention publication. + /// + /// Only [`FilesystemVersionTwoAdmission`] is accepted, so version-one + /// writer authority can never reach retention publication. /// /// This synchronous constructor opens pinned directory capabilities but /// materializes no record bodies and performs no protocol mutation. @@ -51,7 +54,7 @@ impl FilesystemRetentionPublicationAuthority { /// Returns [`FilesystemRetentionAuthorityError`](super::FilesystemRetentionAuthorityError) /// when the root capability cannot be cloned or the retention namespace and /// either immutable pool cannot be pinned without following links. - pub fn open(admission: FilesystemPlatformAdmission) -> Result { + pub fn open(admission: FilesystemVersionTwoAdmission) -> Result { let lock = admission.into_lock(); let root = lock.clone_directory().map_err(|source| Error::Directory { directory: Directory::Root, diff --git a/src/adapters/retention/filesystem_retention_test_fixture.rs b/src/adapters/retention/filesystem_retention_test_fixture.rs index 4bb83b9..1f3a1a0 100644 --- a/src/adapters/retention/filesystem_retention_test_fixture.rs +++ b/src/adapters/retention/filesystem_retention_test_fixture.rs @@ -19,7 +19,7 @@ use crate::adapters::test_support::decode_hex; use crate::adapters::{ AdmittedCatalog, AdmittedSegment, CatalogSnapshot, ChecksummedCatalog, ChecksummedPublicationHead, FilesystemPlatformAdmission, FilesystemStoreMigrationAuthority, - SegmentReadPolicy, SegmentRecordLimit, + FilesystemVersionTwoAdmission, SegmentReadPolicy, SegmentRecordLimit, }; use crate::{ RetentionGenerationExpectation, RetentionNamespace, RetentionPolicy, RetentionRoot, @@ -57,8 +57,7 @@ pub(super) fn open_authority( name: &str, ) -> Result<(TestDirectory, FilesystemRetentionPublicationAuthority), Box> { let sandbox = migrated_store(name)?; - let admission = - FilesystemPlatformAdmission::reopen_version_two_unchecked_for_tests(sandbox.path())?; + let admission = FilesystemVersionTwoAdmission::reopen_unchecked_for_tests(sandbox.path())?; let authority = FilesystemRetentionPublicationAuthority::open(admission)?; Ok((sandbox, authority)) } diff --git a/src/adapters/retention/filesystem_version_two_admission_tests.rs b/src/adapters/retention/filesystem_version_two_admission_tests.rs index 5a5f55c..80b12b4 100644 --- a/src/adapters/retention/filesystem_version_two_admission_tests.rs +++ b/src/adapters/retention/filesystem_version_two_admission_tests.rs @@ -4,7 +4,7 @@ use std::error::Error; use std::fs; use super::filesystem_retention_test_fixture::migrated_store; -use crate::adapters::{FilesystemPlatformAdmission, FilesystemPlatformAdmissionError}; +use crate::adapters::{FilesystemPlatformAdmissionError, FilesystemVersionTwoAdmission}; #[test] fn version_two_reopen_refuses_a_corrupt_format_marker() -> Result<(), Box> { @@ -16,7 +16,7 @@ fn version_two_reopen_refuses_a_corrupt_format_marker() -> Result<(), Box Result<(), Box Result<(), Box> { let sandbox = migrated_store("version-two-admission-exact")?; - let admission = - FilesystemPlatformAdmission::reopen_version_two_unchecked_for_tests(sandbox.path())?; + let admission = FilesystemVersionTwoAdmission::reopen_unchecked_for_tests(sandbox.path())?; drop(admission); sandbox.remove()?; @@ -89,7 +88,7 @@ fn production_version_two_reopen_refuses_an_aliased_protocol_directory() fs::remove_dir(sandbox.path().join("gc"))?; std::os::unix::fs::symlink(&alias_target, sandbox.path().join("gc"))?; - let error = FilesystemPlatformAdmission::reopen_version_two(sandbox.path()) + let error = FilesystemVersionTwoAdmission::reopen(sandbox.path()) .err() .ok_or("aliased gc protocol directory was unexpectedly admitted")?; @@ -106,7 +105,7 @@ fn production_version_two_reopen_refuses_an_aliased_protocol_directory() fn production_version_two_reopen_admits_an_exact_migrated_store() -> Result<(), Box> { let sandbox = migrated_store("version-two-admission-production-exact")?; - let admission = FilesystemPlatformAdmission::reopen_version_two(sandbox.path())?; + let admission = FilesystemVersionTwoAdmission::reopen(sandbox.path())?; drop(admission); sandbox.remove()?; diff --git a/src/lib.rs b/src/lib.rs index 6d54af1..64a75ca 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -80,27 +80,27 @@ pub use adapters::{ FilesystemRecoveryStageCompleter, FilesystemRecoveryStageCompletionOpenError, FilesystemRecoveryStageDiscardOpenError, FilesystemRecoveryStageDiscarder, FilesystemRecoveryStageError, FilesystemSegmentStage, FilesystemStoreMigrationAuthority, - FilesystemStoreMigrationInventoryReader, FilesystemWriterLock, ImmutablePoolInventoryDigest, - InitialGcStateDigest, InitialRetentionStateDigest, LayoutDecodeError, LayoutDecodePolicy, - LayoutEncodeError, LayoutIdBinaryParseError, LayoutIdTextParseError, - MigrationInventoryNamespace, MigrationInventoryPool, MigrationSynchronizationMask, - OpenedReusableSegment, PublicationHeadDecodeError, RecoveryCatalogStage, - RecoveryCatalogStageError, RecoveryEntryName, RecoveryEntryNameError, RecoveryEntryRole, - RecoveryInventory, RecoveryInventoryEntry, RecoveryInventoryError, RecoveryInventoryLimit, - RecoveryInventoryLimitError, RecoveryInventoryOperation, RecoveryInventoryStorage, - RecoveryNameClassificationError, RecoveryNameManifest, RecoveryNamedEntry, RecoveryNamespace, - RecoveryNextHeadFinalizationError, RecoveryNextHeadFinalizationOutcome, - RecoveryNextHeadFinalizationPlanError, RecoveryNextHeadFinalizationReadiness, - RecoveryNextHeadFinalizationReceipt, RecoveryNextHeadFinalizationRequest, - RecoveryNextHeadFinalizationStorage, RecoveryNextHeadFinalizationStorageError, - RecoveryNextHeadFinalizationTarget, RecoveryNextHeadStage, RecoveryNextHeadStageError, - RecoveryPoolNameError, RecoveryRequiredEntry, RecoverySegmentResumeError, - RecoverySegmentResumePlanError, RecoverySegmentResumeRequest, RecoverySegmentResumeStorage, - RecoverySegmentResumeStorageError, RecoverySegmentStage, RecoverySegmentStageError, - RecoverySegmentTruncation, RecoveryStage, RecoveryStageAssessment, - RecoveryStageAssessmentError, RecoveryStageByteAdmissionError, RecoveryStageCompletionError, - RecoveryStageCompletionPlanError, RecoveryStageCompletionPool, RecoveryStageCompletionReceipt, - RecoveryStageCompletionRequest, RecoveryStageCompletionStorage, + FilesystemStoreMigrationInventoryReader, FilesystemVersionTwoAdmission, FilesystemWriterLock, + ImmutablePoolInventoryDigest, InitialGcStateDigest, InitialRetentionStateDigest, + LayoutDecodeError, LayoutDecodePolicy, LayoutEncodeError, LayoutIdBinaryParseError, + LayoutIdTextParseError, MigrationInventoryNamespace, MigrationInventoryPool, + MigrationSynchronizationMask, OpenedReusableSegment, PublicationHeadDecodeError, + RecoveryCatalogStage, RecoveryCatalogStageError, RecoveryEntryName, RecoveryEntryNameError, + RecoveryEntryRole, RecoveryInventory, RecoveryInventoryEntry, RecoveryInventoryError, + RecoveryInventoryLimit, RecoveryInventoryLimitError, RecoveryInventoryOperation, + RecoveryInventoryStorage, RecoveryNameClassificationError, RecoveryNameManifest, + RecoveryNamedEntry, RecoveryNamespace, RecoveryNextHeadFinalizationError, + RecoveryNextHeadFinalizationOutcome, RecoveryNextHeadFinalizationPlanError, + RecoveryNextHeadFinalizationReadiness, RecoveryNextHeadFinalizationReceipt, + RecoveryNextHeadFinalizationRequest, RecoveryNextHeadFinalizationStorage, + RecoveryNextHeadFinalizationStorageError, RecoveryNextHeadFinalizationTarget, + RecoveryNextHeadStage, RecoveryNextHeadStageError, RecoveryPoolNameError, + RecoveryRequiredEntry, RecoverySegmentResumeError, RecoverySegmentResumePlanError, + RecoverySegmentResumeRequest, RecoverySegmentResumeStorage, RecoverySegmentResumeStorageError, + RecoverySegmentStage, RecoverySegmentStageError, RecoverySegmentTruncation, RecoveryStage, + RecoveryStageAssessment, RecoveryStageAssessmentError, RecoveryStageByteAdmissionError, + RecoveryStageCompletionError, RecoveryStageCompletionPlanError, RecoveryStageCompletionPool, + RecoveryStageCompletionReceipt, RecoveryStageCompletionRequest, RecoveryStageCompletionStorage, RecoveryStageCompletionStorageError, RecoveryStageCompletionTarget, RecoveryStageDiscardError, RecoveryStageDiscardOutcome, RecoveryStageDiscardPlanError, RecoveryStageDiscardReason, RecoveryStageDiscardReceipt, RecoveryStageDiscardRequest, RecoveryStageDiscardStorage, diff --git a/tests/version_two_admission_contract.rs b/tests/version_two_admission_contract.rs new file mode 100644 index 0000000..1c33945 --- /dev/null +++ b/tests/version_two_admission_contract.rs @@ -0,0 +1,33 @@ +//! Version-two writer authority is a distinct type that version-one publishers cannot consume. + +const RETENTION_AUTHORITY: &str = + include_str!("../src/adapters/retention/filesystem_retention_authority.rs"); +const CATALOG_PUBLISHER: &str = include_str!("../src/adapters/filesystem_catalog_publisher.rs"); +const STORE_INITIALIZER: &str = include_str!("../src/adapters/filesystem_store_initializer.rs"); +const VERSION_TWO_ADMISSION: &str = + include_str!("../src/adapters/filesystem_version_two_admission.rs"); +const MIGRATION_AUTHORITY: &str = + include_str!("../src/adapters/store_migration/filesystem_migration_authority.rs"); + +#[test] +fn retention_publication_consumes_only_version_two_authority() { + assert!(RETENTION_AUTHORITY.contains("pub fn open(admission: FilesystemVersionTwoAdmission)")); + assert!(!RETENTION_AUTHORITY.contains("admission: FilesystemPlatformAdmission")); +} + +#[test] +fn version_one_publishers_consume_only_version_one_authority() { + assert!(CATALOG_PUBLISHER.contains("admission: FilesystemPlatformAdmission,")); + assert!(!CATALOG_PUBLISHER.contains("FilesystemVersionTwoAdmission")); + assert!(MIGRATION_AUTHORITY.contains("admission: FilesystemPlatformAdmission,")); + assert!(!MIGRATION_AUTHORITY.contains("FilesystemVersionTwoAdmission")); +} + +#[test] +fn version_two_reopen_produces_only_version_two_authority() { + assert!(!STORE_INITIALIZER.contains("fn reopen_version_two")); + assert!(VERSION_TWO_ADMISSION.contains("pub struct FilesystemVersionTwoAdmission")); + assert!(VERSION_TWO_ADMISSION.contains("pub fn reopen(store_root: &Path)")); + assert!(VERSION_TWO_ADMISSION.contains("filesystem_version_two_records::admit")); + assert!(VERSION_TWO_ADMISSION.contains("filesystem_platform_profile::open_version_two")); +} From 380efec7f43b18d236d28e9bd559f6cb84c8d7af Mon Sep 17 00:00:00 2001 From: James Ross Date: Mon, 7 Sep 2026 00:26:42 -0700 Subject: [PATCH 065/111] Fix: make retention stage directory entries durable before dependent phases synchronize_root_stage, synchronize_manifest_stage, and synchronize_head_stage called sync_all on the stage file only. The retention directory was first synchronized at SynchronizeRetentionNamespace, after ReplaceHead, so the root.next and manifest.next directory entries were not durable before the namespace directory creation, pool links, and head replacement that depend on them. recovery.md requires root.next to be durable before a new namespace directory is created; power loss between the manifest pool sync and head replacement could leave pool entries with no stage evidence to classify. Each stage synchronization now also synchronizes the retention directory. No deterministic regression law accompanies this fix: fsync ordering is not observable through the filesystem adapter without a fault-injection seam, and that seam is the KEEP-CRASH-036..=052 process-death matrix owned by plan item 7. This commit is recorded as such. Self-audit finding S2 (P1). Refs #78 --- CHANGELOG.md | 5 ++++- src/adapters/retention/filesystem_retention_storage.rs | 9 ++++++--- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 846de26..a25b897 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -57,7 +57,10 @@ after its public API and format compatibility policies are established. empty state only while both pools are empty and admits only an initial head with no predecessor; a namespace directory must be absent for an `Absent` expectation and present for a `Current` one. Every mismatch refuses as - recovery-required before any stage is written. + recovery-required before any stage is written. Each retention stage + synchronization now also synchronizes the `retention` directory, so the + `root.next`, `manifest.next`, and `head.next` entries are durable before the + namespace directory, pool links, or head replacement that depend on them. - Repository crash-matrix execution now terminates isolated writer process groups at all 105 canonical before/during/after coordinates, retains open writer and stage authority until termination, executes production diff --git a/src/adapters/retention/filesystem_retention_storage.rs b/src/adapters/retention/filesystem_retention_storage.rs index a2f14bb..00f8068 100644 --- a/src/adapters/retention/filesystem_retention_storage.rs +++ b/src/adapters/retention/filesystem_retention_storage.rs @@ -68,7 +68,8 @@ impl RetentionPublicationStorage for FilesystemRetentionPublicationAuthority { } fn synchronize_root_stage(&mut self) -> io::Result<()> { - self.root_stage()?.synchronize(&self.retention) + self.root_stage()?.synchronize(&self.retention)?; + synchronize_directory(&self.retention) } fn admit_root_namespace( @@ -113,7 +114,8 @@ impl RetentionPublicationStorage for FilesystemRetentionPublicationAuthority { } fn synchronize_manifest_stage(&mut self) -> io::Result<()> { - self.manifest_stage()?.synchronize(&self.retention) + self.manifest_stage()?.synchronize(&self.retention)?; + synchronize_directory(&self.retention) } fn link_manifest(&mut self, manifest: &CanonicalRetentionManifest) -> io::Result<()> { @@ -138,7 +140,8 @@ impl RetentionPublicationStorage for FilesystemRetentionPublicationAuthority { } fn synchronize_head_stage(&mut self) -> io::Result<()> { - self.head_stage()?.synchronize(&self.retention) + self.head_stage()?.synchronize(&self.retention)?; + synchronize_directory(&self.retention) } fn replace_head(&mut self) -> io::Result<()> { From 567d4372a78e0b5f347a3f4f7de9f7996f17c2d3 Mon Sep 17 00:00:00 2001 From: James Ross Date: Mon, 7 Sep 2026 00:30:59 -0700 Subject: [PATCH 066/111] Fix: open retention and admission records with O_NONBLOCK The retention stage verifier, the current-state reader, and the version-two record reader opened files with follow(No) but without nonblock(true). A FIFO at retention/HEAD blocked observe_current indefinitely while holding the writer lock; the migration copy of the same verifier already refused by kind because it opened non-blocking. The duplication had drifted. All three read-side opens now set nonblock(true), so a non-regular file is refused by the metadata kind check immediately. Namespace admission and the version-two file-kind check already shielded the pool-name and FORMAT paths; those laws are kept as pins. Regression laws run the operation on a worker thread with a five-second deadline: a FIFO at retention/HEAD, at FORMAT, and at a manifest pool name each refuse promptly. The fixture uses rustix mknodat on Linux and mkfifo(1) on other Unix targets, where rustix compiles mknodat out; no storage path spawns a process. Self-audit finding S10 (P2). The remaining duplication with filesystem_migration_fixed_artifact is logged as a follow-up. Refs #78 --- CHANGELOG.md | 3 + .../filesystem_version_two_records.rs | 4 +- src/adapters/retention.rs | 2 + .../retention/filesystem_retention_current.rs | 4 +- .../filesystem_retention_fifo_tests.rs | 106 ++++++++++++++++++ .../retention/filesystem_retention_stage.rs | 4 +- 6 files changed, 117 insertions(+), 6 deletions(-) create mode 100644 src/adapters/retention/filesystem_retention_fifo_tests.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index a25b897..4ec37d3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -61,6 +61,9 @@ after its public API and format compatibility policies are established. synchronization now also synchronizes the `retention` directory, so the `root.next`, `manifest.next`, and `head.next` entries are durable before the namespace directory, pool links, or head replacement that depend on them. + Every read-side reopen in retention publication and version-two admission + opens with `O_NONBLOCK`, so a FIFO planted at `retention/HEAD`, `FORMAT`, or a + pool name refuses by kind instead of blocking under the writer lock. - Repository crash-matrix execution now terminates isolated writer process groups at all 105 canonical before/during/after coordinates, retains open writer and stage authority until termination, executes production diff --git a/src/adapters/filesystem_version_two_records.rs b/src/adapters/filesystem_version_two_records.rs index 0e2754f..f3b8306 100644 --- a/src/adapters/filesystem_version_two_records.rs +++ b/src/adapters/filesystem_version_two_records.rs @@ -2,7 +2,7 @@ use std::io::{self, Read}; -use cap_fs_ext::{FollowSymlinks, OpenOptionsFollowExt}; +use cap_fs_ext::{FollowSymlinks, OpenOptionsFollowExt, OpenOptionsSyncExt}; use cap_std::fs::{Dir, OpenOptions}; use super::{ @@ -37,7 +37,7 @@ pub(super) fn admit(root: &Dir) -> io::Result<()> { fn read_exact(root: &Dir, name: &str, length: usize) -> io::Result> { let mut options = OpenOptions::new(); - options.read(true).follow(FollowSymlinks::No); + options.read(true).follow(FollowSymlinks::No).nonblock(true); let mut file = root.open_with(name, &options)?; let expected_length = u64::try_from(length) .map_err(|_source| invalid_data(name, &"record length exceeded u64"))?; diff --git a/src/adapters/retention.rs b/src/adapters/retention.rs index faabd1d..6607f14 100644 --- a/src/adapters/retention.rs +++ b/src/adapters/retention.rs @@ -22,6 +22,8 @@ mod filesystem_retention_current; mod filesystem_retention_current_tests; #[cfg(test)] mod filesystem_retention_expectation_tests; +#[cfg(test)] +mod filesystem_retention_fifo_tests; mod filesystem_retention_namespace; #[cfg(test)] mod filesystem_retention_namespace_tests; diff --git a/src/adapters/retention/filesystem_retention_current.rs b/src/adapters/retention/filesystem_retention_current.rs index b72bbcf..c2c2c57 100644 --- a/src/adapters/retention/filesystem_retention_current.rs +++ b/src/adapters/retention/filesystem_retention_current.rs @@ -2,7 +2,7 @@ use std::io::{self, Read}; -use cap_fs_ext::{DirExt, FollowSymlinks, OpenOptionsFollowExt}; +use cap_fs_ext::{DirExt, FollowSymlinks, OpenOptionsFollowExt, OpenOptionsSyncExt}; use cap_std::fs::{Dir, OpenOptions}; use super::filesystem_retention_pool_name as pool_name; @@ -189,7 +189,7 @@ fn read_exact_optional( length: usize, ) -> io::Result>> { let mut options = OpenOptions::new(); - options.read(true).follow(FollowSymlinks::No); + options.read(true).follow(FollowSymlinks::No).nonblock(true); let mut file = match directory.open_with(name, &options) { Ok(file) => file, Err(source) if source.kind() == io::ErrorKind::NotFound => return Ok(None), diff --git a/src/adapters/retention/filesystem_retention_fifo_tests.rs b/src/adapters/retention/filesystem_retention_fifo_tests.rs new file mode 100644 index 0000000..fccc9d1 --- /dev/null +++ b/src/adapters/retention/filesystem_retention_fifo_tests.rs @@ -0,0 +1,106 @@ +//! Filesystem retention non-regular-file laws: a FIFO at a protocol name refuses, never blocks. + +use std::error::Error; +use std::path::Path; +use std::sync::mpsc; +use std::thread; +use std::time::Duration; + +use super::filesystem_retention_test_fixture::{ + ROOT_HEX, fixture, initial_preparation, manifest_pool_path, migrated_store, open_authority, +}; +use crate::adapters::FilesystemVersionTwoAdmission; +use crate::execute_retention_publication; + +const DEADLINE: Duration = Duration::from_secs(5); + +#[test] +fn a_fifo_at_the_retention_head_refuses_instead_of_blocking() -> Result<(), Box> { + let (sandbox, authority) = open_authority("filesystem-retention-fifo-head")?; + make_fifo(&sandbox.path().join("retention").join("HEAD"))?; + + let outcome = completes_within(move || authority.observe_current().map(|_| ()))?; + + assert!(outcome.is_err(), "FIFO head was unexpectedly admitted"); + sandbox.remove()?; + Ok(()) +} + +#[test] +fn a_fifo_at_the_format_marker_refuses_instead_of_blocking() -> Result<(), Box> { + let sandbox = migrated_store("filesystem-retention-fifo-marker")?; + std::fs::remove_file(sandbox.path().join("FORMAT"))?; + make_fifo(&sandbox.path().join("FORMAT"))?; + let path = sandbox.path().to_path_buf(); + + let outcome = completes_within(move || { + FilesystemVersionTwoAdmission::reopen_unchecked_for_tests(&path).map(|_| ()) + })?; + + assert!( + outcome.is_err(), + "FIFO format marker was unexpectedly admitted" + ); + sandbox.remove()?; + Ok(()) +} + +#[test] +fn a_fifo_at_a_manifest_pool_name_refuses_instead_of_blocking() -> Result<(), Box> { + let (sandbox, mut authority) = open_authority("filesystem-retention-fifo-pool")?; + let root_bytes = fixture(ROOT_HEX)?; + make_fifo(&manifest_pool_path( + sandbox.path(), + &initial_preparation(&root_bytes)?, + ))?; + + let outcome = completes_within(move || { + let preparation = initial_preparation(&root_bytes).map_err(|error| error.to_string())?; + execute_retention_publication(&mut authority, &preparation) + .map(|_| ()) + .map_err(|error| error.to_string()) + })?; + + assert!( + outcome.is_err(), + "FIFO pool target was unexpectedly admitted" + ); + sandbox.remove()?; + Ok(()) +} + +/// Creates a FIFO at `path` for the law under test. +/// +/// rustix compiles `mknodat` out on Apple targets, so the fixture falls back +/// to `mkfifo(1)` there. This is test scaffolding only; no storage path +/// spawns a process. +#[cfg(target_os = "linux")] +fn make_fifo(path: &Path) -> Result<(), Box> { + use rustix::fs::{CWD, FileType, Mode, mknodat}; + + mknodat(CWD, path, FileType::Fifo, Mode::RUSR | Mode::WUSR, 0)?; + Ok(()) +} + +#[cfg(not(target_os = "linux"))] +fn make_fifo(path: &Path) -> Result<(), Box> { + let status = std::process::Command::new("mkfifo").arg(path).status()?; + if status.success() { + Ok(()) + } else { + Err(format!("mkfifo exited with {status}").into()) + } +} + +/// Runs `operation` on its own thread and refuses the test if it does not finish. +fn completes_within( + operation: impl FnOnce() -> Result<(), E> + Send + 'static, +) -> Result, Box> { + let (sender, receiver) = mpsc::channel(); + let _worker = thread::spawn(move || { + let _ = sender.send(operation()); + }); + receiver + .recv_timeout(DEADLINE) + .map_err(|_timeout| "operation blocked past the deadline: a FIFO open hung".into()) +} diff --git a/src/adapters/retention/filesystem_retention_stage.rs b/src/adapters/retention/filesystem_retention_stage.rs index d882f56..1ffe7b3 100644 --- a/src/adapters/retention/filesystem_retention_stage.rs +++ b/src/adapters/retention/filesystem_retention_stage.rs @@ -2,7 +2,7 @@ use std::io::{self, Read, Write}; -use cap_fs_ext::{FollowSymlinks, MetadataExt, OpenOptionsFollowExt}; +use cap_fs_ext::{FollowSymlinks, MetadataExt, OpenOptionsFollowExt, OpenOptionsSyncExt}; use cap_std::fs::{Dir, File, Metadata, OpenOptions}; use crate::adapters::filesystem_catalog_artifact; @@ -112,7 +112,7 @@ fn verify_name( identity: StageIdentity, ) -> io::Result<()> { let mut options = OpenOptions::new(); - options.read(true).follow(FollowSymlinks::No); + options.read(true).follow(FollowSymlinks::No).nonblock(true); let mut file = directory.open_with(name, &options)?; require_metadata(&file.metadata()?, expected.len(), identity)?; require_metadata(&directory.symlink_metadata(name)?, expected.len(), identity)?; From 34f197fe00fad4d310edbee474177e334224a026 Mon Sep 17 00:00:00 2001 From: James Ross Date: Mon, 7 Sep 2026 00:39:02 -0700 Subject: [PATCH 067/111] Fix: carry typed current-state refusals as the io::Error source Filesystem current-state verification erased every outcome to io::Error::new(InvalidData, &'static str) and dropped decode errors through map_err(|_source| ...). A byte-identical retry superseded by a newer generation, a corrupt head checksum, an absent manifest, and a head/manifest disagreement were indistinguishable at the type level; RetentionPublicationError's CurrentVerification source() returned None; and retention.md's "returns the precise stale state" was unmet at the one layer that observes concurrent successors. RetentionCurrentStateRefusal names each outcome: retained stage, head absent with artifacts, expected-current over absent head, non-initial over absent head, head or prepared-head or manifest decode refusal with the decode error preserved as source, manifest absent or disagreeing, liveness exhausted, stale committed retry, superseded with the current head's generation and digest, committed selection missing or mismatched, committed namespace unavailable, committed root absent or changed, and record kind, length, trailing-byte, or overflow refusals. Every one is carried as the io::Error source, so the port signature is unchanged and callers downcast. Regression laws downcast the source in four existing tests: Superseded, CommittedRootAbsent, CommittedRootChanged, HeadAbsentWithArtifacts. Self-audit finding S9 (P2). Refs #78 --- CHANGELOG.md | 5 + docs/formats/segment-store-v2/requirements.md | 2 +- src/adapters/retention.rs | 2 + .../retention/filesystem_retention_current.rs | 71 +++++------ .../filesystem_retention_current_tests.rs | 14 ++- .../filesystem_retention_expectation_tests.rs | 12 +- .../retention/filesystem_retention_refusal.rs | 117 ++++++++++++++++++ .../retention/filesystem_retention_storage.rs | 12 +- .../filesystem_retention_successor_tests.rs | 11 +- .../filesystem_retention_test_fixture.rs | 9 +- src/lib.rs | 18 +-- 11 files changed, 207 insertions(+), 66 deletions(-) create mode 100644 src/adapters/retention/filesystem_retention_refusal.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index 4ec37d3..423692b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -64,6 +64,11 @@ after its public API and format compatibility policies are established. Every read-side reopen in retention publication and version-two admission opens with `O_NONBLOCK`, so a FIFO planted at `retention/HEAD`, `FORMAT`, or a pool name refuses by kind instead of blocking under the writer lock. + Filesystem current-state verification now carries a typed + `RetentionCurrentStateRefusal` as the source of every `InvalidData` it + returns, so a superseded candidate, a stale committed retry, an absent head + over populated pools, and each corruption or decode refusal are + distinguishable to callers and preserve their underlying decode errors. - Repository crash-matrix execution now terminates isolated writer process groups at all 105 canonical before/during/after coordinates, retains open writer and stage authority until termination, executes production diff --git a/docs/formats/segment-store-v2/requirements.md b/docs/formats/segment-store-v2/requirements.md index 8a3f1bb..d7e7c33 100644 --- a/docs/formats/segment-store-v2/requirements.md +++ b/docs/formats/segment-store-v2/requirements.md @@ -17,7 +17,7 @@ case is not evidence. | `KEEP-RETENTION-006` | Publication follows the exact ordered durability protocol, including new namespace-directory admission and retention of fixed-stage evidence until head commit, and returns only after cleanup synchronization | typed vocabulary and blocking port in `tests/retention_publication_phase.rs` and `tests/retention_publication_storage.rs`; ordered execution, conditional namespace sync, and all 17 exact storage-fault boundaries in `tests/retention_publication_execution.rs`; production 17-phase forward filesystem execution, exclusive staging, byte-equal inode-substitution refusal, and retained-stage recovery refusal in `filesystem_retention_storage_tests`; orphan namespace directories count against the 4,096 ceiling and refuse a new namespace before any stage is written in `filesystem_retention_capacity_tests`; crash injection remains | In progress in #19 | | `KEEP-RETENTION-007` | Restart resolves every fixed-stage crash prefix to one documented lawful state or typed ambiguity | recovery-required refusals before any mutation in `filesystem_retention_expectation_tests`: an absent head over populated pools, a non-initial head prepared against an absent head, an orphan directory for a namespace expected absent, and an absent directory for a namespace expected current; debug and release crash matrix remains | In progress in #19 | | `KEEP-RETENTION-008` | Readers double-collect catalog and retention heads and bind one complete catalog, manifest, and root-generation view under a `ReaderFence` | immutable snapshot and concurrency tests | Planned in #19 | -| `KEEP-RETENTION-009` | Exact already-committed retry is idempotent only while its successor remains current | byte-identical planning in `tests/retention_transition.rs`; authority-revalidated zero-mutation retry receipt in `tests/retention_publication_execution.rs`; exact already-committed filesystem retry with a byte-identical retention witness in `filesystem_retention_storage_tests`; superseded-candidate filesystem refusal with zero mutation in `filesystem_retention_successor_tests`; committed retry reopens the head-selected manifest entry and root pool bytes, refusing absent, changed, or corrupt evidence in `filesystem_retention_current_tests` | Implemented | +| `KEEP-RETENTION-009` | Exact already-committed retry is idempotent only while its successor remains current | byte-identical planning in `tests/retention_transition.rs`; authority-revalidated zero-mutation retry receipt in `tests/retention_publication_execution.rs`; exact already-committed filesystem retry with a byte-identical retention witness in `filesystem_retention_storage_tests`; superseded-candidate filesystem refusal with zero mutation in `filesystem_retention_successor_tests`; committed retry reopens the head-selected manifest entry and root pool bytes, refusing absent, changed, or corrupt evidence in `filesystem_retention_current_tests`; every refusal is a typed `RetentionCurrentStateRefusal` source, with superseded, committed-root-absent, committed-root-changed, and head-absent-with-artifacts pinned by downcast | Implemented | | `KEEP-RETENTION-010` | Model operation sequences agree with a deterministic namespace-to-anchor-set map and never admit caller identity, paths, clocks, or application policy | model-based and source-architecture tests | Planned in #19 | diff --git a/src/adapters/retention.rs b/src/adapters/retention.rs index 6607f14..2ed7db9 100644 --- a/src/adapters/retention.rs +++ b/src/adapters/retention.rs @@ -28,6 +28,7 @@ mod filesystem_retention_namespace; #[cfg(test)] mod filesystem_retention_namespace_tests; mod filesystem_retention_pool_name; +mod filesystem_retention_refusal; mod filesystem_retention_stage; mod filesystem_retention_storage; #[cfg(test)] @@ -95,6 +96,7 @@ pub use filesystem_retention_authority_error::{ FilesystemRetentionAuthorityError, RetentionAuthorityDirectory, }; pub use filesystem_retention_current::ObservedRetentionState; +pub use filesystem_retention_refusal::RetentionCurrentStateRefusal; pub use head_decode_error::RetentionHeadDecodeError; pub use manifest_decode_error::RetentionManifestDecodeError; pub use manifest_encode_error::RetentionManifestEncodeError; diff --git a/src/adapters/retention/filesystem_retention_current.rs b/src/adapters/retention/filesystem_retention_current.rs index c2c2c57..76aa7c1 100644 --- a/src/adapters/retention/filesystem_retention_current.rs +++ b/src/adapters/retention/filesystem_retention_current.rs @@ -6,10 +6,9 @@ use cap_fs_ext::{DirExt, FollowSymlinks, OpenOptionsFollowExt, OpenOptionsSyncEx use cap_std::fs::{Dir, OpenOptions}; use super::filesystem_retention_pool_name as pool_name; -use super::filesystem_retention_stage::invalid_data; use super::{ AdmittedRetentionManifest, AdmittedRetentionRoot, ChecksummedRetentionHead, - RetentionPublicationPreparation, RetentionTransitionDisposition, + RetentionCurrentStateRefusal, RetentionPublicationPreparation, RetentionTransitionDisposition, }; use crate::RetentionGenerationExpectation; @@ -54,21 +53,19 @@ pub(super) fn observe( return Ok(None); }; let decoded = ChecksummedRetentionHead::decode(&head) - .map_err(|_source| invalid_data("current retention head refused admission"))?; + .map_err(|source| RetentionCurrentStateRefusal::HeadRefused { source }.into_io())?; let selected = decoded.head(); let length = usize::try_from(selected.manifest_length().get()) - .map_err(|_source| invalid_data("current manifest length exceeded usize"))?; + .map_err(|_source| RetentionCurrentStateRefusal::RecordLengthOverflow.into_io())?; let name = pool_name::manifest(selected.generation(), selected.manifest_digest()); let manifest = read_exact_optional(manifests, &name, length)? - .ok_or_else(|| invalid_data("current retention head names an absent manifest"))?; + .ok_or_else(|| RetentionCurrentStateRefusal::ManifestAbsent.into_io())?; let admitted = AdmittedRetentionManifest::decode(&manifest) - .map_err(|_source| invalid_data("current retention manifest refused admission"))?; + .map_err(|source| RetentionCurrentStateRefusal::ManifestRefused { source }.into_io())?; if admitted.digest() != selected.manifest_digest() || admitted.manifest().generation() != selected.generation() { - return Err(invalid_data( - "current retention manifest disagreed with its head", - )); + return Err(RetentionCurrentStateRefusal::ManifestDisagreed.into_io()); } Ok(Some(ObservedRetentionState { head, manifest })) } @@ -86,13 +83,13 @@ pub(super) fn disposition( let Some(current) = current else { return match preparation.expected() { RetentionGenerationExpectation::Absent => require_initial_publication(preparation), - RetentionGenerationExpectation::Current(_) => Err(invalid_data( - "expected a current retention generation but no head is published", - )), + RetentionGenerationExpectation::Current(_) => { + Err(RetentionCurrentStateRefusal::ExpectedCurrentOverAbsentHead.into_io()) + } }; }; let head = ChecksummedRetentionHead::decode(current.head_bytes()) - .map_err(|_source| invalid_data("observed retention head refused admission"))?; + .map_err(|source| RetentionCurrentStateRefusal::HeadRefused { source }.into_io())?; let head = head.head(); let committed = ( preparation.liveness_generation(), @@ -101,23 +98,25 @@ pub(super) fn disposition( if (head.generation(), head.manifest_digest()) == committed { return Ok(RetentionTransitionDisposition::AlreadyCommitted); } - let publication = preparation.publication().ok_or_else(|| { - invalid_data("already-committed retry is stale: another successor is current") - })?; + let publication = preparation + .publication() + .ok_or_else(|| RetentionCurrentStateRefusal::StaleCommittedRetry.into_io())?; let prepared = ChecksummedRetentionHead::decode(publication.head().encoded()) - .map_err(|_source| invalid_data("prepared retention head refused admission"))?; + .map_err(|source| RetentionCurrentStateRefusal::PreparedHeadRefused { source }.into_io())?; let expected_generation = head .generation() .successor() - .map_err(|_source| invalid_data("current liveness generation cannot advance"))?; + .map_err(|_source| RetentionCurrentStateRefusal::LivenessExhausted.into_io())?; if prepared.head().predecessor() == Some(head.manifest_digest()) && prepared.head().generation() == expected_generation { Ok(RetentionTransitionDisposition::Publish) } else { - Err(invalid_data( - "current retention head is not the prepared predecessor; the candidate is superseded", - )) + Err(RetentionCurrentStateRefusal::Superseded { + current_generation: head.generation(), + current_digest: head.manifest_digest(), + } + .into_io()) } } @@ -133,33 +132,29 @@ pub(super) fn verify_committed( candidate: &AdmittedRetentionRoot<'_>, ) -> io::Result<()> { let manifest = AdmittedRetentionManifest::decode(current.manifest_bytes()) - .map_err(|_source| invalid_data("observed retention manifest refused admission"))?; + .map_err(|source| RetentionCurrentStateRefusal::ManifestRefused { source }.into_io())?; let namespace = candidate.root().namespace().digest(); let entries = manifest.manifest().entries(); let entry = entries .binary_search_by_key(&namespace, |entry| entry.namespace()) .ok() .and_then(|index| entries.get(index).copied()) - .ok_or_else(|| { - invalid_data("committed manifest does not select the candidate namespace") - })?; + .ok_or_else(|| RetentionCurrentStateRefusal::CommittedSelectionMissing.into_io())?; if entry.root_generation() != candidate.root().generation() || entry.root_digest() != candidate.digest() { - return Err(invalid_data( - "committed manifest selects a different root for the candidate namespace", - )); + return Err(RetentionCurrentStateRefusal::CommittedSelectionMismatch.into_io()); } let directory = roots .open_dir_nofollow(pool_name::namespace(namespace)) - .map_err(|_source| invalid_data("committed root namespace directory is unavailable"))?; + .map_err(|_source| RetentionCurrentStateRefusal::CommittedNamespaceUnavailable.into_io())?; let name = pool_name::root(candidate.root().generation(), candidate.digest()); let observed = read_exact_optional(&directory, &name, candidate.encoded().len())? - .ok_or_else(|| invalid_data("committed root pool entry is absent"))?; + .ok_or_else(|| RetentionCurrentStateRefusal::CommittedRootAbsent.into_io())?; if observed.as_ref() == candidate.encoded() { Ok(()) } else { - Err(invalid_data("committed root pool entry bytes disagreed")) + Err(RetentionCurrentStateRefusal::CommittedRootChanged.into_io()) } } @@ -169,17 +164,15 @@ fn require_initial_publication( ) -> io::Result { let publication = preparation .publication() - .ok_or_else(|| invalid_data("already-committed retry against an absent retention head"))?; + .ok_or_else(|| RetentionCurrentStateRefusal::StaleCommittedRetry.into_io())?; let prepared = ChecksummedRetentionHead::decode(publication.head().encoded()) - .map_err(|_source| invalid_data("prepared retention head refused admission"))?; + .map_err(|source| RetentionCurrentStateRefusal::PreparedHeadRefused { source }.into_io())?; if prepared.head().generation() == crate::LivenessGeneration::INITIAL && prepared.head().predecessor().is_none() { Ok(RetentionTransitionDisposition::Publish) } else { - Err(invalid_data( - "absent retention head admits only an initial publication with no predecessor", - )) + Err(RetentionCurrentStateRefusal::NonInitialOverAbsentHead.into_io()) } } @@ -196,16 +189,16 @@ fn read_exact_optional( Err(source) => return Err(source), }; let expected_length = u64::try_from(length) - .map_err(|_source| invalid_data("retention record length exceeded u64"))?; + .map_err(|_source| RetentionCurrentStateRefusal::RecordLengthOverflow.into_io())?; let metadata = file.metadata()?; if !metadata.is_file() || metadata.len() != expected_length { - return Err(invalid_data("retention record kind or length disagreed")); + return Err(RetentionCurrentStateRefusal::RecordKindOrLength.into_io()); } let mut bytes = vec![0_u8; length]; file.read_exact(&mut bytes)?; let mut trailing = [0_u8; 1]; if file.read(&mut trailing)? != 0 { - return Err(invalid_data("retention record carried trailing bytes")); + return Err(RetentionCurrentStateRefusal::RecordTrailingBytes.into_io()); } Ok(Some(bytes.into_boxed_slice())) } diff --git a/src/adapters/retention/filesystem_retention_current_tests.rs b/src/adapters/retention/filesystem_retention_current_tests.rs index 27eac2f..a90ce65 100644 --- a/src/adapters/retention/filesystem_retention_current_tests.rs +++ b/src/adapters/retention/filesystem_retention_current_tests.rs @@ -4,11 +4,11 @@ use std::error::Error; use std::fs; use std::io; -use super::RetentionPublicationError; use super::filesystem_retention_test_fixture::{ - ROOT_HEX, fixture, initial_preparation, manifest_pool_path, open_authority, retention_witness, - root_pool_path, + ROOT_HEX, fixture, initial_preparation, manifest_pool_path, open_authority, refusal, + retention_witness, root_pool_path, }; +use super::{RetentionCurrentStateRefusal, RetentionPublicationError}; use crate::execute_retention_publication; #[test] @@ -29,6 +29,10 @@ fn committed_retry_refuses_when_the_selected_root_is_absent() -> Result<(), Box< return Err("absent root refused outside current-state verification".into()); }; assert_eq!(source.kind(), io::ErrorKind::InvalidData); + assert!(matches!( + refusal(&source), + Some(RetentionCurrentStateRefusal::CommittedRootAbsent) + )); assert_eq!(retention_witness(sandbox.path())?, before); drop(authority); sandbox.remove()?; @@ -55,6 +59,10 @@ fn committed_retry_refuses_when_the_selected_root_bytes_changed() -> Result<(), return Err("changed root refused outside current-state verification".into()); }; assert_eq!(source.kind(), io::ErrorKind::InvalidData); + assert!(matches!( + refusal(&source), + Some(RetentionCurrentStateRefusal::CommittedRootChanged) + )); drop(authority); sandbox.remove()?; Ok(()) diff --git a/src/adapters/retention/filesystem_retention_expectation_tests.rs b/src/adapters/retention/filesystem_retention_expectation_tests.rs index 6e54cf6..c19515e 100644 --- a/src/adapters/retention/filesystem_retention_expectation_tests.rs +++ b/src/adapters/retention/filesystem_retention_expectation_tests.rs @@ -6,9 +6,13 @@ use std::io; use super::filesystem_retention_test_fixture::{ ROOT_HEX, fixture, head_path, initial_preparation, initial_root, new_namespace_preparation, - open_authority, retention_witness, root_pool_path, successor_preparation, successor_root, + open_authority, refusal, retention_witness, root_pool_path, successor_preparation, + successor_root, +}; +use super::{ + AdmittedRetentionManifest, AdmittedRetentionRoot, RetentionCurrentStateRefusal, + RetentionPublicationStorage, }; -use super::{AdmittedRetentionManifest, AdmittedRetentionRoot, RetentionPublicationStorage}; use crate::execute_retention_publication; #[test] @@ -26,6 +30,10 @@ fn absent_head_with_retention_artifacts_refuses_as_recovery() -> Result<(), Box< .ok_or("absent head over populated pools was unexpectedly treated as empty")?; assert_eq!(error.kind(), io::ErrorKind::InvalidData); + assert!(matches!( + refusal(&error), + Some(RetentionCurrentStateRefusal::HeadAbsentWithArtifacts) + )); assert_eq!(retention_witness(sandbox.path())?, before); drop(authority); sandbox.remove()?; diff --git a/src/adapters/retention/filesystem_retention_refusal.rs b/src/adapters/retention/filesystem_retention_refusal.rs new file mode 100644 index 0000000..8d4cd3e --- /dev/null +++ b/src/adapters/retention/filesystem_retention_refusal.rs @@ -0,0 +1,117 @@ +//! This boundary module owns typed refusals from filesystem current-state verification. + +use std::error::Error; +use std::fmt; +use std::io; + +use super::{RetentionHeadDecodeError, RetentionManifestDecodeError}; +use crate::{LivenessGeneration, RetentionManifestDigest}; + +/// Exact reason filesystem current-state verification refused a transition. +/// +/// Every variant is carried as the source of the `io::Error` that +/// [`RetentionPublicationStorage::verify_current`](super::RetentionPublicationStorage::verify_current) +/// returns, so callers can distinguish a lawful stale state that should be +/// replanned from corruption or ambiguity that must route through recovery. +#[derive(Debug)] +#[non_exhaustive] +pub enum RetentionCurrentStateRefusal { + /// A retained `root.next`, `manifest.next`, or `head.next` exists. + RetainedStage, + /// `retention/HEAD` is absent while a pool holds artifacts. + HeadAbsentWithArtifacts, + /// `retention/HEAD` is absent but a current generation was expected. + ExpectedCurrentOverAbsentHead, + /// `retention/HEAD` is absent but the prepared head is not an initial head. + NonInitialOverAbsentHead, + /// The published head refused admission. + HeadRefused { + /// The exact decode refusal. + source: RetentionHeadDecodeError, + }, + /// The prepared successor head refused admission. + PreparedHeadRefused { + /// The exact decode refusal. + source: RetentionHeadDecodeError, + }, + /// The head names a manifest that is absent from the pool. + ManifestAbsent, + /// The head-selected manifest refused admission. + ManifestRefused { + /// The exact decode refusal. + source: RetentionManifestDecodeError, + }, + /// The head-selected manifest disagrees with the head's digest or generation. + ManifestDisagreed, + /// The current liveness generation has no successor. + LivenessExhausted, + /// A byte-identical retry found that another successor is current. + StaleCommittedRetry, + /// The prepared successor does not name the current head as its predecessor. + Superseded { + /// The generation the current head names. + current_generation: LivenessGeneration, + /// The manifest digest the current head names. + current_digest: RetentionManifestDigest, + }, + /// The committed manifest carries no entry for the candidate namespace. + CommittedSelectionMissing, + /// The committed manifest selects a different root for the candidate namespace. + CommittedSelectionMismatch, + /// The committed namespace directory cannot be opened. + CommittedNamespaceUnavailable, + /// The committed root pool entry is absent. + CommittedRootAbsent, + /// The committed root pool entry holds different bytes. + CommittedRootChanged, + /// A record's kind or length disagreed with its declaration. + RecordKindOrLength, + /// A record carried bytes beyond its declared length. + RecordTrailingBytes, + /// A declared record length exceeded the platform's addressable range. + RecordLengthOverflow, +} + +impl RetentionCurrentStateRefusal { + /// Wraps the refusal as the `InvalidData` error the storage port returns. + pub(super) fn into_io(self) -> io::Error { + io::Error::new(io::ErrorKind::InvalidData, self) + } +} + +impl fmt::Display for RetentionCurrentStateRefusal { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::RetainedStage => formatter.write_str("retained retention stage requires recovery before publication"), + Self::HeadAbsentWithArtifacts => formatter.write_str("retention head is absent while retention pools hold artifacts; recovery is required"), + Self::ExpectedCurrentOverAbsentHead => formatter.write_str("expected a current retention generation but no head is published"), + Self::NonInitialOverAbsentHead => formatter.write_str("absent retention head admits only an initial publication with no predecessor"), + Self::HeadRefused { .. } => formatter.write_str("current retention head refused admission"), + Self::PreparedHeadRefused { .. } => formatter.write_str("prepared retention head refused admission"), + Self::ManifestAbsent => formatter.write_str("current retention head names an absent manifest"), + Self::ManifestRefused { .. } => formatter.write_str("current retention manifest refused admission"), + Self::ManifestDisagreed => formatter.write_str("current retention manifest disagreed with its head"), + Self::LivenessExhausted => formatter.write_str("current liveness generation cannot advance"), + Self::StaleCommittedRetry => formatter.write_str("already-committed retry is stale: another successor is current"), + Self::Superseded { current_generation, .. } => write!(formatter, "candidate is superseded: the current head is liveness generation {}", current_generation.get()), + Self::CommittedSelectionMissing => formatter.write_str("committed manifest does not select the candidate namespace"), + Self::CommittedSelectionMismatch => formatter.write_str("committed manifest selects a different root for the candidate namespace"), + Self::CommittedNamespaceUnavailable => formatter.write_str("committed root namespace directory is unavailable"), + Self::CommittedRootAbsent => formatter.write_str("committed root pool entry is absent"), + Self::CommittedRootChanged => formatter.write_str("committed root pool entry bytes disagreed"), + Self::RecordKindOrLength => formatter.write_str("retention record kind or length disagreed"), + Self::RecordTrailingBytes => formatter.write_str("retention record carried trailing bytes"), + Self::RecordLengthOverflow => formatter.write_str("retention record length exceeded the addressable range"), + } + } +} + +impl Error for RetentionCurrentStateRefusal { + fn source(&self) -> Option<&(dyn Error + 'static)> { + match self { + Self::HeadRefused { source } | Self::PreparedHeadRefused { source } => Some(source), + Self::ManifestRefused { source } => Some(source), + _ => None, + } + } +} diff --git a/src/adapters/retention/filesystem_retention_storage.rs b/src/adapters/retention/filesystem_retention_storage.rs index 00f8068..04b819a 100644 --- a/src/adapters/retention/filesystem_retention_storage.rs +++ b/src/adapters/retention/filesystem_retention_storage.rs @@ -12,8 +12,8 @@ use super::filesystem_retention_pool_name as pool_name; use super::filesystem_retention_stage::{FilesystemRetentionStage, invalid_data}; use super::{ AdmittedRetentionRoot, CanonicalRetentionHead, CanonicalRetentionManifest, - RetentionNamespaceAdmission, RetentionPublicationPreparation, RetentionPublicationStorage, - RetentionTransitionDisposition, + RetentionCurrentStateRefusal, RetentionNamespaceAdmission, RetentionPublicationPreparation, + RetentionPublicationStorage, RetentionTransitionDisposition, }; use crate::adapters::filesystem_catalog_artifact::synchronize_directory; @@ -28,9 +28,7 @@ impl RetentionPublicationStorage for FilesystemRetentionPublicationAuthority { filesystem_retention_namespace::admit(&self.retention, &self.roots, &self.manifests)?; let current = filesystem_retention_current::observe(&self.retention, &self.manifests)?; if current.is_none() && !census.is_empty() { - return Err(invalid_data( - "retention head is absent while retention pools hold artifacts; recovery is required", - )); + return Err(RetentionCurrentStateRefusal::HeadAbsentWithArtifacts.into_io()); } let disposition = filesystem_retention_current::disposition(preparation, current.as_ref())?; if disposition == RetentionTransitionDisposition::Publish { @@ -180,9 +178,7 @@ fn require_no_retained_stage(retention: &Dir) -> io::Result<()> { match retention.symlink_metadata(stage) { Err(source) if source.kind() == io::ErrorKind::NotFound => {} Ok(_) => { - return Err(invalid_data( - "retained retention stage requires recovery before publication", - )); + return Err(RetentionCurrentStateRefusal::RetainedStage.into_io()); } Err(source) => return Err(source), } diff --git a/src/adapters/retention/filesystem_retention_successor_tests.rs b/src/adapters/retention/filesystem_retention_successor_tests.rs index 56d91f6..a716fc2 100644 --- a/src/adapters/retention/filesystem_retention_successor_tests.rs +++ b/src/adapters/retention/filesystem_retention_successor_tests.rs @@ -6,12 +6,13 @@ use std::io; use super::filesystem_retention_test_fixture::{ ROOT_HEX, fixture, head_path, initial_generation, initial_preparation, manifest_pool_path, - open_authority, retention_witness, root_pool_path, successor_preparation, successor_root, + open_authority, refusal, retention_witness, root_pool_path, successor_preparation, + successor_root, }; use super::{ AdmittedRetentionManifest, AdmittedRetentionRoot, ChecksummedRetentionHead, - RetentionNamespaceAdmission, RetentionPublicationError, RetentionPublicationOutcome, - RetentionPublicationStorage, + RetentionCurrentStateRefusal, RetentionNamespaceAdmission, RetentionPublicationError, + RetentionPublicationOutcome, RetentionPublicationStorage, }; use crate::execute_retention_publication; @@ -90,6 +91,10 @@ fn superseded_candidate_refuses_once_a_successor_is_current() -> Result<(), Box< return Err("superseded candidate refused outside current-state verification".into()); }; assert_eq!(source.kind(), io::ErrorKind::InvalidData); + assert!(matches!( + refusal(&source), + Some(RetentionCurrentStateRefusal::Superseded { .. }) + )); assert_eq!(retention_witness(sandbox.path())?, after_successor); drop(authority); sandbox.remove()?; diff --git a/src/adapters/retention/filesystem_retention_test_fixture.rs b/src/adapters/retention/filesystem_retention_test_fixture.rs index 1f3a1a0..a72aca1 100644 --- a/src/adapters/retention/filesystem_retention_test_fixture.rs +++ b/src/adapters/retention/filesystem_retention_test_fixture.rs @@ -11,7 +11,7 @@ use std::path::{Path, PathBuf}; use super::filesystem_retention_authority::FilesystemRetentionPublicationAuthority; use super::{ AdmittedRetentionManifest, AdmittedRetentionRoot, CanonicalRetentionRoot, - RetentionPublicationPreparation, RetentionTransitionDisposition, + RetentionCurrentStateRefusal, RetentionPublicationPreparation, RetentionTransitionDisposition, }; use crate::LayoutEntryLimit; use crate::adapters::filesystem_test_sandbox::TestDirectory; @@ -169,6 +169,13 @@ pub(super) fn successor_root( CanonicalRetentionRoot::from_root(&root).map_err(Into::into) } +/// Extracts the typed current-state refusal carried by a verification error. +pub(super) fn refusal(source: &io::Error) -> Option<&RetentionCurrentStateRefusal> { + source + .get_ref() + .and_then(|inner| inner.downcast_ref::()) +} + pub(super) fn head_path(root: &Path) -> PathBuf { root.join("retention").join("HEAD") } diff --git a/src/lib.rs b/src/lib.rs index 64a75ca..2ce7c18 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -137,15 +137,15 @@ pub use adapters::{ CanonicalRetentionManifest, CanonicalRetentionRoot, ChecksummedRetentionHead, FilesystemRetentionAuthorityError, FilesystemRetentionPublicationAuthority, ObservedRetentionState, PreparedRetentionPublication, RetentionAuthorityDirectory, - RetentionClosureVerificationError, RetentionHeadDecodeError, RetentionManifestDecodeError, - RetentionManifestEncodeError, RetentionNamespaceAdmission, RetentionPublicationError, - RetentionPublicationOutcome, RetentionPublicationPhase, RetentionPublicationPreparation, - RetentionPublicationPreparationError, RetentionPublicationReceipt, RetentionPublicationStorage, - RetentionRootDecodeError, RetentionRootEncodeError, RetentionTransitionDisposition, - RetentionTransitionError, RetentionTransitionPreflight, RetentionTransitionPreflightError, - RetentionTransitionReadiness, VerifiedRetentionClosure, execute_retention_publication, - plan_retention_transition, preflight_retention_transition, prepare_retention_publication, - verify_retention_closure, + RetentionClosureVerificationError, RetentionCurrentStateRefusal, RetentionHeadDecodeError, + RetentionManifestDecodeError, RetentionManifestEncodeError, RetentionNamespaceAdmission, + RetentionPublicationError, RetentionPublicationOutcome, RetentionPublicationPhase, + RetentionPublicationPreparation, RetentionPublicationPreparationError, + RetentionPublicationReceipt, RetentionPublicationStorage, RetentionRootDecodeError, + RetentionRootEncodeError, RetentionTransitionDisposition, RetentionTransitionError, + RetentionTransitionPreflight, RetentionTransitionPreflightError, RetentionTransitionReadiness, + VerifiedRetentionClosure, execute_retention_publication, plan_retention_transition, + preflight_retention_transition, prepare_retention_publication, verify_retention_closure, }; pub use blob::{ BlobHashError, BlobHasher, BlobId, BlobLength, BlobReadError, ByteLength, ByteOffset, From e398c5d0e2da3fc4670c4152f9ba9d9d3bea4753 Mon Sep 17 00:00:00 2001 From: James Ross Date: Mon, 7 Sep 2026 00:45:23 -0700 Subject: [PATCH 068/111] Fix: bind observed retention state to this store's catalog and its own history Two Codex second-pass findings on observed-state consistency, fixed together because both add refusals to the same observation path and share one invariant: the state a publication proceeds from must be internally consistent and must belong to this store. Catalog binding. A preparation carries a closure verified against one pinned CatalogSnapshot that the caller may have taken from another store, and no filesystem phase revalidated it. Before any forward write the authority now reopens this store's catalog HEAD and requires it to name exactly the closure's catalog generation and digest; a foreign or undecodable head refuses (CatalogDisagreed, CatalogHeadRefused). The authority pins the store root for this purpose. Verifying the closure's records against the pools remains the reader-fence and verification work tracked in #19 and #20. Predecessor cross-check. observe() accepted a head and manifest that were each well-formed but named different predecessors, letting a later publication advance from the manifest while replacing the inconsistent head. The head's predecessor must now equal the admitted manifest's predecessor (HeadPredecessorDisagreed). Regression laws: a same-generation head naming a different catalog refuses before staging with an unchanged witness; a correctly checksummed head whose predecessor disagrees with its manifest refuses on observation. Addresses Codex threads on filesystem_retention_storage.rs (closure binding) and filesystem_retention_current.rs:67. Refs #78 --- CHANGELOG.md | 6 ++ docs/formats/segment-store-v2/requirements.md | 2 +- src/adapters/retention.rs | 3 + .../filesystem_retention_authority.rs | 2 + .../retention/filesystem_retention_catalog.rs | 46 +++++++++++++++ .../filesystem_retention_catalog_tests.rs | 40 +++++++++++++ .../retention/filesystem_retention_current.rs | 5 +- .../filesystem_retention_current_tests.rs | 56 +++++++++++++++++-- .../retention/filesystem_retention_refusal.rs | 16 +++++- .../retention/filesystem_retention_storage.rs | 2 + 10 files changed, 171 insertions(+), 7 deletions(-) create mode 100644 src/adapters/retention/filesystem_retention_catalog.rs create mode 100644 src/adapters/retention/filesystem_retention_catalog_tests.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index 423692b..4b275e1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -69,6 +69,12 @@ after its public API and format compatibility policies are established. returns, so a superseded candidate, a stale committed retry, an absent head over populated pools, and each corruption or decode refusal are distinguishable to callers and preserve their underlying decode errors. + Before any forward retention write, the authority reopens this store's own + catalog `HEAD` and requires it to name exactly the catalog generation and + digest the candidate closure was verified against, so a preparation built + from another store's `CatalogSnapshot` refuses instead of publishing anchors + whose records these pools may not hold. Observing the current state also + requires the head's predecessor digest to equal its manifest's predecessor. - Repository crash-matrix execution now terminates isolated writer process groups at all 105 canonical before/during/after coordinates, retains open writer and stage authority until termination, executes production diff --git a/docs/formats/segment-store-v2/requirements.md b/docs/formats/segment-store-v2/requirements.md index d7e7c33..b81d7a2 100644 --- a/docs/formats/segment-store-v2/requirements.md +++ b/docs/formats/segment-store-v2/requirements.md @@ -12,7 +12,7 @@ case is not evidence. | `KEEP-RETENTION-001` | `RetentionNamespace`, `RootGeneration`, `LivenessGeneration`, profile coordinates, limits, anchors, and digests are validated typed values | `tests/retention_values.rs`, `tests/retention_root_encoding.rs`, and typed verified anchor-set evidence in `tests/retention_root_decoding.rs` | Implemented | | `KEEP-RETENTION-002` | Root, manifest, and head codecs implement the exact canonical grammars and fixed bounds | independent golden corpus plus `tests/retention_root_encoding.rs`, `tests/retention_root_decoding.rs`, `tests/retention_manifest_codec.rs`, and `tests/retention_head_codec.rs` | Implemented | | `KEEP-RETENTION-003` | Every structural field, truncation boundary, ordering law, duplicate, overflow, flag, reserved byte, digest, checksum, and trailing byte has a precise refusal | seeded `retention_format` fuzz target plus the root, manifest, and head corruption matrix; mutation coverage remains | In progress in #19 | -| `KEEP-RETENTION-004` | Retain and release compare expected and observed generations and publish exact successors only | unforgeable readiness and preflight proofs in `tests/retention_transition.rs` and `tests/retention_preflight.rs`; exact successor preparation and complete receipt evidence in `tests/retention_publication_preparation.rs` and `tests/retention_publication_execution.rs`; writer-locked initial filesystem publication in `filesystem_retention_storage_tests`; observed-head successor publication, exact predecessor binding, and absent-head refusal in `filesystem_retention_successor_tests` | Implemented | +| `KEEP-RETENTION-004` | Retain and release compare expected and observed generations and publish exact successors only | unforgeable readiness and preflight proofs in `tests/retention_transition.rs` and `tests/retention_preflight.rs`; exact successor preparation and complete receipt evidence in `tests/retention_publication_preparation.rs` and `tests/retention_publication_execution.rs`; writer-locked initial filesystem publication in `filesystem_retention_storage_tests`; observed-head successor publication, exact predecessor binding, and absent-head refusal in `filesystem_retention_successor_tests`; the store's catalog head must name the closure's catalog generation and digest before any forward write in `filesystem_retention_catalog_tests`; a head whose predecessor disagrees with its manifest refuses in `filesystem_retention_current_tests` | Implemented | | `KEEP-RETENTION-005` | Closure derivation is deterministic, bounded, cycle-safe, fail-closed, and verifies complete blob reconstruction | exact accounting, reconstruction, adversarial-catalog, and exhaustive model laws in `tests/retention_closure.rs`; corrupt members refuse through the inherited segment-record admission laws and seeded `segment_format` fuzz target routed by `closure-corruption.md` | Implemented | | `KEEP-RETENTION-006` | Publication follows the exact ordered durability protocol, including new namespace-directory admission and retention of fixed-stage evidence until head commit, and returns only after cleanup synchronization | typed vocabulary and blocking port in `tests/retention_publication_phase.rs` and `tests/retention_publication_storage.rs`; ordered execution, conditional namespace sync, and all 17 exact storage-fault boundaries in `tests/retention_publication_execution.rs`; production 17-phase forward filesystem execution, exclusive staging, byte-equal inode-substitution refusal, and retained-stage recovery refusal in `filesystem_retention_storage_tests`; orphan namespace directories count against the 4,096 ceiling and refuse a new namespace before any stage is written in `filesystem_retention_capacity_tests`; crash injection remains | In progress in #19 | | `KEEP-RETENTION-007` | Restart resolves every fixed-stage crash prefix to one documented lawful state or typed ambiguity | recovery-required refusals before any mutation in `filesystem_retention_expectation_tests`: an absent head over populated pools, a non-initial head prepared against an absent head, an orphan directory for a namespace expected absent, and an absent directory for a namespace expected current; debug and release crash matrix remains | In progress in #19 | diff --git a/src/adapters/retention.rs b/src/adapters/retention.rs index 2ed7db9..4dc84eb 100644 --- a/src/adapters/retention.rs +++ b/src/adapters/retention.rs @@ -17,6 +17,9 @@ mod filesystem_retention_authority; mod filesystem_retention_authority_error; #[cfg(test)] mod filesystem_retention_capacity_tests; +mod filesystem_retention_catalog; +#[cfg(test)] +mod filesystem_retention_catalog_tests; mod filesystem_retention_current; #[cfg(test)] mod filesystem_retention_current_tests; diff --git a/src/adapters/retention/filesystem_retention_authority.rs b/src/adapters/retention/filesystem_retention_authority.rs index 3faeda1..e1a09d9 100644 --- a/src/adapters/retention/filesystem_retention_authority.rs +++ b/src/adapters/retention/filesystem_retention_authority.rs @@ -27,6 +27,7 @@ use crate::adapters::{FilesystemVersionTwoAdmission, FilesystemWriterLock}; /// prefix remains a separate recovery boundary. #[must_use] pub struct FilesystemRetentionPublicationAuthority { + pub(super) root: Dir, pub(super) retention: Dir, pub(super) roots: Dir, pub(super) manifests: Dir, @@ -64,6 +65,7 @@ impl FilesystemRetentionPublicationAuthority { let roots = open_directory(&retention, pool_name::ROOTS, Directory::Roots)?; let manifests = open_directory(&retention, pool_name::MANIFESTS, Directory::Manifests)?; Ok(Self { + root, retention, roots, manifests, diff --git a/src/adapters/retention/filesystem_retention_catalog.rs b/src/adapters/retention/filesystem_retention_catalog.rs new file mode 100644 index 0000000..850bdec --- /dev/null +++ b/src/adapters/retention/filesystem_retention_catalog.rs @@ -0,0 +1,46 @@ +//! This module binds retention closure evidence to this store's own catalog head. + +use std::io; + +use cap_std::fs::Dir; + +use super::filesystem_retention_current::read_exact_optional; +use super::{RetentionCurrentStateRefusal, RetentionPublicationPreparation}; +use crate::adapters::ChecksummedPublicationHead; + +const HEAD_NAME: &str = "HEAD"; +const HEAD_LENGTH: usize = 128; + +/// Requires the store's catalog head to name the catalog the closure was verified against. +/// +/// A preparation carries a closure verified against one pinned `CatalogSnapshot`, +/// which the caller may have taken from another store. Publication must not +/// proceed unless this store's own `HEAD` names exactly that catalog generation +/// and digest; otherwise the receipt would cite foreign evidence for anchors +/// whose records may be absent from these pools. +pub(super) fn require_current_catalog( + root: &Dir, + preparation: &RetentionPublicationPreparation<'_>, +) -> io::Result<()> { + let closure = preparation.closure(); + let expected_generation = closure.catalog_generation(); + let bytes = read_exact_optional(root, HEAD_NAME, HEAD_LENGTH)?.ok_or_else(|| { + RetentionCurrentStateRefusal::CatalogDisagreed { + expected_generation, + observed_generation: None, + } + .into_io() + })?; + let head = ChecksummedPublicationHead::decode(&bytes) + .map_err(|_source| RetentionCurrentStateRefusal::CatalogHeadRefused.into_io())?; + if head.generation() == expected_generation && head.catalog_digest() == closure.catalog_digest() + { + Ok(()) + } else { + Err(RetentionCurrentStateRefusal::CatalogDisagreed { + expected_generation, + observed_generation: Some(head.generation()), + } + .into_io()) + } +} diff --git a/src/adapters/retention/filesystem_retention_catalog_tests.rs b/src/adapters/retention/filesystem_retention_catalog_tests.rs new file mode 100644 index 0000000..2eff165 --- /dev/null +++ b/src/adapters/retention/filesystem_retention_catalog_tests.rs @@ -0,0 +1,40 @@ +//! Filesystem retention catalog-binding laws: closure evidence must belong to this store. + +use std::error::Error; +use std::fs; +use std::io; + +use super::filesystem_retention_test_fixture::{ + ROOT_HEX, fixture, initial_preparation, open_authority, refusal, retention_witness, +}; +use super::{RetentionCurrentStateRefusal, RetentionPublicationStorage}; + +/// A generation-one version-one head that names a different catalog digest. +const FOREIGN_CATALOG_HEAD_HEX: &str = + include_str!("../../../conformance/segment-store/v1/one-zero-head.hex"); + +#[test] +fn closure_verified_against_another_catalog_refuses_before_staging() -> Result<(), Box> { + let (sandbox, mut authority) = open_authority("filesystem-retention-foreign-catalog")?; + fs::write( + sandbox.path().join("HEAD"), + fixture(FOREIGN_CATALOG_HEAD_HEX)?, + )?; + let before = retention_witness(sandbox.path())?; + let root_bytes = fixture(ROOT_HEX)?; + let preparation = initial_preparation(&root_bytes)?; + + let error = RetentionPublicationStorage::verify_current(&mut authority, &preparation) + .err() + .ok_or("closure verified against a foreign catalog was unexpectedly admitted")?; + + assert_eq!(error.kind(), io::ErrorKind::InvalidData); + assert!(matches!( + refusal(&error), + Some(RetentionCurrentStateRefusal::CatalogDisagreed { .. }) + )); + assert_eq!(retention_witness(sandbox.path())?, before); + drop(authority); + sandbox.remove()?; + Ok(()) +} diff --git a/src/adapters/retention/filesystem_retention_current.rs b/src/adapters/retention/filesystem_retention_current.rs index 76aa7c1..78dccdc 100644 --- a/src/adapters/retention/filesystem_retention_current.rs +++ b/src/adapters/retention/filesystem_retention_current.rs @@ -67,6 +67,9 @@ pub(super) fn observe( { return Err(RetentionCurrentStateRefusal::ManifestDisagreed.into_io()); } + if admitted.manifest().predecessor() != selected.predecessor() { + return Err(RetentionCurrentStateRefusal::HeadPredecessorDisagreed.into_io()); + } Ok(Some(ObservedRetentionState { head, manifest })) } @@ -176,7 +179,7 @@ fn require_initial_publication( } } -fn read_exact_optional( +pub(super) fn read_exact_optional( directory: &Dir, name: &str, length: usize, diff --git a/src/adapters/retention/filesystem_retention_current_tests.rs b/src/adapters/retention/filesystem_retention_current_tests.rs index a90ce65..efcb861 100644 --- a/src/adapters/retention/filesystem_retention_current_tests.rs +++ b/src/adapters/retention/filesystem_retention_current_tests.rs @@ -5,11 +5,14 @@ use std::fs; use std::io; use super::filesystem_retention_test_fixture::{ - ROOT_HEX, fixture, initial_preparation, manifest_pool_path, open_authority, refusal, - retention_witness, root_pool_path, + ROOT_HEX, fixture, head_path, initial_preparation, manifest_pool_path, open_authority, refusal, + retention_witness, root_pool_path, successor_preparation, successor_root, }; -use super::{RetentionCurrentStateRefusal, RetentionPublicationError}; -use crate::execute_retention_publication; +use super::{ + AdmittedRetentionManifest, AdmittedRetentionRoot, CanonicalRetentionHead, + RetentionCurrentStateRefusal, RetentionPublicationError, +}; +use crate::{RetentionHead, RetentionManifestLength, execute_retention_publication}; #[test] fn committed_retry_refuses_when_the_selected_root_is_absent() -> Result<(), Box> { @@ -93,3 +96,48 @@ fn committed_retry_refuses_when_the_selected_manifest_is_corrupt() -> Result<(), sandbox.remove()?; Ok(()) } + +#[test] +fn head_predecessor_disagreeing_with_its_manifest_refuses() -> Result<(), Box> { + let (sandbox, mut authority) = open_authority("filesystem-retention-predecessor-disagrees")?; + let root_bytes = fixture(ROOT_HEX)?; + let _published = + execute_retention_publication(&mut authority, &initial_preparation(&root_bytes)?)?; + let current = authority + .observe_current()? + .ok_or("published retention head was not observed")?; + let current_root = AdmittedRetentionRoot::decode(&root_bytes)?; + let current_manifest = AdmittedRetentionManifest::decode(current.manifest_bytes())?; + let candidate = successor_root(¤t_root)?; + let successor = successor_preparation(¤t_root, ¤t_manifest, candidate.encoded())?; + let _advanced = execute_retention_publication(&mut authority, &successor)?; + let advanced = authority + .observe_current()? + .ok_or("advanced retention head was not observed")?; + let advanced_manifest = AdmittedRetentionManifest::decode(advanced.manifest_bytes())?; + let wrong_predecessor = advanced_manifest.digest(); + let inconsistent = RetentionHead::new( + advanced_manifest.manifest().generation(), + RetentionManifestLength::new(u64::try_from(advanced.manifest_bytes().len())?)?, + advanced_manifest.digest(), + Some(wrong_predecessor), + )?; + fs::write( + head_path(sandbox.path()), + CanonicalRetentionHead::from_head(&inconsistent).encoded(), + )?; + + let error = authority + .observe_current() + .err() + .ok_or("head with a disagreeing predecessor was unexpectedly observed")?; + + assert_eq!(error.kind(), io::ErrorKind::InvalidData); + assert!(matches!( + refusal(&error), + Some(RetentionCurrentStateRefusal::HeadPredecessorDisagreed) + )); + drop(authority); + sandbox.remove()?; + Ok(()) +} diff --git a/src/adapters/retention/filesystem_retention_refusal.rs b/src/adapters/retention/filesystem_retention_refusal.rs index 8d4cd3e..1ff0960 100644 --- a/src/adapters/retention/filesystem_retention_refusal.rs +++ b/src/adapters/retention/filesystem_retention_refusal.rs @@ -5,7 +5,7 @@ use std::fmt; use std::io; use super::{RetentionHeadDecodeError, RetentionManifestDecodeError}; -use crate::{LivenessGeneration, RetentionManifestDigest}; +use crate::{CatalogGeneration, LivenessGeneration, RetentionManifestDigest}; /// Exact reason filesystem current-state verification refused a transition. /// @@ -43,6 +43,17 @@ pub enum RetentionCurrentStateRefusal { }, /// The head-selected manifest disagrees with the head's digest or generation. ManifestDisagreed, + /// The head's predecessor digest disagrees with its manifest's predecessor. + HeadPredecessorDisagreed, + /// The store's catalog head is not the catalog the closure was verified against. + CatalogDisagreed { + /// The catalog generation the closure was verified against. + expected_generation: CatalogGeneration, + /// The catalog generation the store's head names, if it decoded. + observed_generation: Option, + }, + /// The store's catalog head refused admission. + CatalogHeadRefused, /// The current liveness generation has no successor. LivenessExhausted, /// A byte-identical retry found that another successor is current. @@ -91,6 +102,9 @@ impl fmt::Display for RetentionCurrentStateRefusal { Self::ManifestAbsent => formatter.write_str("current retention head names an absent manifest"), Self::ManifestRefused { .. } => formatter.write_str("current retention manifest refused admission"), Self::ManifestDisagreed => formatter.write_str("current retention manifest disagreed with its head"), + Self::HeadPredecessorDisagreed => formatter.write_str("current retention head and its manifest name different predecessors"), + Self::CatalogDisagreed { expected_generation, .. } => write!(formatter, "closure was verified against catalog generation {} which is not this store's current catalog", expected_generation.get()), + Self::CatalogHeadRefused => formatter.write_str("this store's catalog head refused admission"), Self::LivenessExhausted => formatter.write_str("current liveness generation cannot advance"), Self::StaleCommittedRetry => formatter.write_str("already-committed retry is stale: another successor is current"), Self::Superseded { current_generation, .. } => write!(formatter, "candidate is superseded: the current head is liveness generation {}", current_generation.get()), diff --git a/src/adapters/retention/filesystem_retention_storage.rs b/src/adapters/retention/filesystem_retention_storage.rs index 04b819a..05d85b1 100644 --- a/src/adapters/retention/filesystem_retention_storage.rs +++ b/src/adapters/retention/filesystem_retention_storage.rs @@ -6,6 +6,7 @@ use cap_fs_ext::DirExt; use cap_std::fs::Dir; use super::filesystem_retention_authority::FilesystemRetentionPublicationAuthority; +use super::filesystem_retention_catalog; use super::filesystem_retention_current; use super::filesystem_retention_namespace; use super::filesystem_retention_pool_name as pool_name; @@ -32,6 +33,7 @@ impl RetentionPublicationStorage for FilesystemRetentionPublicationAuthority { } let disposition = filesystem_retention_current::disposition(preparation, current.as_ref())?; if disposition == RetentionTransitionDisposition::Publish { + filesystem_retention_catalog::require_current_catalog(&self.root, preparation)?; filesystem_retention_namespace::admit_expectation( &self.roots, preparation.candidate(), From 9643a02e4c5ba075c5edcc338094744bba12724e Mon Sep 17 00:00:00 2001 From: James Ross Date: Mon, 7 Sep 2026 09:53:33 -0700 Subject: [PATCH 069/111] Refactor: restore the direct restart artifact read and remove the transfer layer catalog_restart_io::read_exact pumped every artifact through an 8 KiB stack buffer via ExactTransfer, copy_exact, copy_exact_to_chunks, and VecWrite into a Vec that was already try_reserve_exact'd to the full length. The layer cost about 131,072 read(2) calls per 1 GiB segment on every restart and lowered no peak memory: an artifact that cannot fit in process memory refused at the reservation before and refuses at the reservation now. read_exact was the layer's only non-test caller. read_exact is restored to one exact read into the pre-reserved buffer followed by trailing-byte rejection, generic over Read so its laws run against a Cursor. The three streaming laws that tested the deleted API and the buffer's memory budget are removed with it, along with catalog_restart_io_test_doubles.rs. The short-read and trailing-byte refusals are retargeted at read_exact and a positive exact-bytes law is added. Every CatalogRestartError variant is unchanged; only the short-read source's Display text becomes std's, which nothing asserts. Self-review finding A10 (P3). Decision recorded in the plan: restore and delete rather than keep the layer for #72. Refs #78 --- CHANGELOG.md | 4 + src/adapters/catalog_restart_io.rs | 399 +++--------------- .../catalog_restart_io_test_doubles.rs | 180 -------- src/adapters/mod.rs | 2 - 4 files changed, 52 insertions(+), 533 deletions(-) delete mode 100644 src/adapters/catalog_restart_io_test_doubles.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index 4b275e1..7f29fb4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,10 @@ after its public API and format compatibility policies are established. ### Changed +- Restart artifact reads use one exact read into a pre-reserved buffer followed + by trailing-byte rejection; the interim chunked transfer layer, which pumped + every artifact through an 8 KiB buffer without lowering peak memory, is + removed with no change to refusal behaviour. - Version-2 marker, typed canonical intent/receipt construction, and record admission bind exact catalog, predecessor, root, definition, store, empty-state, and checksum, digest, and synchronization-mask coordinates; migration fuzzing drives all diff --git a/src/adapters/catalog_restart_io.rs b/src/adapters/catalog_restart_io.rs index 3af6b0f..24c721d 100644 --- a/src/adapters/catalog_restart_io.rs +++ b/src/adapters/catalog_restart_io.rs @@ -1,7 +1,6 @@ -//! This module owns exact capability-relative restart artifact reads and -//! bounded, exact-transfer streaming. +//! This module owns exact capability-relative restart artifact reads. -use std::io::{self, Read, Write}; +use std::io::{self, Read}; use std::path::Path; use cap_fs_ext::{FollowSymlinks, OpenOptionsFollowExt, OpenOptionsSyncExt}; @@ -10,41 +9,6 @@ use cap_std::fs::{Dir, File, OpenOptions}; use super::{CatalogRestartArtifact, CatalogRestartError, CatalogRestartPhase}; -const CATALOG_RESTART_READ_BUFFER_LENGTH: usize = 8_192; - -#[derive(Clone, Copy, Debug)] -pub(super) struct ExactTransfer { - artifact: CatalogRestartArtifact, - phase: CatalogRestartPhase, - expected: u64, -} - -impl ExactTransfer { - pub(super) const fn new( - artifact: CatalogRestartArtifact, - phase: CatalogRestartPhase, - expected: u64, - ) -> Self { - Self { - artifact, - phase, - expected, - } - } - - pub(super) const fn artifact(&self) -> CatalogRestartArtifact { - self.artifact - } - - pub(super) const fn phase(&self) -> CatalogRestartPhase { - self.phase - } - - pub(super) const fn expected(&self) -> u64 { - self.expected - } -} - pub(super) fn open_root(root: &Path) -> Result { Dir::open_ambient_dir(root, ambient_authority()) .map_err(|source| CatalogRestartError::io(CatalogRestartPhase::OpenRoot, source)) @@ -70,111 +34,41 @@ pub(super) fn open_regular( Ok((file, metadata.len())) } -pub(super) fn read_exact( - mut file: File, +/// Reads exactly `expected` bytes into one pre-reserved buffer and refuses any trailing byte. +/// +/// The complete artifact is reserved before the first read, so an artifact +/// that cannot fit in process memory refuses with +/// [`CatalogRestartError::Allocation`] and never allocates. A short source +/// refuses with the `phase` I/O error, and a longer source refuses with the +/// exact observed length. +pub(super) fn read_exact( + mut source: R, artifact: CatalogRestartArtifact, phase: CatalogRestartPhase, expected: u64, ) -> Result, CatalogRestartError> { - let transfer = ExactTransfer::new(artifact, phase, expected); - let host_length = usize::try_from(transfer.expected()).map_err(|_source| { - CatalogRestartError::Allocation { - artifact: transfer.artifact(), - byte_count: transfer.expected(), + let host_length = + usize::try_from(expected).map_err(|_source| CatalogRestartError::Allocation { + artifact, + byte_count: expected, source: None, - } - })?; - + })?; let mut encoded = Vec::new(); encoded .try_reserve_exact(host_length) .map_err(|source| CatalogRestartError::Allocation { - artifact: transfer.artifact(), + artifact, byte_count: expected, source: Some(source), })?; - let mut sink = VecWrite { - encoded: &mut encoded, - }; - copy_exact(&mut file, &mut sink, transfer)?; + encoded.resize(host_length, 0); + source + .read_exact(&mut encoded) + .map_err(|source| CatalogRestartError::io(phase, source))?; + reject_trailing_bytes(&mut source, artifact, phase, expected)?; Ok(encoded) } -pub(super) fn copy_exact( - source: &mut R, - destination: &mut W, - transfer: ExactTransfer, -) -> Result -where - R: Read, - W: io::Write, -{ - copy_exact_to_chunks(source, transfer, |chunk| { - destination - .write_all(chunk) - .map_err(|source| CatalogRestartError::io(transfer.phase(), source)) - }) -} - -pub(super) fn copy_exact_to_chunks( - source: &mut R, - transfer: ExactTransfer, - mut on_chunk: F, -) -> Result -where - R: Read, - F: FnMut(&[u8]) -> Result<(), CatalogRestartError>, -{ - let artifact = transfer.artifact(); - let phase = transfer.phase(); - let expected = transfer.expected(); - - let mut observed = 0_u64; - let mut buffer = [0_u8; CATALOG_RESTART_READ_BUFFER_LENGTH]; - let chunk_length = u64::try_from(buffer.len()) - .map_err(|_source| CatalogRestartError::LengthArithmetic { artifact, expected })?; - - while observed < expected { - let remaining = expected - .checked_sub(observed) - .ok_or(CatalogRestartError::LengthArithmetic { artifact, expected })?; - let offered = remaining - .min(chunk_length) - .try_into() - .map_err(|_source| CatalogRestartError::LengthArithmetic { artifact, expected })?; - let read_buffer = buffer - .get_mut(..offered) - .ok_or(CatalogRestartError::LengthArithmetic { artifact, expected })?; - match source.read(read_buffer) { - Ok(0) => { - return Err(CatalogRestartError::io( - phase, - io::Error::new( - io::ErrorKind::UnexpectedEof, - "restart artifact ended before the expected boundary", - ), - )); - } - Ok(count) => { - let bytes = read_buffer - .get(..count) - .ok_or(CatalogRestartError::LengthArithmetic { artifact, expected })?; - on_chunk(bytes)?; - let increment = u64::try_from(count).map_err(|_source| { - CatalogRestartError::LengthArithmetic { artifact, expected } - })?; - observed = observed - .checked_add(increment) - .ok_or(CatalogRestartError::LengthArithmetic { artifact, expected })?; - } - Err(source) if source.kind() == io::ErrorKind::Interrupted => {} - Err(source) => return Err(CatalogRestartError::io(phase, source)), - } - } - reject_trailing_bytes(source, artifact, phase, expected)?; - Ok(observed) -} - fn reject_trailing_bytes( source: &mut R, artifact: CatalogRestartArtifact, @@ -205,228 +99,37 @@ fn reject_trailing_bytes( } } -struct VecWrite<'a> { - encoded: &'a mut Vec, -} - -impl Write for VecWrite<'_> { - fn write(&mut self, bytes: &[u8]) -> io::Result { - self.encoded.extend_from_slice(bytes); - Ok(bytes.len()) - } - - fn flush(&mut self) -> io::Result<()> { - Ok(()) - } -} - #[cfg(test)] mod tests { use std::error::Error; - use std::io; - use std::io::{Cursor, ErrorKind}; - use std::mem::size_of; + use std::io::{self, Cursor, ErrorKind}; - use super::super::catalog_restart_io_test_doubles::{ - StreamingCallbackBudget, StreamingWriteSink, SyntheticStreamingReader, - }; use super::*; #[test] - fn read_exact_to_streams_large_virtual_file_with_small_callback_memory() - -> Result<(), Box> { - const TOTAL_BYTES: u64 = 64_u64 * 1024 * 1024; - const READER_STRIDE_BYTES: usize = 2_usize * 1024; - const CALLBACK_BUDGET_BYTES: usize = 16 * 1024; - let Ok(reader_stride) = u64::try_from(READER_STRIDE_BYTES) else { - return Err(Box::new(io::Error::new( - io::ErrorKind::InvalidData, - "reader stride is outside supported range", - ))); - }; - - let mut source = SyntheticStreamingReader::new(TOTAL_BYTES, reader_stride); - let mut budget = StreamingCallbackBudget::new(TOTAL_BYTES, CALLBACK_BUDGET_BYTES); - - let _observed = copy_exact_to_chunks( - &mut source, - ExactTransfer::new( - CatalogRestartArtifact::Head, - CatalogRestartPhase::ReadCatalog, - TOTAL_BYTES, - ), - |chunk| budget.consume(chunk), - )?; - - assert!(budget.observed_bytes() > 0); - assert_eq!(budget.observed_bytes(), TOTAL_BYTES); - assert!( - budget.max_chunk() >= READER_STRIDE_BYTES, - "reader stride should be observed" - ); - assert!( - budget.max_chunk() <= budget.callback_limit(), - "callback should remain in budget" - ); - assert!( - budget.max_chunk() <= CATALOG_RESTART_READ_BUFFER_LENGTH, - "read buffer bounds should hold" - ); - assert!(budget.total_chunks() > 0); - assert!( - size_of::() < 128, - "callback state should stay compact" - ); - assert_eq!(budget.observed_bytes(), TOTAL_BYTES); - Ok(()) - } - - #[test] - fn copy_exact_streams_large_virtual_file_with_small_writer_state() -> Result<(), Box> - { - const TOTAL_BYTES: u64 = 64_u64 * 1024 * 1024; - const READER_STRIDE_BYTES: usize = 2_usize * 1024; - const WRITER_BUDGET_BYTES: usize = 4 * 1024; - let Ok(reader_stride) = u64::try_from(READER_STRIDE_BYTES) else { - return Err(Box::new(io::Error::new( - io::ErrorKind::InvalidData, - "reader stride is outside supported range", - ))); - }; - - let mut source = SyntheticStreamingReader::new(TOTAL_BYTES, reader_stride); - let mut sink = StreamingWriteSink::new(WRITER_BUDGET_BYTES); - - let observed = copy_exact( - &mut source, - &mut sink, - ExactTransfer::new( - CatalogRestartArtifact::Head, - CatalogRestartPhase::ReadCatalog, - TOTAL_BYTES, - ), + fn read_exact_returns_exact_bytes() -> Result<(), Box> { + let source = Cursor::new(b"abcdefg".to_vec()); + + let encoded = read_exact( + source, + CatalogRestartArtifact::Head, + CatalogRestartPhase::ReadCatalog, + 7, )?; - assert_eq!(observed, TOTAL_BYTES); - assert_eq!(sink.observed_bytes(), TOTAL_BYTES); - assert!(sink.total_chunks() > 0); - assert!(sink.max_chunk() <= WRITER_BUDGET_BYTES); - assert!(sink.max_chunk() <= CATALOG_RESTART_READ_BUFFER_LENGTH); - assert!(sink.max_chunk() >= READER_STRIDE_BYTES); - assert!(size_of::() < 64); - Ok(()) - } - - #[test] - fn copy_exact_to_chunks_streams() -> Result<(), Box> { - let mut source = Cursor::new(vec![b'a', b'b', b'c', b'd', b'e', b'f', b'g']); - let mut observed = Vec::>::new(); - - let _observed = copy_exact_to_chunks( - &mut source, - ExactTransfer::new( - CatalogRestartArtifact::Head, - CatalogRestartPhase::ReadCatalog, - 7, - ), - |chunk| { - observed.push(chunk.to_vec()); - Ok(()) - }, - )?; - - assert_eq!(observed.concat(), b"abcdefg"); - Ok(()) - } - - #[test] - fn copy_exact_to_chunks_rejects_short_artifacts() -> Result<(), Box> { - let mut source = Cursor::new(vec![b'a', b'b']); - let mut seen = 0_u8; - - let result = copy_exact_to_chunks( - &mut source, - ExactTransfer::new( - CatalogRestartArtifact::Head, - CatalogRestartPhase::ReadCatalog, - 4, - ), - |_chunk| { - seen = match seen.checked_add(1) { - Some(total) => total, - None => { - return Err(CatalogRestartError::LengthArithmetic { - artifact: CatalogRestartArtifact::Head, - expected: 4, - }); - } - }; - Ok(()) - }, - ); - - let Err(error) = result else { - return Err(Box::new(io::Error::new( - io::ErrorKind::InvalidData, - "short artifact should have been rejected", - ))); - }; - assert_eq!(seen, 1); - assert!(matches!( - error, - CatalogRestartError::Io { - phase: CatalogRestartPhase::ReadCatalog, - ref source, - } if source.kind() == ErrorKind::UnexpectedEof - )); - Ok(()) - } - - #[test] - fn copy_exact_to_chunks_rejects_trailing_bytes() -> Result<(), Box> { - let mut source = Cursor::new(vec![b'a', b'b', b'c']); - - let result = copy_exact_to_chunks( - &mut source, - ExactTransfer::new( - CatalogRestartArtifact::Head, - CatalogRestartPhase::ReadCatalog, - 2, - ), - |_| Ok(()), - ); - let Err(error) = result else { - return Err(Box::new(io::Error::new( - io::ErrorKind::InvalidData, - "trailing bytes should have been rejected", - ))); - }; - let expected = 2_u64; - assert!(matches!( - error, - CatalogRestartError::Length { - artifact: CatalogRestartArtifact::Head, - minimum, - maximum, - observed: 3 - } if minimum == expected && maximum == expected - )); + assert_eq!(encoded, b"abcdefg"); Ok(()) } #[test] - fn copy_exact_rejects_short_artifacts() -> Result<(), Box> { - let mut source = Cursor::new(vec![b'a', b'b']); - let mut sink = StreamingWriteSink::new(16 * 1024); - - let result = copy_exact( - &mut source, - &mut sink, - ExactTransfer::new( - CatalogRestartArtifact::Head, - CatalogRestartPhase::ReadCatalog, - 4, - ), + fn read_exact_rejects_short_artifacts() -> Result<(), Box> { + let source = Cursor::new(vec![b'a', b'b']); + + let result = read_exact( + source, + CatalogRestartArtifact::Head, + CatalogRestartPhase::ReadCatalog, + 4, ); let Err(error) = result else { @@ -435,7 +138,6 @@ mod tests { "short artifact should have been rejected", ))); }; - assert_eq!(sink.observed_bytes(), 2); assert!(matches!( error, CatalogRestartError::Io { @@ -447,18 +149,14 @@ mod tests { } #[test] - fn copy_exact_rejects_trailing_bytes() -> Result<(), Box> { - let mut source = Cursor::new(vec![b'a', b'b', b'c']); - let mut sink = StreamingWriteSink::new(16 * 1024); - - let result = copy_exact( - &mut source, - &mut sink, - ExactTransfer::new( - CatalogRestartArtifact::Head, - CatalogRestartPhase::ReadCatalog, - 2, - ), + fn read_exact_rejects_trailing_bytes() -> Result<(), Box> { + let source = Cursor::new(vec![b'a', b'b', b'c']); + + let result = read_exact( + source, + CatalogRestartArtifact::Head, + CatalogRestartPhase::ReadCatalog, + 2, ); let Err(error) = result else { @@ -468,7 +166,6 @@ mod tests { ))); }; let expected = 2_u64; - assert_eq!(sink.observed_bytes(), 2); assert!(matches!( error, CatalogRestartError::Length { diff --git a/src/adapters/catalog_restart_io_test_doubles.rs b/src/adapters/catalog_restart_io_test_doubles.rs deleted file mode 100644 index ebe5026..0000000 --- a/src/adapters/catalog_restart_io_test_doubles.rs +++ /dev/null @@ -1,180 +0,0 @@ -//! This module owns bounded streaming doubles for catalog restart I/O laws. - -use std::io::{self, Read, Write}; - -use super::{CatalogRestartArtifact, CatalogRestartError}; - -pub(super) struct SyntheticStreamingReader { - remaining: u64, - emit_stride: u64, -} - -impl SyntheticStreamingReader { - pub(super) fn new(total: u64, emit_stride: u64) -> Self { - Self { - remaining: total, - emit_stride, - } - } -} - -impl Read for SyntheticStreamingReader { - fn read(&mut self, sink: &mut [u8]) -> io::Result { - if self.remaining == 0 { - return Ok(0); - } - - let Ok(sink_capacity) = u64::try_from(sink.len()) else { - return Err(io::Error::new( - io::ErrorKind::InvalidInput, - "sink capacity exceeds supported range", - )); - }; - let emitted: usize = match self - .emit_stride - .min(self.remaining) - .min(sink_capacity) - .try_into() - { - Ok(size) => size, - Err(_) => { - return Err(io::Error::new( - io::ErrorKind::InvalidInput, - "requested read size exceeds supported range", - )); - } - }; - - let read_window = sink - .get_mut(..emitted) - .ok_or_else(|| io::Error::new(io::ErrorKind::InvalidInput, "read window overflow"))?; - read_window.fill(0x5a); - let emitted_u64 = u64::try_from(emitted) - .map_err(|_| io::Error::new(io::ErrorKind::InvalidInput, "emit size overflow"))?; - self.remaining = self - .remaining - .checked_sub(emitted_u64) - .ok_or_else(|| io::Error::new(io::ErrorKind::InvalidInput, "emit underflow"))?; - Ok(emitted) - } -} - -pub(super) struct StreamingCallbackBudget { - observed_bytes: u64, - total_chunks: u64, - max_chunk: usize, - callback_limit: usize, - expected_total: u64, -} - -impl StreamingCallbackBudget { - pub(super) fn new(expected_total: u64, callback_limit: usize) -> Self { - Self { - observed_bytes: 0, - total_chunks: 0, - max_chunk: 0, - callback_limit, - expected_total, - } - } - - pub(super) fn consume(&mut self, chunk: &[u8]) -> Result<(), CatalogRestartError> { - self.total_chunks = - self.total_chunks - .checked_add(1) - .ok_or(CatalogRestartError::LengthArithmetic { - artifact: CatalogRestartArtifact::Head, - expected: self.expected_total, - })?; - - self.max_chunk = self.max_chunk.max(chunk.len()); - - self.observed_bytes = self - .observed_bytes - .checked_add(u64::try_from(chunk.len()).map_err(|_source| { - CatalogRestartError::LengthArithmetic { - artifact: CatalogRestartArtifact::Head, - expected: self.expected_total, - } - })?) - .ok_or(CatalogRestartError::LengthArithmetic { - artifact: CatalogRestartArtifact::Head, - expected: self.expected_total, - })?; - - Ok(()) - } - - pub(super) fn observed_bytes(&self) -> u64 { - self.observed_bytes - } - - pub(super) fn max_chunk(&self) -> usize { - self.max_chunk - } - - pub(super) fn callback_limit(&self) -> usize { - self.callback_limit - } - - pub(super) fn total_chunks(&self) -> u64 { - self.total_chunks - } -} - -pub(super) struct StreamingWriteSink { - observed_bytes: u64, - observed_chunks: u64, - max_chunk: usize, - writer_memory_limit: usize, -} - -impl StreamingWriteSink { - pub(super) fn new(writer_memory_limit: usize) -> Self { - Self { - observed_bytes: 0, - observed_chunks: 0, - max_chunk: 0, - writer_memory_limit, - } - } - - pub(super) fn observed_bytes(&self) -> u64 { - self.observed_bytes - } - - pub(super) fn total_chunks(&self) -> u64 { - self.observed_chunks - } - - pub(super) fn max_chunk(&self) -> usize { - self.max_chunk - } -} - -impl Write for StreamingWriteSink { - fn write(&mut self, bytes: &[u8]) -> io::Result { - self.observed_chunks = self - .observed_chunks - .checked_add(1) - .ok_or_else(|| io::Error::new(io::ErrorKind::InvalidInput, "chunk count overflow"))?; - let observed = u64::try_from(bytes.len()) - .map_err(|_| io::Error::new(io::ErrorKind::InvalidInput, "chunk length overflow"))?; - self.observed_bytes = self - .observed_bytes - .checked_add(observed) - .ok_or_else(|| io::Error::new(io::ErrorKind::InvalidInput, "write count overflow"))?; - self.max_chunk = self.max_chunk.max(bytes.len()); - if bytes.len() > self.writer_memory_limit { - return Err(io::Error::new( - io::ErrorKind::InvalidInput, - "sink memory budget exceeded", - )); - } - Ok(bytes.len()) - } - - fn flush(&mut self) -> io::Result<()> { - Ok(()) - } -} diff --git a/src/adapters/mod.rs b/src/adapters/mod.rs index cee5da7..b0ba60b 100644 --- a/src/adapters/mod.rs +++ b/src/adapters/mod.rs @@ -50,8 +50,6 @@ mod catalog_restart_artifact; mod catalog_restart_byte_limit; mod catalog_restart_error; mod catalog_restart_io; -#[cfg(test)] -mod catalog_restart_io_test_doubles; mod catalog_restart_loader; mod catalog_restart_phase; mod catalog_restart_policy; From b292c3ce742d3555f3c23439ad340f4d62e4f655 Mon Sep 17 00:00:00 2001 From: James Ross Date: Mon, 7 Sep 2026 10:12:15 -0700 Subject: [PATCH 070/111] Refactor: split the adapters root into an export surface and a recovery facade MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit src/adapters/mod.rs stood at 498 lines against the 500-line hard ceiling that source-structure-check enforces; the next adapter module would have failed the build. Two mechanical moves bring it to 238 with no public path changed: 1. The 174 re-export lines move to adapters/exports.rs as `pub use super::…;` with the root re-exporting `exports::*`. A child module may name the parent's private siblings, and unreachable_pub follows glob re-exports, so no visibility changes. 2. The 82 recovery_* modules move under adapters/recovery/ behind a facade (adapters/recovery.rs) that declares them, re-exports their 77 public items, and privately imports the 38 adapter-level names the cluster borrows through `super::`, so no moved file needed an import rewrite. Every `pub(super)` inside the moved files becomes `pub(in crate::adapters)`, which is exactly what it meant before the move. The three cross-cluster consumers (staged_segment.rs, and the two store_migration inventory readers using recovery_pool_name) are repointed. A Graft structural diff over the 89 touched paths shows zero added or removed symbols and one rustfmt-only signature rewrap. tests/adapters_layout_contract.rs pins the root under 300 lines and restricted to module declarations, re-exports, and attributes. During this slice an inserted `mod recovery;` briefly captured a preceding `#[cfg(feature = "repository-tasks")]` attribute, which all-features gates could not see; the default-features build caught it. Every gate in this pass now runs under both feature sets. Self-review finding R1 (P1). Refs #78 --- CHANGELOG.md | 4 + src/adapters/exports.rs | 105 +++++++ src/adapters/mod.rs | 264 +----------------- src/adapters/recovery.rs | 178 ++++++++++++ .../{ => recovery}/recovery_catalog_stage.rs | 0 .../recovery_catalog_stage_error.rs | 0 .../{ => recovery}/recovery_entry_name.rs | 0 .../{ => recovery}/recovery_entry_role.rs | 2 +- .../recovery_fixed_field_prefix.rs | 2 +- .../{ => recovery}/recovery_inventory.rs | 4 +- .../recovery_inventory_error.rs | 2 +- .../recovery_inventory_limit.rs | 0 .../recovery_inventory_operation.rs | 0 .../recovery_inventory_storage.rs | 0 .../recovery_name_classification.rs | 0 .../recovery_name_classification_error.rs | 0 .../{ => recovery}/recovery_name_manifest.rs | 4 +- .../{ => recovery}/recovery_namespace.rs | 0 .../recovery_next_head_finalization_error.rs | 0 ...ecovery_next_head_finalization_executor.rs | 0 ...recovery_next_head_finalization_outcome.rs | 0 ...overy_next_head_finalization_plan_error.rs | 0 ...recovery_next_head_finalization_planner.rs | 0 ...covery_next_head_finalization_readiness.rs | 0 ...recovery_next_head_finalization_receipt.rs | 2 +- ...recovery_next_head_finalization_request.rs | 2 +- ...recovery_next_head_finalization_storage.rs | 0 ...ry_next_head_finalization_storage_error.rs | 0 .../recovery_next_head_finalization_target.rs | 2 +- .../recovery_next_head_stage.rs | 0 .../recovery_next_head_stage_error.rs | 0 .../{ => recovery}/recovery_pool_name.rs | 6 +- .../recovery_pool_name_error.rs | 0 .../recovery_publication_fixed_framing.rs | 4 +- .../recovery_publication_stage_classifier.rs | 0 .../{ => recovery}/recovery_required_entry.rs | 0 .../recovery_segment_classifier.rs | 0 .../recovery_segment_fixed_framing.rs | 4 +- .../recovery_segment_resume_error.rs | 0 .../recovery_segment_resume_executor.rs | 0 .../recovery_segment_resume_plan_error.rs | 0 .../recovery_segment_resume_planner.rs | 0 .../recovery_segment_resume_request.rs | 2 +- .../recovery_segment_resume_state.rs | 14 +- .../recovery_segment_resume_storage.rs | 0 .../recovery_segment_resume_storage_error.rs | 0 .../{ => recovery}/recovery_segment_stage.rs | 2 +- .../recovery_segment_stage_error.rs | 0 .../recovery_segment_truncation.rs | 0 src/adapters/{ => recovery}/recovery_stage.rs | 2 +- .../recovery_stage_assessment.rs | 0 .../recovery_stage_assessment_error.rs | 0 .../{ => recovery}/recovery_stage_assessor.rs | 0 .../recovery_stage_byte_admission.rs | 0 .../recovery_stage_byte_admission_error.rs | 0 .../recovery_stage_completion_error.rs | 0 .../recovery_stage_completion_executor.rs | 0 .../recovery_stage_completion_plan_error.rs | 0 .../recovery_stage_completion_planner.rs | 0 .../recovery_stage_completion_pool.rs | 0 .../recovery_stage_completion_receipt.rs | 2 +- .../recovery_stage_completion_request.rs | 2 +- .../recovery_stage_completion_storage.rs | 0 ...recovery_stage_completion_storage_error.rs | 2 +- .../recovery_stage_completion_target.rs | 0 .../recovery_stage_discard_error.rs | 0 .../recovery_stage_discard_executor.rs | 0 .../recovery_stage_discard_outcome.rs | 0 .../recovery_stage_discard_plan_error.rs | 0 .../recovery_stage_discard_planner.rs | 0 .../recovery_stage_discard_reason.rs | 0 .../recovery_stage_discard_receipt.rs | 2 +- .../recovery_stage_discard_request.rs | 2 +- .../recovery_stage_discard_storage.rs | 0 .../recovery_stage_discard_storage_error.rs | 0 .../{ => recovery}/recovery_stage_evidence.rs | 2 +- .../recovery_stage_fingerprint.rs | 2 +- .../recovery_stage_fingerprint_algorithm.rs | 0 .../recovery_stage_fingerprint_error.rs | 0 .../recovery_stage_fingerprinter.rs | 0 .../{ => recovery}/recovery_stage_length.rs | 2 +- .../{ => recovery}/recovery_stage_metadata.rs | 0 .../recovery_stage_metadata_error.rs | 0 .../{ => recovery}/recovery_stage_parent.rs | 0 .../recovery_stage_pool_outcome.rs | 0 .../recovery_stage_synchronization_outcome.rs | 0 src/adapters/staged_segment.rs | 2 +- .../filesystem_inventory_catalogs.rs | 3 +- .../filesystem_inventory_segments.rs | 3 +- tests/adapters_layout_contract.rs | 37 +++ 90 files changed, 368 insertions(+), 298 deletions(-) create mode 100644 src/adapters/exports.rs create mode 100644 src/adapters/recovery.rs rename src/adapters/{ => recovery}/recovery_catalog_stage.rs (100%) rename src/adapters/{ => recovery}/recovery_catalog_stage_error.rs (100%) rename src/adapters/{ => recovery}/recovery_entry_name.rs (100%) rename src/adapters/{ => recovery}/recovery_entry_role.rs (95%) rename src/adapters/{ => recovery}/recovery_fixed_field_prefix.rs (86%) rename src/adapters/{ => recovery}/recovery_inventory.rs (96%) rename src/adapters/{ => recovery}/recovery_inventory_error.rs (98%) rename src/adapters/{ => recovery}/recovery_inventory_limit.rs (100%) rename src/adapters/{ => recovery}/recovery_inventory_operation.rs (100%) rename src/adapters/{ => recovery}/recovery_inventory_storage.rs (100%) rename src/adapters/{ => recovery}/recovery_name_classification.rs (100%) rename src/adapters/{ => recovery}/recovery_name_classification_error.rs (100%) rename src/adapters/{ => recovery}/recovery_name_manifest.rs (91%) rename src/adapters/{ => recovery}/recovery_namespace.rs (100%) rename src/adapters/{ => recovery}/recovery_next_head_finalization_error.rs (100%) rename src/adapters/{ => recovery}/recovery_next_head_finalization_executor.rs (100%) rename src/adapters/{ => recovery}/recovery_next_head_finalization_outcome.rs (100%) rename src/adapters/{ => recovery}/recovery_next_head_finalization_plan_error.rs (100%) rename src/adapters/{ => recovery}/recovery_next_head_finalization_planner.rs (100%) rename src/adapters/{ => recovery}/recovery_next_head_finalization_readiness.rs (100%) rename src/adapters/{ => recovery}/recovery_next_head_finalization_receipt.rs (96%) rename src/adapters/{ => recovery}/recovery_next_head_finalization_request.rs (96%) rename src/adapters/{ => recovery}/recovery_next_head_finalization_storage.rs (100%) rename src/adapters/{ => recovery}/recovery_next_head_finalization_storage_error.rs (100%) rename src/adapters/{ => recovery}/recovery_next_head_finalization_target.rs (97%) rename src/adapters/{ => recovery}/recovery_next_head_stage.rs (100%) rename src/adapters/{ => recovery}/recovery_next_head_stage_error.rs (100%) rename src/adapters/{ => recovery}/recovery_pool_name.rs (95%) rename src/adapters/{ => recovery}/recovery_pool_name_error.rs (100%) rename src/adapters/{ => recovery}/recovery_publication_fixed_framing.rs (96%) rename src/adapters/{ => recovery}/recovery_publication_stage_classifier.rs (100%) rename src/adapters/{ => recovery}/recovery_required_entry.rs (100%) rename src/adapters/{ => recovery}/recovery_segment_classifier.rs (100%) rename src/adapters/{ => recovery}/recovery_segment_fixed_framing.rs (95%) rename src/adapters/{ => recovery}/recovery_segment_resume_error.rs (100%) rename src/adapters/{ => recovery}/recovery_segment_resume_executor.rs (100%) rename src/adapters/{ => recovery}/recovery_segment_resume_plan_error.rs (100%) rename src/adapters/{ => recovery}/recovery_segment_resume_planner.rs (100%) rename src/adapters/{ => recovery}/recovery_segment_resume_request.rs (97%) rename src/adapters/{ => recovery}/recovery_segment_resume_state.rs (83%) rename src/adapters/{ => recovery}/recovery_segment_resume_storage.rs (100%) rename src/adapters/{ => recovery}/recovery_segment_resume_storage_error.rs (100%) rename src/adapters/{ => recovery}/recovery_segment_stage.rs (92%) rename src/adapters/{ => recovery}/recovery_segment_stage_error.rs (100%) rename src/adapters/{ => recovery}/recovery_segment_truncation.rs (100%) rename src/adapters/{ => recovery}/recovery_stage.rs (95%) rename src/adapters/{ => recovery}/recovery_stage_assessment.rs (100%) rename src/adapters/{ => recovery}/recovery_stage_assessment_error.rs (100%) rename src/adapters/{ => recovery}/recovery_stage_assessor.rs (100%) rename src/adapters/{ => recovery}/recovery_stage_byte_admission.rs (100%) rename src/adapters/{ => recovery}/recovery_stage_byte_admission_error.rs (100%) rename src/adapters/{ => recovery}/recovery_stage_completion_error.rs (100%) rename src/adapters/{ => recovery}/recovery_stage_completion_executor.rs (100%) rename src/adapters/{ => recovery}/recovery_stage_completion_plan_error.rs (100%) rename src/adapters/{ => recovery}/recovery_stage_completion_planner.rs (100%) rename src/adapters/{ => recovery}/recovery_stage_completion_pool.rs (100%) rename src/adapters/{ => recovery}/recovery_stage_completion_receipt.rs (98%) rename src/adapters/{ => recovery}/recovery_stage_completion_request.rs (96%) rename src/adapters/{ => recovery}/recovery_stage_completion_storage.rs (100%) rename src/adapters/{ => recovery}/recovery_stage_completion_storage_error.rs (96%) rename src/adapters/{ => recovery}/recovery_stage_completion_target.rs (100%) rename src/adapters/{ => recovery}/recovery_stage_discard_error.rs (100%) rename src/adapters/{ => recovery}/recovery_stage_discard_executor.rs (100%) rename src/adapters/{ => recovery}/recovery_stage_discard_outcome.rs (100%) rename src/adapters/{ => recovery}/recovery_stage_discard_plan_error.rs (100%) rename src/adapters/{ => recovery}/recovery_stage_discard_planner.rs (100%) rename src/adapters/{ => recovery}/recovery_stage_discard_reason.rs (100%) rename src/adapters/{ => recovery}/recovery_stage_discard_receipt.rs (96%) rename src/adapters/{ => recovery}/recovery_stage_discard_request.rs (96%) rename src/adapters/{ => recovery}/recovery_stage_discard_storage.rs (100%) rename src/adapters/{ => recovery}/recovery_stage_discard_storage_error.rs (100%) rename src/adapters/{ => recovery}/recovery_stage_evidence.rs (96%) rename src/adapters/{ => recovery}/recovery_stage_fingerprint.rs (89%) rename src/adapters/{ => recovery}/recovery_stage_fingerprint_algorithm.rs (100%) rename src/adapters/{ => recovery}/recovery_stage_fingerprint_error.rs (100%) rename src/adapters/{ => recovery}/recovery_stage_fingerprinter.rs (100%) rename src/adapters/{ => recovery}/recovery_stage_length.rs (84%) rename src/adapters/{ => recovery}/recovery_stage_metadata.rs (100%) rename src/adapters/{ => recovery}/recovery_stage_metadata_error.rs (100%) rename src/adapters/{ => recovery}/recovery_stage_parent.rs (100%) rename src/adapters/{ => recovery}/recovery_stage_pool_outcome.rs (100%) rename src/adapters/{ => recovery}/recovery_stage_synchronization_outcome.rs (100%) create mode 100644 tests/adapters_layout_contract.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index 7f29fb4..32ba343 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,10 @@ after its public API and format compatibility policies are established. ### Changed +- The adapters module root now declares modules only: its public re-export + surface lives in `adapters/exports.rs` and the recovery-stage adapters live + under `adapters/recovery/` behind one facade. No public path changed; the + root fell from 498 lines to 238 against the 500-line hard ceiling. - Restart artifact reads use one exact read into a pre-reserved buffer followed by trailing-byte rejection; the interim chunked transfer layer, which pumped every artifact through an 8 KiB buffer without lowering peak memory, is diff --git a/src/adapters/exports.rs b/src/adapters/exports.rs new file mode 100644 index 0000000..4b18787 --- /dev/null +++ b/src/adapters/exports.rs @@ -0,0 +1,105 @@ +//! This module owns the public re-export surface of the adapters layer. + +pub use super::admitted_catalog::AdmittedCatalog; +pub use super::admitted_recovery_stage_bytes::AdmittedRecoveryStageBytes; +pub use super::admitted_segment::AdmittedSegment; +pub use super::admitted_segment_record::AdmittedSegmentRecord; +pub use super::blob_id_binary_error::BlobIdBinaryParseError; +pub use super::blob_id_text_error::BlobIdTextParseError; +pub use super::canonical_catalog::CanonicalCatalog; +pub use super::canonical_publication_head::CanonicalPublicationHead; +pub use super::catalog_admission_error::CatalogAdmissionError; +pub use super::catalog_allocation_phase::CatalogAllocationPhase; +pub use super::catalog_decode_error::CatalogDecodeError; +pub use super::catalog_encode_error::CatalogEncodeError; +pub use super::catalog_entry_decode_error::CatalogEntryDecodeError; +pub use super::catalog_publication::publish_catalog_generation; +pub use super::catalog_publication_error::CatalogPublicationError; +pub use super::catalog_publication_expectation::CatalogPublicationExpectation; +pub use super::catalog_publication_outcome::CatalogPublicationOutcome; +pub use super::catalog_publication_phase::CatalogPublicationPhase; +pub use super::catalog_publication_readiness::CatalogPublicationReadiness; +pub use super::catalog_publication_receipt::CatalogPublicationReceipt; +pub use super::catalog_publication_storage::CatalogPublicationStorage; +pub use super::catalog_restart_artifact::CatalogRestartArtifact; +pub use super::catalog_restart_byte_limit::{ + CatalogRestartByteLimit, CatalogRestartByteLimitError, +}; +pub use super::catalog_restart_error::CatalogRestartError; +pub use super::catalog_restart_phase::CatalogRestartPhase; +pub use super::catalog_restart_policy::CatalogRestartPolicy; +pub use super::catalog_snapshot::CatalogSnapshot; +pub use super::catalog_snapshot_error::CatalogSnapshotError; +pub use super::catalog_successor::CatalogSuccessor; +pub use super::catalog_transition_error::CatalogTransitionError; +pub use super::checksummed_catalog::ChecksummedCatalog; +pub use super::checksummed_publication_head::ChecksummedPublicationHead; +pub use super::checksummed_segment_record::ChecksummedSegmentRecord; +pub use super::closed_segment::ClosedSegment; +pub use super::filesystem_catalog_publication_error::FilesystemCatalogPublicationError; +pub use super::filesystem_catalog_publisher::FilesystemCatalogPublisher; +pub use super::filesystem_catalog_snapshot::FilesystemCatalogSnapshot; +pub use super::filesystem_platform_admission::FilesystemPlatformAdmission; +pub use super::filesystem_platform_admission_error::FilesystemPlatformAdmissionError; +pub use super::filesystem_recovery_inventory_reader::FilesystemRecoveryInventoryReader; +pub use super::filesystem_recovery_next_head_finalization_open_error::FilesystemRecoveryNextHeadFinalizationOpenError; +pub use super::filesystem_recovery_next_head_finalizer::FilesystemRecoveryNextHeadFinalizer; +pub use super::filesystem_recovery_segment_resume_open_error::FilesystemRecoverySegmentResumeOpenError; +pub use super::filesystem_recovery_segment_resumer::FilesystemRecoverySegmentResumer; +pub use super::filesystem_recovery_segment_stage::FilesystemRecoverySegmentStage; +pub use super::filesystem_recovery_stage_completer::FilesystemRecoveryStageCompleter; +pub use super::filesystem_recovery_stage_completion_open_error::FilesystemRecoveryStageCompletionOpenError; +pub use super::filesystem_recovery_stage_discard_open_error::FilesystemRecoveryStageDiscardOpenError; +pub use super::filesystem_recovery_stage_discarder::FilesystemRecoveryStageDiscarder; +pub use super::filesystem_recovery_stage_error::{ + FilesystemRecoveryStageError, RecoveryStageNamespacePhase, +}; +pub use super::filesystem_segment_stage::FilesystemSegmentStage; +pub use super::filesystem_version_two_admission::FilesystemVersionTwoAdmission; +pub use super::filesystem_writer_lock::FilesystemWriterLock; +pub use super::layout_decode_error::LayoutDecodeError; +pub use super::layout_decode_policy::LayoutDecodePolicy; +pub use super::layout_encode_error::LayoutEncodeError; +pub use super::layout_id_binary_error::LayoutIdBinaryParseError; +pub use super::layout_id_text_error::LayoutIdTextParseError; +pub use super::layout_record::CanonicalLayoutRecord; +pub use super::opened_reusable_segment::OpenedReusableSegment; +pub use super::publication_head_decode_error::PublicationHeadDecodeError; +pub use super::recovery::*; +#[cfg(feature = "repository-tasks")] +pub use super::repository_initialization_storage::RepositoryInitializationStorage; +pub use super::retention::*; +pub use super::sealed_segment::SealedSegment; +pub use super::segment_digest::SegmentDigest; +pub use super::segment_header::SegmentHeader; +pub use super::segment_header_error::SegmentHeaderError; +pub use super::segment_publication::SegmentPublication; +pub use super::segment_publication_error::SegmentPublicationError; +pub use super::segment_read_error::SegmentReadError; +pub use super::segment_read_policy::SegmentReadPolicy; +pub use super::segment_record_admission_error::SegmentRecordAdmissionError; +pub use super::segment_record_checksum::SegmentRecordChecksum; +pub use super::segment_record_decode_error::SegmentRecordDecodeError; +pub use super::segment_record_header::SegmentRecordHeader; +pub use super::segment_record_header_error::SegmentRecordHeaderError; +pub use super::segment_record_identity::SegmentRecordIdentity; +pub use super::segment_record_length::SegmentRecordLength; +pub use super::segment_record_limit::{SegmentRecordLimit, SegmentRecordLimitError}; +pub use super::segment_record_payload_length::SegmentRecordPayloadLength; +pub use super::segment_records::SegmentRecords; +pub use super::segment_seal::SegmentSeal; +pub use super::segment_seal_error::SegmentSealError; +pub use super::segment_stage::SegmentStage; +pub use super::segment_stage_create_error::SegmentStageCreateError; +pub use super::segment_write_error::SegmentWriteError; +pub use super::segment_write_phase::{SegmentDurabilityPhase, SegmentWritePhase}; +pub use super::staged_segment::StagedSegment; +pub use super::storage_profile_id_text_error::StorageProfileIdParseError; +pub use super::store_initialization::initialize_store; +pub use super::store_initialization_error::StoreInitializationError; +pub use super::store_initialization_phase::StoreInitializationPhase; +pub use super::store_initialization_receipt::StoreInitializationReceipt; +pub use super::store_initialization_storage::StoreInitializationStorage; +pub use super::store_migration::*; +pub use super::writer_lock_acquire_error::WriterLockAcquireError; +pub use super::writer_lock_acquire_phase::WriterLockAcquirePhase; diff --git a/src/adapters/mod.rs b/src/adapters/mod.rs index b0ba60b..3a6333f 100644 --- a/src/adapters/mod.rs +++ b/src/adapters/mod.rs @@ -65,6 +65,7 @@ mod checksummed_publication_head; mod checksummed_segment_record; mod closed_segment; mod decoded_catalog_entry; +mod exports; mod filesystem_catalog_artifact; mod filesystem_catalog_catalog; mod filesystem_catalog_current; @@ -153,88 +154,7 @@ mod publication_head_decode_error; mod publication_head_decode_error_display; mod publication_head_decoder; mod publication_head_encoder; -mod recovery_catalog_stage; -mod recovery_catalog_stage_error; -mod recovery_entry_name; -mod recovery_entry_role; -mod recovery_fixed_field_prefix; -mod recovery_inventory; -mod recovery_inventory_error; -mod recovery_inventory_limit; -mod recovery_inventory_operation; -mod recovery_inventory_storage; -mod recovery_name_classification; -mod recovery_name_classification_error; -mod recovery_name_manifest; -mod recovery_namespace; -mod recovery_next_head_finalization_error; -mod recovery_next_head_finalization_executor; -mod recovery_next_head_finalization_outcome; -mod recovery_next_head_finalization_plan_error; -mod recovery_next_head_finalization_planner; -mod recovery_next_head_finalization_readiness; -mod recovery_next_head_finalization_receipt; -mod recovery_next_head_finalization_request; -mod recovery_next_head_finalization_storage; -mod recovery_next_head_finalization_storage_error; -mod recovery_next_head_finalization_target; -mod recovery_next_head_stage; -mod recovery_next_head_stage_error; -mod recovery_pool_name; -mod recovery_pool_name_error; -mod recovery_publication_fixed_framing; -mod recovery_publication_stage_classifier; -mod recovery_required_entry; -mod recovery_segment_classifier; -mod recovery_segment_fixed_framing; -mod recovery_segment_resume_error; -mod recovery_segment_resume_executor; -mod recovery_segment_resume_plan_error; -mod recovery_segment_resume_planner; -mod recovery_segment_resume_request; -mod recovery_segment_resume_state; -mod recovery_segment_resume_storage; -mod recovery_segment_resume_storage_error; -mod recovery_segment_stage; -mod recovery_segment_stage_error; -mod recovery_segment_truncation; -mod recovery_stage; -mod recovery_stage_assessment; -mod recovery_stage_assessment_error; -mod recovery_stage_assessor; -mod recovery_stage_byte_admission; -mod recovery_stage_byte_admission_error; -mod recovery_stage_completion_error; -mod recovery_stage_completion_executor; -mod recovery_stage_completion_plan_error; -mod recovery_stage_completion_planner; -mod recovery_stage_completion_pool; -mod recovery_stage_completion_receipt; -mod recovery_stage_completion_request; -mod recovery_stage_completion_storage; -mod recovery_stage_completion_storage_error; -mod recovery_stage_completion_target; -mod recovery_stage_discard_error; -mod recovery_stage_discard_executor; -mod recovery_stage_discard_outcome; -mod recovery_stage_discard_plan_error; -mod recovery_stage_discard_planner; -mod recovery_stage_discard_reason; -mod recovery_stage_discard_receipt; -mod recovery_stage_discard_request; -mod recovery_stage_discard_storage; -mod recovery_stage_discard_storage_error; -mod recovery_stage_evidence; -mod recovery_stage_fingerprint; -mod recovery_stage_fingerprint_algorithm; -mod recovery_stage_fingerprint_error; -mod recovery_stage_fingerprinter; -mod recovery_stage_length; -mod recovery_stage_metadata; -mod recovery_stage_metadata_error; -mod recovery_stage_parent; -mod recovery_stage_pool_outcome; -mod recovery_stage_synchronization_outcome; +mod recovery; #[cfg(feature = "repository-tasks")] mod repository_initialization_storage; mod retention; @@ -310,185 +230,7 @@ mod test_support; mod writer_lock_acquire_error; mod writer_lock_acquire_phase; -pub use admitted_catalog::AdmittedCatalog; -pub use admitted_recovery_stage_bytes::AdmittedRecoveryStageBytes; -pub use admitted_segment::AdmittedSegment; -pub use admitted_segment_record::AdmittedSegmentRecord; -pub use blob_id_binary_error::BlobIdBinaryParseError; -pub use blob_id_text_error::BlobIdTextParseError; -pub use canonical_catalog::CanonicalCatalog; -pub use canonical_publication_head::CanonicalPublicationHead; -pub use catalog_admission_error::CatalogAdmissionError; -pub use catalog_allocation_phase::CatalogAllocationPhase; -pub use catalog_decode_error::CatalogDecodeError; -pub use catalog_encode_error::CatalogEncodeError; -pub use catalog_entry_decode_error::CatalogEntryDecodeError; -pub use catalog_publication::publish_catalog_generation; -pub use catalog_publication_error::CatalogPublicationError; -pub use catalog_publication_expectation::CatalogPublicationExpectation; -pub use catalog_publication_outcome::CatalogPublicationOutcome; -pub use catalog_publication_phase::CatalogPublicationPhase; -pub use catalog_publication_readiness::CatalogPublicationReadiness; -pub use catalog_publication_receipt::CatalogPublicationReceipt; -pub use catalog_publication_storage::CatalogPublicationStorage; -pub use catalog_restart_artifact::CatalogRestartArtifact; -pub use catalog_restart_byte_limit::{CatalogRestartByteLimit, CatalogRestartByteLimitError}; -pub use catalog_restart_error::CatalogRestartError; -pub use catalog_restart_phase::CatalogRestartPhase; -pub use catalog_restart_policy::CatalogRestartPolicy; -pub use catalog_snapshot::CatalogSnapshot; -pub use catalog_snapshot_error::CatalogSnapshotError; -pub use catalog_successor::CatalogSuccessor; -pub use catalog_transition_error::CatalogTransitionError; -pub use checksummed_catalog::ChecksummedCatalog; -pub use checksummed_publication_head::ChecksummedPublicationHead; -pub use checksummed_segment_record::ChecksummedSegmentRecord; -pub use closed_segment::ClosedSegment; -pub use filesystem_catalog_publication_error::FilesystemCatalogPublicationError; -pub use filesystem_catalog_publisher::FilesystemCatalogPublisher; -pub use filesystem_catalog_snapshot::FilesystemCatalogSnapshot; -pub use filesystem_platform_admission::FilesystemPlatformAdmission; -pub use filesystem_platform_admission_error::FilesystemPlatformAdmissionError; -pub use filesystem_recovery_inventory_reader::FilesystemRecoveryInventoryReader; -pub use filesystem_recovery_next_head_finalization_open_error::FilesystemRecoveryNextHeadFinalizationOpenError; -pub use filesystem_recovery_next_head_finalizer::FilesystemRecoveryNextHeadFinalizer; -pub use filesystem_recovery_segment_resume_open_error::FilesystemRecoverySegmentResumeOpenError; -pub use filesystem_recovery_segment_resumer::FilesystemRecoverySegmentResumer; -pub use filesystem_recovery_segment_stage::FilesystemRecoverySegmentStage; -pub use filesystem_recovery_stage_completer::FilesystemRecoveryStageCompleter; -pub use filesystem_recovery_stage_completion_open_error::FilesystemRecoveryStageCompletionOpenError; -pub use filesystem_recovery_stage_discard_open_error::FilesystemRecoveryStageDiscardOpenError; -pub use filesystem_recovery_stage_discarder::FilesystemRecoveryStageDiscarder; -pub use filesystem_recovery_stage_error::{ - FilesystemRecoveryStageError, RecoveryStageNamespacePhase, -}; -pub use filesystem_segment_stage::FilesystemSegmentStage; -pub use filesystem_version_two_admission::FilesystemVersionTwoAdmission; -pub use filesystem_writer_lock::FilesystemWriterLock; -pub use layout_decode_error::LayoutDecodeError; -pub use layout_decode_policy::LayoutDecodePolicy; -pub use layout_encode_error::LayoutEncodeError; -pub use layout_id_binary_error::LayoutIdBinaryParseError; -pub use layout_id_text_error::LayoutIdTextParseError; -pub use layout_record::CanonicalLayoutRecord; -pub use opened_reusable_segment::OpenedReusableSegment; -pub use publication_head_decode_error::PublicationHeadDecodeError; -pub use recovery_catalog_stage::RecoveryCatalogStage; -pub use recovery_catalog_stage_error::RecoveryCatalogStageError; -pub use recovery_entry_name::{RecoveryEntryName, RecoveryEntryNameError}; -pub use recovery_entry_role::RecoveryEntryRole; -pub use recovery_inventory::{RecoveryInventory, RecoveryInventoryEntry, read_recovery_inventory}; -pub use recovery_inventory_error::RecoveryInventoryError; -pub use recovery_inventory_limit::{RecoveryInventoryLimit, RecoveryInventoryLimitError}; -pub use recovery_inventory_operation::RecoveryInventoryOperation; -pub use recovery_inventory_storage::RecoveryInventoryStorage; -pub use recovery_name_classification::classify_recovery_names; -pub use recovery_name_classification_error::RecoveryNameClassificationError; -pub use recovery_name_manifest::{RecoveryNameManifest, RecoveryNamedEntry}; -pub use recovery_namespace::RecoveryNamespace; -pub use recovery_next_head_finalization_error::RecoveryNextHeadFinalizationError; -pub use recovery_next_head_finalization_executor::execute_recovery_next_head_finalization; -pub use recovery_next_head_finalization_outcome::RecoveryNextHeadFinalizationOutcome; -pub use recovery_next_head_finalization_plan_error::RecoveryNextHeadFinalizationPlanError; -pub use recovery_next_head_finalization_planner::plan_recovery_next_head_finalization; -pub use recovery_next_head_finalization_readiness::RecoveryNextHeadFinalizationReadiness; -pub use recovery_next_head_finalization_receipt::RecoveryNextHeadFinalizationReceipt; -pub use recovery_next_head_finalization_request::RecoveryNextHeadFinalizationRequest; -pub use recovery_next_head_finalization_storage::RecoveryNextHeadFinalizationStorage; -pub use recovery_next_head_finalization_storage_error::RecoveryNextHeadFinalizationStorageError; -pub use recovery_next_head_finalization_target::RecoveryNextHeadFinalizationTarget; -pub use recovery_next_head_stage::RecoveryNextHeadStage; -pub use recovery_next_head_stage_error::RecoveryNextHeadStageError; -pub use recovery_pool_name_error::RecoveryPoolNameError; -pub use recovery_publication_stage_classifier::{ - classify_recovery_catalog_stage, classify_recovery_next_head_stage, -}; -pub use recovery_required_entry::RecoveryRequiredEntry; -pub use recovery_segment_classifier::classify_recovery_segment_stage; -pub use recovery_segment_resume_error::RecoverySegmentResumeError; -pub use recovery_segment_resume_executor::execute_recovery_segment_resume; -pub use recovery_segment_resume_plan_error::RecoverySegmentResumePlanError; -pub use recovery_segment_resume_planner::plan_recovery_segment_resume; -pub use recovery_segment_resume_request::RecoverySegmentResumeRequest; -pub use recovery_segment_resume_storage::RecoverySegmentResumeStorage; -pub use recovery_segment_resume_storage_error::RecoverySegmentResumeStorageError; -pub use recovery_segment_stage::{RecoverySegmentStage, ReusableRecoverySegment}; -pub use recovery_segment_stage_error::RecoverySegmentStageError; -pub use recovery_segment_truncation::RecoverySegmentTruncation; -pub use recovery_stage::RecoveryStage; -pub use recovery_stage_assessment::RecoveryStageAssessment; -pub use recovery_stage_assessment_error::RecoveryStageAssessmentError; -pub use recovery_stage_assessor::assess_recovery_stage; -pub use recovery_stage_byte_admission::admit_recovery_stage_bytes; -pub use recovery_stage_byte_admission_error::RecoveryStageByteAdmissionError; -pub use recovery_stage_completion_error::RecoveryStageCompletionError; -pub use recovery_stage_completion_executor::execute_recovery_stage_completion; -pub use recovery_stage_completion_plan_error::RecoveryStageCompletionPlanError; -pub use recovery_stage_completion_planner::plan_recovery_stage_completion; -pub use recovery_stage_completion_pool::RecoveryStageCompletionPool; -pub use recovery_stage_completion_receipt::RecoveryStageCompletionReceipt; -pub use recovery_stage_completion_request::RecoveryStageCompletionRequest; -pub use recovery_stage_completion_storage::RecoveryStageCompletionStorage; -pub use recovery_stage_completion_storage_error::RecoveryStageCompletionStorageError; -pub use recovery_stage_completion_target::RecoveryStageCompletionTarget; -pub use recovery_stage_discard_error::RecoveryStageDiscardError; -pub use recovery_stage_discard_executor::execute_recovery_stage_discard; -pub use recovery_stage_discard_outcome::RecoveryStageDiscardOutcome; -pub use recovery_stage_discard_plan_error::RecoveryStageDiscardPlanError; -pub use recovery_stage_discard_planner::plan_recovery_stage_discard; -pub use recovery_stage_discard_reason::RecoveryStageDiscardReason; -pub use recovery_stage_discard_receipt::RecoveryStageDiscardReceipt; -pub use recovery_stage_discard_request::RecoveryStageDiscardRequest; -pub use recovery_stage_discard_storage::RecoveryStageDiscardStorage; -pub use recovery_stage_discard_storage_error::RecoveryStageDiscardStorageError; -pub use recovery_stage_evidence::RecoveryStageEvidence; -pub use recovery_stage_fingerprint::RecoveryStageFingerprint; -pub use recovery_stage_fingerprint_algorithm::RecoveryStageFingerprintAlgorithm; -pub use recovery_stage_fingerprint_error::RecoveryStageFingerprintError; -pub use recovery_stage_fingerprinter::fingerprint_recovery_stage; -pub use recovery_stage_length::RecoveryStageLength; -pub use recovery_stage_metadata::RecoveryStageMetadata; -pub use recovery_stage_metadata_error::RecoveryStageMetadataError; -pub use recovery_stage_parent::RecoveryStageParent; -pub use recovery_stage_pool_outcome::RecoveryStagePoolOutcome; -pub use recovery_stage_synchronization_outcome::RecoveryStageSynchronizationOutcome; -#[cfg(feature = "repository-tasks")] -pub use repository_initialization_storage::RepositoryInitializationStorage; -pub use retention::*; -pub use sealed_segment::SealedSegment; -pub use segment_digest::SegmentDigest; -pub use segment_header::SegmentHeader; -pub use segment_header_error::SegmentHeaderError; -pub use segment_publication::SegmentPublication; -pub use segment_publication_error::SegmentPublicationError; -pub use segment_read_error::SegmentReadError; -pub use segment_read_policy::SegmentReadPolicy; -pub use segment_record_admission_error::SegmentRecordAdmissionError; -pub use segment_record_checksum::SegmentRecordChecksum; -pub use segment_record_decode_error::SegmentRecordDecodeError; -pub use segment_record_header::SegmentRecordHeader; -pub use segment_record_header_error::SegmentRecordHeaderError; -pub use segment_record_identity::SegmentRecordIdentity; -pub use segment_record_length::SegmentRecordLength; -pub use segment_record_limit::{SegmentRecordLimit, SegmentRecordLimitError}; -pub use segment_record_payload_length::SegmentRecordPayloadLength; -pub use segment_records::SegmentRecords; -pub use segment_seal::SegmentSeal; -pub use segment_seal_error::SegmentSealError; -pub use segment_stage::SegmentStage; -pub use segment_stage_create_error::SegmentStageCreateError; -pub use segment_write_error::SegmentWriteError; -pub use segment_write_phase::{SegmentDurabilityPhase, SegmentWritePhase}; -pub use staged_segment::StagedSegment; -pub use storage_profile_id_text_error::StorageProfileIdParseError; -pub use store_initialization::initialize_store; -pub use store_initialization_error::StoreInitializationError; -pub use store_initialization_phase::StoreInitializationPhase; -pub use store_initialization_receipt::StoreInitializationReceipt; -pub use store_initialization_storage::StoreInitializationStorage; -pub use store_migration::*; -pub use writer_lock_acquire_error::WriterLockAcquireError; -pub use writer_lock_acquire_phase::WriterLockAcquirePhase; +pub use exports::*; use catalog_encoding_entry::CatalogEncodingEntry; use catalog_entries::CatalogEntries; diff --git a/src/adapters/recovery.rs b/src/adapters/recovery.rs new file mode 100644 index 0000000..02f3550 --- /dev/null +++ b/src/adapters/recovery.rs @@ -0,0 +1,178 @@ +//! This module owns recovery-stage classification, planning, resumption, and +//! finalization adapters. Children borrow adapter-level items through `super`. + +mod recovery_catalog_stage; +mod recovery_catalog_stage_error; +mod recovery_entry_name; +mod recovery_entry_role; +mod recovery_fixed_field_prefix; +mod recovery_inventory; +mod recovery_inventory_error; +mod recovery_inventory_limit; +mod recovery_inventory_operation; +mod recovery_inventory_storage; +mod recovery_name_classification; +mod recovery_name_classification_error; +mod recovery_name_manifest; +mod recovery_namespace; +mod recovery_next_head_finalization_error; +mod recovery_next_head_finalization_executor; +mod recovery_next_head_finalization_outcome; +mod recovery_next_head_finalization_plan_error; +mod recovery_next_head_finalization_planner; +mod recovery_next_head_finalization_readiness; +mod recovery_next_head_finalization_receipt; +mod recovery_next_head_finalization_request; +mod recovery_next_head_finalization_storage; +mod recovery_next_head_finalization_storage_error; +mod recovery_next_head_finalization_target; +mod recovery_next_head_stage; +mod recovery_next_head_stage_error; +pub(in crate::adapters) mod recovery_pool_name; +mod recovery_pool_name_error; +mod recovery_publication_fixed_framing; +mod recovery_publication_stage_classifier; +mod recovery_required_entry; +mod recovery_segment_classifier; +mod recovery_segment_fixed_framing; +mod recovery_segment_resume_error; +mod recovery_segment_resume_executor; +mod recovery_segment_resume_plan_error; +mod recovery_segment_resume_planner; +mod recovery_segment_resume_request; +pub(in crate::adapters) mod recovery_segment_resume_state; +mod recovery_segment_resume_storage; +mod recovery_segment_resume_storage_error; +mod recovery_segment_stage; +mod recovery_segment_stage_error; +mod recovery_segment_truncation; +mod recovery_stage; +mod recovery_stage_assessment; +mod recovery_stage_assessment_error; +mod recovery_stage_assessor; +mod recovery_stage_byte_admission; +mod recovery_stage_byte_admission_error; +mod recovery_stage_completion_error; +mod recovery_stage_completion_executor; +mod recovery_stage_completion_plan_error; +mod recovery_stage_completion_planner; +mod recovery_stage_completion_pool; +mod recovery_stage_completion_receipt; +mod recovery_stage_completion_request; +mod recovery_stage_completion_storage; +mod recovery_stage_completion_storage_error; +mod recovery_stage_completion_target; +mod recovery_stage_discard_error; +mod recovery_stage_discard_executor; +mod recovery_stage_discard_outcome; +mod recovery_stage_discard_plan_error; +mod recovery_stage_discard_planner; +mod recovery_stage_discard_reason; +mod recovery_stage_discard_receipt; +mod recovery_stage_discard_request; +mod recovery_stage_discard_storage; +mod recovery_stage_discard_storage_error; +mod recovery_stage_evidence; +mod recovery_stage_fingerprint; +mod recovery_stage_fingerprint_algorithm; +mod recovery_stage_fingerprint_error; +mod recovery_stage_fingerprinter; +mod recovery_stage_length; +mod recovery_stage_metadata; +mod recovery_stage_metadata_error; +mod recovery_stage_parent; +mod recovery_stage_pool_outcome; +mod recovery_stage_synchronization_outcome; + +use super::{ + AdmittedRecoveryStageBytes, AdmittedSegment, CatalogDecodeError, CatalogPublicationExpectation, + CatalogRestartError, CatalogSnapshot, CatalogTransitionError, ChecksummedCatalog, + ChecksummedPublicationHead, FilesystemRecoveryStageError, OpenedReusableSegment, + PublicationHeadDecodeError, SegmentDigest, SegmentHeader, SegmentHeaderError, SegmentReadError, + SegmentReadPolicy, SegmentRecordHeaderError, SegmentRecordIdentity, SegmentRecordLimit, + SegmentSeal, SegmentStage, StagedSegment, catalog_decoder, catalog_header_decoder, + catalog_publication_expectation, catalog_transition, framed_blake3, lower_hex, + publication_head_decoder, segment_digest_builder, segment_header, segment_identity_index, + segment_record_cursor, segment_record_cursor_decode, segment_record_header, + segment_record_kind, segment_seal, +}; + +pub use recovery_catalog_stage::RecoveryCatalogStage; +pub use recovery_catalog_stage_error::RecoveryCatalogStageError; +pub use recovery_entry_name::{RecoveryEntryName, RecoveryEntryNameError}; +pub use recovery_entry_role::RecoveryEntryRole; +pub use recovery_inventory::{RecoveryInventory, RecoveryInventoryEntry, read_recovery_inventory}; +pub use recovery_inventory_error::RecoveryInventoryError; +pub use recovery_inventory_limit::{RecoveryInventoryLimit, RecoveryInventoryLimitError}; +pub use recovery_inventory_operation::RecoveryInventoryOperation; +pub use recovery_inventory_storage::RecoveryInventoryStorage; +pub use recovery_name_classification::classify_recovery_names; +pub use recovery_name_classification_error::RecoveryNameClassificationError; +pub use recovery_name_manifest::{RecoveryNameManifest, RecoveryNamedEntry}; +pub use recovery_namespace::RecoveryNamespace; +pub use recovery_next_head_finalization_error::RecoveryNextHeadFinalizationError; +pub use recovery_next_head_finalization_executor::execute_recovery_next_head_finalization; +pub use recovery_next_head_finalization_outcome::RecoveryNextHeadFinalizationOutcome; +pub use recovery_next_head_finalization_plan_error::RecoveryNextHeadFinalizationPlanError; +pub use recovery_next_head_finalization_planner::plan_recovery_next_head_finalization; +pub use recovery_next_head_finalization_readiness::RecoveryNextHeadFinalizationReadiness; +pub use recovery_next_head_finalization_receipt::RecoveryNextHeadFinalizationReceipt; +pub use recovery_next_head_finalization_request::RecoveryNextHeadFinalizationRequest; +pub use recovery_next_head_finalization_storage::RecoveryNextHeadFinalizationStorage; +pub use recovery_next_head_finalization_storage_error::RecoveryNextHeadFinalizationStorageError; +pub use recovery_next_head_finalization_target::RecoveryNextHeadFinalizationTarget; +pub use recovery_next_head_stage::RecoveryNextHeadStage; +pub use recovery_next_head_stage_error::RecoveryNextHeadStageError; +pub use recovery_pool_name_error::RecoveryPoolNameError; +pub use recovery_publication_stage_classifier::{ + classify_recovery_catalog_stage, classify_recovery_next_head_stage, +}; +pub use recovery_required_entry::RecoveryRequiredEntry; +pub use recovery_segment_classifier::classify_recovery_segment_stage; +pub use recovery_segment_resume_error::RecoverySegmentResumeError; +pub use recovery_segment_resume_executor::execute_recovery_segment_resume; +pub use recovery_segment_resume_plan_error::RecoverySegmentResumePlanError; +pub use recovery_segment_resume_planner::plan_recovery_segment_resume; +pub use recovery_segment_resume_request::RecoverySegmentResumeRequest; +pub use recovery_segment_resume_storage::RecoverySegmentResumeStorage; +pub use recovery_segment_resume_storage_error::RecoverySegmentResumeStorageError; +pub use recovery_segment_stage::{RecoverySegmentStage, ReusableRecoverySegment}; +pub use recovery_segment_stage_error::RecoverySegmentStageError; +pub use recovery_segment_truncation::RecoverySegmentTruncation; +pub use recovery_stage::RecoveryStage; +pub use recovery_stage_assessment::RecoveryStageAssessment; +pub use recovery_stage_assessment_error::RecoveryStageAssessmentError; +pub use recovery_stage_assessor::assess_recovery_stage; +pub use recovery_stage_byte_admission::admit_recovery_stage_bytes; +pub use recovery_stage_byte_admission_error::RecoveryStageByteAdmissionError; +pub use recovery_stage_completion_error::RecoveryStageCompletionError; +pub use recovery_stage_completion_executor::execute_recovery_stage_completion; +pub use recovery_stage_completion_plan_error::RecoveryStageCompletionPlanError; +pub use recovery_stage_completion_planner::plan_recovery_stage_completion; +pub use recovery_stage_completion_pool::RecoveryStageCompletionPool; +pub use recovery_stage_completion_receipt::RecoveryStageCompletionReceipt; +pub use recovery_stage_completion_request::RecoveryStageCompletionRequest; +pub use recovery_stage_completion_storage::RecoveryStageCompletionStorage; +pub use recovery_stage_completion_storage_error::RecoveryStageCompletionStorageError; +pub use recovery_stage_completion_target::RecoveryStageCompletionTarget; +pub use recovery_stage_discard_error::RecoveryStageDiscardError; +pub use recovery_stage_discard_executor::execute_recovery_stage_discard; +pub use recovery_stage_discard_outcome::RecoveryStageDiscardOutcome; +pub use recovery_stage_discard_plan_error::RecoveryStageDiscardPlanError; +pub use recovery_stage_discard_planner::plan_recovery_stage_discard; +pub use recovery_stage_discard_reason::RecoveryStageDiscardReason; +pub use recovery_stage_discard_receipt::RecoveryStageDiscardReceipt; +pub use recovery_stage_discard_request::RecoveryStageDiscardRequest; +pub use recovery_stage_discard_storage::RecoveryStageDiscardStorage; +pub use recovery_stage_discard_storage_error::RecoveryStageDiscardStorageError; +pub use recovery_stage_evidence::RecoveryStageEvidence; +pub use recovery_stage_fingerprint::RecoveryStageFingerprint; +pub use recovery_stage_fingerprint_algorithm::RecoveryStageFingerprintAlgorithm; +pub use recovery_stage_fingerprint_error::RecoveryStageFingerprintError; +pub use recovery_stage_fingerprinter::fingerprint_recovery_stage; +pub use recovery_stage_length::RecoveryStageLength; +pub use recovery_stage_metadata::RecoveryStageMetadata; +pub use recovery_stage_metadata_error::RecoveryStageMetadataError; +pub use recovery_stage_parent::RecoveryStageParent; +pub use recovery_stage_pool_outcome::RecoveryStagePoolOutcome; +pub use recovery_stage_synchronization_outcome::RecoveryStageSynchronizationOutcome; diff --git a/src/adapters/recovery_catalog_stage.rs b/src/adapters/recovery/recovery_catalog_stage.rs similarity index 100% rename from src/adapters/recovery_catalog_stage.rs rename to src/adapters/recovery/recovery_catalog_stage.rs diff --git a/src/adapters/recovery_catalog_stage_error.rs b/src/adapters/recovery/recovery_catalog_stage_error.rs similarity index 100% rename from src/adapters/recovery_catalog_stage_error.rs rename to src/adapters/recovery/recovery_catalog_stage_error.rs diff --git a/src/adapters/recovery_entry_name.rs b/src/adapters/recovery/recovery_entry_name.rs similarity index 100% rename from src/adapters/recovery_entry_name.rs rename to src/adapters/recovery/recovery_entry_name.rs diff --git a/src/adapters/recovery_entry_role.rs b/src/adapters/recovery/recovery_entry_role.rs similarity index 95% rename from src/adapters/recovery_entry_role.rs rename to src/adapters/recovery/recovery_entry_role.rs index 53932f1..a9aef90 100644 --- a/src/adapters/recovery_entry_role.rs +++ b/src/adapters/recovery/recovery_entry_role.rs @@ -37,7 +37,7 @@ pub enum RecoveryEntryRole { } impl RecoveryEntryRole { - pub(super) const fn is_stage(self) -> bool { + pub(in crate::adapters) const fn is_stage(self) -> bool { matches!( self, Self::NextHeadStage | Self::SegmentStage | Self::CatalogStage diff --git a/src/adapters/recovery_fixed_field_prefix.rs b/src/adapters/recovery/recovery_fixed_field_prefix.rs similarity index 86% rename from src/adapters/recovery_fixed_field_prefix.rs rename to src/adapters/recovery/recovery_fixed_field_prefix.rs index d7b5ad9..99804b5 100644 --- a/src/adapters/recovery_fixed_field_prefix.rs +++ b/src/adapters/recovery/recovery_fixed_field_prefix.rs @@ -1,6 +1,6 @@ //! This module owns completion of available fixed-field recovery prefixes. -pub(super) fn observed_field( +pub(in crate::adapters) fn observed_field( encoded: &[u8], offset: usize, canonical: [u8; LENGTH], diff --git a/src/adapters/recovery_inventory.rs b/src/adapters/recovery/recovery_inventory.rs similarity index 96% rename from src/adapters/recovery_inventory.rs rename to src/adapters/recovery/recovery_inventory.rs index ba58212..c7d03d3 100644 --- a/src/adapters/recovery_inventory.rs +++ b/src/adapters/recovery/recovery_inventory.rs @@ -32,7 +32,7 @@ impl RecoveryInventoryEntry { &self.name } - pub(super) fn into_parts(self) -> (RecoveryNamespace, RecoveryEntryName) { + pub(in crate::adapters) fn into_parts(self) -> (RecoveryNamespace, RecoveryEntryName) { (self.namespace, self.name) } } @@ -50,7 +50,7 @@ impl RecoveryInventory { &self.entries } - pub(super) fn into_entries(self) -> Vec { + pub(in crate::adapters) fn into_entries(self) -> Vec { self.entries } } diff --git a/src/adapters/recovery_inventory_error.rs b/src/adapters/recovery/recovery_inventory_error.rs similarity index 98% rename from src/adapters/recovery_inventory_error.rs rename to src/adapters/recovery/recovery_inventory_error.rs index a9eae72..100874c 100644 --- a/src/adapters/recovery_inventory_error.rs +++ b/src/adapters/recovery/recovery_inventory_error.rs @@ -49,7 +49,7 @@ pub enum RecoveryInventoryError { } impl RecoveryInventoryError { - pub(super) const fn io( + pub(in crate::adapters) const fn io( namespace: RecoveryNamespace, operation: RecoveryInventoryOperation, source: io::Error, diff --git a/src/adapters/recovery_inventory_limit.rs b/src/adapters/recovery/recovery_inventory_limit.rs similarity index 100% rename from src/adapters/recovery_inventory_limit.rs rename to src/adapters/recovery/recovery_inventory_limit.rs diff --git a/src/adapters/recovery_inventory_operation.rs b/src/adapters/recovery/recovery_inventory_operation.rs similarity index 100% rename from src/adapters/recovery_inventory_operation.rs rename to src/adapters/recovery/recovery_inventory_operation.rs diff --git a/src/adapters/recovery_inventory_storage.rs b/src/adapters/recovery/recovery_inventory_storage.rs similarity index 100% rename from src/adapters/recovery_inventory_storage.rs rename to src/adapters/recovery/recovery_inventory_storage.rs diff --git a/src/adapters/recovery_name_classification.rs b/src/adapters/recovery/recovery_name_classification.rs similarity index 100% rename from src/adapters/recovery_name_classification.rs rename to src/adapters/recovery/recovery_name_classification.rs diff --git a/src/adapters/recovery_name_classification_error.rs b/src/adapters/recovery/recovery_name_classification_error.rs similarity index 100% rename from src/adapters/recovery_name_classification_error.rs rename to src/adapters/recovery/recovery_name_classification_error.rs diff --git a/src/adapters/recovery_name_manifest.rs b/src/adapters/recovery/recovery_name_manifest.rs similarity index 91% rename from src/adapters/recovery_name_manifest.rs rename to src/adapters/recovery/recovery_name_manifest.rs index 277cf40..893e792 100644 --- a/src/adapters/recovery_name_manifest.rs +++ b/src/adapters/recovery/recovery_name_manifest.rs @@ -11,7 +11,7 @@ pub struct RecoveryNamedEntry { } impl RecoveryNamedEntry { - pub(super) const fn new( + pub(in crate::adapters) const fn new( namespace: RecoveryNamespace, name: RecoveryEntryName, role: RecoveryEntryRole, @@ -49,7 +49,7 @@ pub struct RecoveryNameManifest { } impl RecoveryNameManifest { - pub(super) const fn new(entries: Vec) -> Self { + pub(in crate::adapters) const fn new(entries: Vec) -> Self { Self { entries } } diff --git a/src/adapters/recovery_namespace.rs b/src/adapters/recovery/recovery_namespace.rs similarity index 100% rename from src/adapters/recovery_namespace.rs rename to src/adapters/recovery/recovery_namespace.rs diff --git a/src/adapters/recovery_next_head_finalization_error.rs b/src/adapters/recovery/recovery_next_head_finalization_error.rs similarity index 100% rename from src/adapters/recovery_next_head_finalization_error.rs rename to src/adapters/recovery/recovery_next_head_finalization_error.rs diff --git a/src/adapters/recovery_next_head_finalization_executor.rs b/src/adapters/recovery/recovery_next_head_finalization_executor.rs similarity index 100% rename from src/adapters/recovery_next_head_finalization_executor.rs rename to src/adapters/recovery/recovery_next_head_finalization_executor.rs diff --git a/src/adapters/recovery_next_head_finalization_outcome.rs b/src/adapters/recovery/recovery_next_head_finalization_outcome.rs similarity index 100% rename from src/adapters/recovery_next_head_finalization_outcome.rs rename to src/adapters/recovery/recovery_next_head_finalization_outcome.rs diff --git a/src/adapters/recovery_next_head_finalization_plan_error.rs b/src/adapters/recovery/recovery_next_head_finalization_plan_error.rs similarity index 100% rename from src/adapters/recovery_next_head_finalization_plan_error.rs rename to src/adapters/recovery/recovery_next_head_finalization_plan_error.rs diff --git a/src/adapters/recovery_next_head_finalization_planner.rs b/src/adapters/recovery/recovery_next_head_finalization_planner.rs similarity index 100% rename from src/adapters/recovery_next_head_finalization_planner.rs rename to src/adapters/recovery/recovery_next_head_finalization_planner.rs diff --git a/src/adapters/recovery_next_head_finalization_readiness.rs b/src/adapters/recovery/recovery_next_head_finalization_readiness.rs similarity index 100% rename from src/adapters/recovery_next_head_finalization_readiness.rs rename to src/adapters/recovery/recovery_next_head_finalization_readiness.rs diff --git a/src/adapters/recovery_next_head_finalization_receipt.rs b/src/adapters/recovery/recovery_next_head_finalization_receipt.rs similarity index 96% rename from src/adapters/recovery_next_head_finalization_receipt.rs rename to src/adapters/recovery/recovery_next_head_finalization_receipt.rs index 5fd8133..9c1e4bd 100644 --- a/src/adapters/recovery_next_head_finalization_receipt.rs +++ b/src/adapters/recovery/recovery_next_head_finalization_receipt.rs @@ -14,7 +14,7 @@ pub struct RecoveryNextHeadFinalizationReceipt { } impl RecoveryNextHeadFinalizationReceipt { - pub(super) const fn new( + pub(in crate::adapters) const fn new( request: RecoveryNextHeadFinalizationRequest, outcome: RecoveryNextHeadFinalizationOutcome, ) -> Self { diff --git a/src/adapters/recovery_next_head_finalization_request.rs b/src/adapters/recovery/recovery_next_head_finalization_request.rs similarity index 96% rename from src/adapters/recovery_next_head_finalization_request.rs rename to src/adapters/recovery/recovery_next_head_finalization_request.rs index 73a433b..9a1c20b 100644 --- a/src/adapters/recovery_next_head_finalization_request.rs +++ b/src/adapters/recovery/recovery_next_head_finalization_request.rs @@ -14,7 +14,7 @@ pub struct RecoveryNextHeadFinalizationRequest { } impl RecoveryNextHeadFinalizationRequest { - pub(super) const fn new( + pub(in crate::adapters) const fn new( evidence: RecoveryStageEvidence, expectation: CatalogPublicationExpectation, target: RecoveryNextHeadFinalizationTarget, diff --git a/src/adapters/recovery_next_head_finalization_storage.rs b/src/adapters/recovery/recovery_next_head_finalization_storage.rs similarity index 100% rename from src/adapters/recovery_next_head_finalization_storage.rs rename to src/adapters/recovery/recovery_next_head_finalization_storage.rs diff --git a/src/adapters/recovery_next_head_finalization_storage_error.rs b/src/adapters/recovery/recovery_next_head_finalization_storage_error.rs similarity index 100% rename from src/adapters/recovery_next_head_finalization_storage_error.rs rename to src/adapters/recovery/recovery_next_head_finalization_storage_error.rs diff --git a/src/adapters/recovery_next_head_finalization_target.rs b/src/adapters/recovery/recovery_next_head_finalization_target.rs similarity index 97% rename from src/adapters/recovery_next_head_finalization_target.rs rename to src/adapters/recovery/recovery_next_head_finalization_target.rs index b3e9277..3991977 100644 --- a/src/adapters/recovery_next_head_finalization_target.rs +++ b/src/adapters/recovery/recovery_next_head_finalization_target.rs @@ -22,7 +22,7 @@ impl RecoveryNextHeadFinalizationTarget { ) } - pub(super) const fn new( + pub(in crate::adapters) const fn new( generation: CatalogGeneration, length: CatalogLength, digest: CatalogDigest, diff --git a/src/adapters/recovery_next_head_stage.rs b/src/adapters/recovery/recovery_next_head_stage.rs similarity index 100% rename from src/adapters/recovery_next_head_stage.rs rename to src/adapters/recovery/recovery_next_head_stage.rs diff --git a/src/adapters/recovery_next_head_stage_error.rs b/src/adapters/recovery/recovery_next_head_stage_error.rs similarity index 100% rename from src/adapters/recovery_next_head_stage_error.rs rename to src/adapters/recovery/recovery_next_head_stage_error.rs diff --git a/src/adapters/recovery_pool_name.rs b/src/adapters/recovery/recovery_pool_name.rs similarity index 95% rename from src/adapters/recovery_pool_name.rs rename to src/adapters/recovery/recovery_pool_name.rs index 751ddc3..def5a18 100644 --- a/src/adapters/recovery_pool_name.rs +++ b/src/adapters/recovery/recovery_pool_name.rs @@ -8,7 +8,9 @@ const SEGMENT_NAME_LENGTH: usize = 68; const CATALOG_NAME_LENGTH: usize = 85; const GENERATION_LENGTH: usize = 16; -pub(super) fn segment(name: &RecoveryEntryName) -> Result { +pub(in crate::adapters) fn segment( + name: &RecoveryEntryName, +) -> Result { let bytes = name.as_bytes(); require_length(bytes, SEGMENT_NAME_LENGTH)?; let digest = bytes @@ -17,7 +19,7 @@ pub(super) fn segment(name: &RecoveryEntryName) -> Result Result<(CatalogGeneration, CatalogDigest), RecoveryPoolNameError> { let bytes = name.as_bytes(); diff --git a/src/adapters/recovery_pool_name_error.rs b/src/adapters/recovery/recovery_pool_name_error.rs similarity index 100% rename from src/adapters/recovery_pool_name_error.rs rename to src/adapters/recovery/recovery_pool_name_error.rs diff --git a/src/adapters/recovery_publication_fixed_framing.rs b/src/adapters/recovery/recovery_publication_fixed_framing.rs similarity index 96% rename from src/adapters/recovery_publication_fixed_framing.rs rename to src/adapters/recovery/recovery_publication_fixed_framing.rs index e26375f..7134a28 100644 --- a/src/adapters/recovery_publication_fixed_framing.rs +++ b/src/adapters/recovery/recovery_publication_fixed_framing.rs @@ -6,7 +6,7 @@ use super::{ publication_head_decoder, }; -pub(super) fn catalog_header(encoded: &[u8]) -> Result<(), CatalogDecodeError> { +pub(in crate::adapters) fn catalog_header(encoded: &[u8]) -> Result<(), CatalogDecodeError> { let magic = observed_field(encoded, 0, catalog_decoder::MAGIC); if magic != catalog_decoder::MAGIC { return Err(CatalogDecodeError::InvalidMagic { observed: magic }); @@ -87,7 +87,7 @@ fn validate_catalog_coordinates(encoded: &[u8]) -> Result<(), CatalogDecodeError } } -pub(super) fn next_head(encoded: &[u8]) -> Result<(), PublicationHeadDecodeError> { +pub(in crate::adapters) fn next_head(encoded: &[u8]) -> Result<(), PublicationHeadDecodeError> { let magic = observed_field(encoded, 0, publication_head_decoder::MAGIC); if magic != publication_head_decoder::MAGIC { return Err(PublicationHeadDecodeError::InvalidMagic { observed: magic }); diff --git a/src/adapters/recovery_publication_stage_classifier.rs b/src/adapters/recovery/recovery_publication_stage_classifier.rs similarity index 100% rename from src/adapters/recovery_publication_stage_classifier.rs rename to src/adapters/recovery/recovery_publication_stage_classifier.rs diff --git a/src/adapters/recovery_required_entry.rs b/src/adapters/recovery/recovery_required_entry.rs similarity index 100% rename from src/adapters/recovery_required_entry.rs rename to src/adapters/recovery/recovery_required_entry.rs diff --git a/src/adapters/recovery_segment_classifier.rs b/src/adapters/recovery/recovery_segment_classifier.rs similarity index 100% rename from src/adapters/recovery_segment_classifier.rs rename to src/adapters/recovery/recovery_segment_classifier.rs diff --git a/src/adapters/recovery_segment_fixed_framing.rs b/src/adapters/recovery/recovery_segment_fixed_framing.rs similarity index 95% rename from src/adapters/recovery_segment_fixed_framing.rs rename to src/adapters/recovery/recovery_segment_fixed_framing.rs index c748245..61be6ea 100644 --- a/src/adapters/recovery_segment_fixed_framing.rs +++ b/src/adapters/recovery/recovery_segment_fixed_framing.rs @@ -10,12 +10,12 @@ use super::{ segment_record_kind::SegmentRecordKind, segment_seal, }; -pub(super) fn segment_header(encoded: &[u8]) -> Result<(), SegmentHeaderError> { +pub(in crate::adapters) fn segment_header(encoded: &[u8]) -> Result<(), SegmentHeaderError> { let completed = observed_field(encoded, 0, SegmentHeader::admitted().encode()); SegmentHeader::decode(&completed).map(|_header| ()) } -pub(super) fn segment_tail(encoded: &[u8]) -> Result<(), SegmentRecordHeaderError> { +pub(in crate::adapters) fn segment_tail(encoded: &[u8]) -> Result<(), SegmentRecordHeaderError> { if segment_seal::MAGIC.starts_with(encoded) { return Ok(()); } diff --git a/src/adapters/recovery_segment_resume_error.rs b/src/adapters/recovery/recovery_segment_resume_error.rs similarity index 100% rename from src/adapters/recovery_segment_resume_error.rs rename to src/adapters/recovery/recovery_segment_resume_error.rs diff --git a/src/adapters/recovery_segment_resume_executor.rs b/src/adapters/recovery/recovery_segment_resume_executor.rs similarity index 100% rename from src/adapters/recovery_segment_resume_executor.rs rename to src/adapters/recovery/recovery_segment_resume_executor.rs diff --git a/src/adapters/recovery_segment_resume_plan_error.rs b/src/adapters/recovery/recovery_segment_resume_plan_error.rs similarity index 100% rename from src/adapters/recovery_segment_resume_plan_error.rs rename to src/adapters/recovery/recovery_segment_resume_plan_error.rs diff --git a/src/adapters/recovery_segment_resume_planner.rs b/src/adapters/recovery/recovery_segment_resume_planner.rs similarity index 100% rename from src/adapters/recovery_segment_resume_planner.rs rename to src/adapters/recovery/recovery_segment_resume_planner.rs diff --git a/src/adapters/recovery_segment_resume_request.rs b/src/adapters/recovery/recovery_segment_resume_request.rs similarity index 97% rename from src/adapters/recovery_segment_resume_request.rs rename to src/adapters/recovery/recovery_segment_resume_request.rs index dcafb6b..ceda76d 100644 --- a/src/adapters/recovery_segment_resume_request.rs +++ b/src/adapters/recovery/recovery_segment_resume_request.rs @@ -13,7 +13,7 @@ pub struct RecoverySegmentResumeRequest { } impl RecoverySegmentResumeRequest { - pub(super) const fn new( + pub(in crate::adapters) const fn new( evidence: RecoveryStageEvidence, record_count: u32, length: RecoveryStageLength, diff --git a/src/adapters/recovery_segment_resume_state.rs b/src/adapters/recovery/recovery_segment_resume_state.rs similarity index 83% rename from src/adapters/recovery_segment_resume_state.rs rename to src/adapters/recovery/recovery_segment_resume_state.rs index 9632291..6ab205d 100644 --- a/src/adapters/recovery_segment_resume_state.rs +++ b/src/adapters/recovery/recovery_segment_resume_state.rs @@ -9,16 +9,16 @@ use super::{ SegmentRecordLimit, }; -pub(super) struct RecoverySegmentResumeState { - pub(super) digest: SegmentDigestBuilder, - pub(super) identities: HashSet, - pub(super) record_limit: SegmentRecordLimit, - pub(super) record_count: u32, - pub(super) bytes_written: u64, +pub(in crate::adapters) struct RecoverySegmentResumeState { + pub(in crate::adapters) digest: SegmentDigestBuilder, + pub(in crate::adapters) identities: HashSet, + pub(in crate::adapters) record_limit: SegmentRecordLimit, + pub(in crate::adapters) record_count: u32, + pub(in crate::adapters) bytes_written: u64, } impl RecoverySegmentResumeState { - pub(super) fn rebuild( + pub(in crate::adapters) fn rebuild( encoded: &[u8], request: RecoverySegmentResumeRequest, ) -> Result { diff --git a/src/adapters/recovery_segment_resume_storage.rs b/src/adapters/recovery/recovery_segment_resume_storage.rs similarity index 100% rename from src/adapters/recovery_segment_resume_storage.rs rename to src/adapters/recovery/recovery_segment_resume_storage.rs diff --git a/src/adapters/recovery_segment_resume_storage_error.rs b/src/adapters/recovery/recovery_segment_resume_storage_error.rs similarity index 100% rename from src/adapters/recovery_segment_resume_storage_error.rs rename to src/adapters/recovery/recovery_segment_resume_storage_error.rs diff --git a/src/adapters/recovery_segment_stage.rs b/src/adapters/recovery/recovery_segment_stage.rs similarity index 92% rename from src/adapters/recovery_segment_stage.rs rename to src/adapters/recovery/recovery_segment_stage.rs index 2e2b968..f5b628f 100644 --- a/src/adapters/recovery_segment_stage.rs +++ b/src/adapters/recovery/recovery_segment_stage.rs @@ -22,7 +22,7 @@ pub struct ReusableRecoverySegment { } impl ReusableRecoverySegment { - pub(super) const fn new(record_count: u32, length: RecoveryStageLength) -> Self { + pub(in crate::adapters) const fn new(record_count: u32, length: RecoveryStageLength) -> Self { Self { record_count, length, diff --git a/src/adapters/recovery_segment_stage_error.rs b/src/adapters/recovery/recovery_segment_stage_error.rs similarity index 100% rename from src/adapters/recovery_segment_stage_error.rs rename to src/adapters/recovery/recovery_segment_stage_error.rs diff --git a/src/adapters/recovery_segment_truncation.rs b/src/adapters/recovery/recovery_segment_truncation.rs similarity index 100% rename from src/adapters/recovery_segment_truncation.rs rename to src/adapters/recovery/recovery_segment_truncation.rs diff --git a/src/adapters/recovery_stage.rs b/src/adapters/recovery/recovery_stage.rs similarity index 95% rename from src/adapters/recovery_stage.rs rename to src/adapters/recovery/recovery_stage.rs index 34156be..170a3fb 100644 --- a/src/adapters/recovery_stage.rs +++ b/src/adapters/recovery/recovery_stage.rs @@ -17,7 +17,7 @@ pub enum RecoveryStage { } impl RecoveryStage { - pub(super) const fn file_name(self) -> &'static str { + pub(in crate::adapters) const fn file_name(self) -> &'static str { match self { Self::Segment => "current.seg", Self::Catalog => "current.cat", diff --git a/src/adapters/recovery_stage_assessment.rs b/src/adapters/recovery/recovery_stage_assessment.rs similarity index 100% rename from src/adapters/recovery_stage_assessment.rs rename to src/adapters/recovery/recovery_stage_assessment.rs diff --git a/src/adapters/recovery_stage_assessment_error.rs b/src/adapters/recovery/recovery_stage_assessment_error.rs similarity index 100% rename from src/adapters/recovery_stage_assessment_error.rs rename to src/adapters/recovery/recovery_stage_assessment_error.rs diff --git a/src/adapters/recovery_stage_assessor.rs b/src/adapters/recovery/recovery_stage_assessor.rs similarity index 100% rename from src/adapters/recovery_stage_assessor.rs rename to src/adapters/recovery/recovery_stage_assessor.rs diff --git a/src/adapters/recovery_stage_byte_admission.rs b/src/adapters/recovery/recovery_stage_byte_admission.rs similarity index 100% rename from src/adapters/recovery_stage_byte_admission.rs rename to src/adapters/recovery/recovery_stage_byte_admission.rs diff --git a/src/adapters/recovery_stage_byte_admission_error.rs b/src/adapters/recovery/recovery_stage_byte_admission_error.rs similarity index 100% rename from src/adapters/recovery_stage_byte_admission_error.rs rename to src/adapters/recovery/recovery_stage_byte_admission_error.rs diff --git a/src/adapters/recovery_stage_completion_error.rs b/src/adapters/recovery/recovery_stage_completion_error.rs similarity index 100% rename from src/adapters/recovery_stage_completion_error.rs rename to src/adapters/recovery/recovery_stage_completion_error.rs diff --git a/src/adapters/recovery_stage_completion_executor.rs b/src/adapters/recovery/recovery_stage_completion_executor.rs similarity index 100% rename from src/adapters/recovery_stage_completion_executor.rs rename to src/adapters/recovery/recovery_stage_completion_executor.rs diff --git a/src/adapters/recovery_stage_completion_plan_error.rs b/src/adapters/recovery/recovery_stage_completion_plan_error.rs similarity index 100% rename from src/adapters/recovery_stage_completion_plan_error.rs rename to src/adapters/recovery/recovery_stage_completion_plan_error.rs diff --git a/src/adapters/recovery_stage_completion_planner.rs b/src/adapters/recovery/recovery_stage_completion_planner.rs similarity index 100% rename from src/adapters/recovery_stage_completion_planner.rs rename to src/adapters/recovery/recovery_stage_completion_planner.rs diff --git a/src/adapters/recovery_stage_completion_pool.rs b/src/adapters/recovery/recovery_stage_completion_pool.rs similarity index 100% rename from src/adapters/recovery_stage_completion_pool.rs rename to src/adapters/recovery/recovery_stage_completion_pool.rs diff --git a/src/adapters/recovery_stage_completion_receipt.rs b/src/adapters/recovery/recovery_stage_completion_receipt.rs similarity index 98% rename from src/adapters/recovery_stage_completion_receipt.rs rename to src/adapters/recovery/recovery_stage_completion_receipt.rs index 2cad1eb..49bb042 100644 --- a/src/adapters/recovery_stage_completion_receipt.rs +++ b/src/adapters/recovery/recovery_stage_completion_receipt.rs @@ -19,7 +19,7 @@ pub struct RecoveryStageCompletionReceipt { } impl RecoveryStageCompletionReceipt { - pub(super) const fn new( + pub(in crate::adapters) const fn new( request: RecoveryStageCompletionRequest, synchronization_outcome: RecoveryStageSynchronizationOutcome, pool_outcome: RecoveryStagePoolOutcome, diff --git a/src/adapters/recovery_stage_completion_request.rs b/src/adapters/recovery/recovery_stage_completion_request.rs similarity index 96% rename from src/adapters/recovery_stage_completion_request.rs rename to src/adapters/recovery/recovery_stage_completion_request.rs index 8a5ca02..20e64f1 100644 --- a/src/adapters/recovery_stage_completion_request.rs +++ b/src/adapters/recovery/recovery_stage_completion_request.rs @@ -11,7 +11,7 @@ pub struct RecoveryStageCompletionRequest { } impl RecoveryStageCompletionRequest { - pub(super) const fn new( + pub(in crate::adapters) const fn new( evidence: RecoveryStageEvidence, target: RecoveryStageCompletionTarget, ) -> Self { diff --git a/src/adapters/recovery_stage_completion_storage.rs b/src/adapters/recovery/recovery_stage_completion_storage.rs similarity index 100% rename from src/adapters/recovery_stage_completion_storage.rs rename to src/adapters/recovery/recovery_stage_completion_storage.rs diff --git a/src/adapters/recovery_stage_completion_storage_error.rs b/src/adapters/recovery/recovery_stage_completion_storage_error.rs similarity index 96% rename from src/adapters/recovery_stage_completion_storage_error.rs rename to src/adapters/recovery/recovery_stage_completion_storage_error.rs index c4fda75..5c6dc99 100644 --- a/src/adapters/recovery_stage_completion_storage_error.rs +++ b/src/adapters/recovery/recovery_stage_completion_storage_error.rs @@ -29,7 +29,7 @@ pub enum RecoveryStageCompletionStorageError { } impl RecoveryStageCompletionStorageError { - pub(super) const fn storage(source: io::Error) -> Self { + pub(in crate::adapters) const fn storage(source: io::Error) -> Self { Self::Storage { source } } } diff --git a/src/adapters/recovery_stage_completion_target.rs b/src/adapters/recovery/recovery_stage_completion_target.rs similarity index 100% rename from src/adapters/recovery_stage_completion_target.rs rename to src/adapters/recovery/recovery_stage_completion_target.rs diff --git a/src/adapters/recovery_stage_discard_error.rs b/src/adapters/recovery/recovery_stage_discard_error.rs similarity index 100% rename from src/adapters/recovery_stage_discard_error.rs rename to src/adapters/recovery/recovery_stage_discard_error.rs diff --git a/src/adapters/recovery_stage_discard_executor.rs b/src/adapters/recovery/recovery_stage_discard_executor.rs similarity index 100% rename from src/adapters/recovery_stage_discard_executor.rs rename to src/adapters/recovery/recovery_stage_discard_executor.rs diff --git a/src/adapters/recovery_stage_discard_outcome.rs b/src/adapters/recovery/recovery_stage_discard_outcome.rs similarity index 100% rename from src/adapters/recovery_stage_discard_outcome.rs rename to src/adapters/recovery/recovery_stage_discard_outcome.rs diff --git a/src/adapters/recovery_stage_discard_plan_error.rs b/src/adapters/recovery/recovery_stage_discard_plan_error.rs similarity index 100% rename from src/adapters/recovery_stage_discard_plan_error.rs rename to src/adapters/recovery/recovery_stage_discard_plan_error.rs diff --git a/src/adapters/recovery_stage_discard_planner.rs b/src/adapters/recovery/recovery_stage_discard_planner.rs similarity index 100% rename from src/adapters/recovery_stage_discard_planner.rs rename to src/adapters/recovery/recovery_stage_discard_planner.rs diff --git a/src/adapters/recovery_stage_discard_reason.rs b/src/adapters/recovery/recovery_stage_discard_reason.rs similarity index 100% rename from src/adapters/recovery_stage_discard_reason.rs rename to src/adapters/recovery/recovery_stage_discard_reason.rs diff --git a/src/adapters/recovery_stage_discard_receipt.rs b/src/adapters/recovery/recovery_stage_discard_receipt.rs similarity index 96% rename from src/adapters/recovery_stage_discard_receipt.rs rename to src/adapters/recovery/recovery_stage_discard_receipt.rs index 371237b..3f8d9a7 100644 --- a/src/adapters/recovery_stage_discard_receipt.rs +++ b/src/adapters/recovery/recovery_stage_discard_receipt.rs @@ -14,7 +14,7 @@ pub struct RecoveryStageDiscardReceipt { } impl RecoveryStageDiscardReceipt { - pub(super) const fn new( + pub(in crate::adapters) const fn new( request: RecoveryStageDiscardRequest, outcome: RecoveryStageDiscardOutcome, ) -> Self { diff --git a/src/adapters/recovery_stage_discard_request.rs b/src/adapters/recovery/recovery_stage_discard_request.rs similarity index 96% rename from src/adapters/recovery_stage_discard_request.rs rename to src/adapters/recovery/recovery_stage_discard_request.rs index 69b5eaf..43e3a99 100644 --- a/src/adapters/recovery_stage_discard_request.rs +++ b/src/adapters/recovery/recovery_stage_discard_request.rs @@ -11,7 +11,7 @@ pub struct RecoveryStageDiscardRequest { } impl RecoveryStageDiscardRequest { - pub(super) const fn new( + pub(in crate::adapters) const fn new( evidence: RecoveryStageEvidence, reason: RecoveryStageDiscardReason, ) -> Self { diff --git a/src/adapters/recovery_stage_discard_storage.rs b/src/adapters/recovery/recovery_stage_discard_storage.rs similarity index 100% rename from src/adapters/recovery_stage_discard_storage.rs rename to src/adapters/recovery/recovery_stage_discard_storage.rs diff --git a/src/adapters/recovery_stage_discard_storage_error.rs b/src/adapters/recovery/recovery_stage_discard_storage_error.rs similarity index 100% rename from src/adapters/recovery_stage_discard_storage_error.rs rename to src/adapters/recovery/recovery_stage_discard_storage_error.rs diff --git a/src/adapters/recovery_stage_evidence.rs b/src/adapters/recovery/recovery_stage_evidence.rs similarity index 96% rename from src/adapters/recovery_stage_evidence.rs rename to src/adapters/recovery/recovery_stage_evidence.rs index 15bc1d8..b68d37f 100644 --- a/src/adapters/recovery_stage_evidence.rs +++ b/src/adapters/recovery/recovery_stage_evidence.rs @@ -12,7 +12,7 @@ pub struct RecoveryStageEvidence { } impl RecoveryStageEvidence { - pub(super) const fn new( + pub(in crate::adapters) const fn new( stage: RecoveryStage, length: RecoveryStageLength, fingerprint: RecoveryStageFingerprint, diff --git a/src/adapters/recovery_stage_fingerprint.rs b/src/adapters/recovery/recovery_stage_fingerprint.rs similarity index 89% rename from src/adapters/recovery_stage_fingerprint.rs rename to src/adapters/recovery/recovery_stage_fingerprint.rs index dcbbd85..f1f393f 100644 --- a/src/adapters/recovery_stage_fingerprint.rs +++ b/src/adapters/recovery/recovery_stage_fingerprint.rs @@ -8,7 +8,7 @@ use super::RecoveryStageFingerprintAlgorithm; pub struct RecoveryStageFingerprint([u8; 32]); impl RecoveryStageFingerprint { - pub(super) const fn from_validated(bytes: [u8; 32]) -> Self { + pub(in crate::adapters) const fn from_validated(bytes: [u8; 32]) -> Self { Self(bytes) } diff --git a/src/adapters/recovery_stage_fingerprint_algorithm.rs b/src/adapters/recovery/recovery_stage_fingerprint_algorithm.rs similarity index 100% rename from src/adapters/recovery_stage_fingerprint_algorithm.rs rename to src/adapters/recovery/recovery_stage_fingerprint_algorithm.rs diff --git a/src/adapters/recovery_stage_fingerprint_error.rs b/src/adapters/recovery/recovery_stage_fingerprint_error.rs similarity index 100% rename from src/adapters/recovery_stage_fingerprint_error.rs rename to src/adapters/recovery/recovery_stage_fingerprint_error.rs diff --git a/src/adapters/recovery_stage_fingerprinter.rs b/src/adapters/recovery/recovery_stage_fingerprinter.rs similarity index 100% rename from src/adapters/recovery_stage_fingerprinter.rs rename to src/adapters/recovery/recovery_stage_fingerprinter.rs diff --git a/src/adapters/recovery_stage_length.rs b/src/adapters/recovery/recovery_stage_length.rs similarity index 84% rename from src/adapters/recovery_stage_length.rs rename to src/adapters/recovery/recovery_stage_length.rs index 7aafa82..b4cc5d8 100644 --- a/src/adapters/recovery_stage_length.rs +++ b/src/adapters/recovery/recovery_stage_length.rs @@ -6,7 +6,7 @@ pub struct RecoveryStageLength(u64); impl RecoveryStageLength { - pub(super) const fn from_validated(value: u64) -> Self { + pub(in crate::adapters) const fn from_validated(value: u64) -> Self { Self(value) } diff --git a/src/adapters/recovery_stage_metadata.rs b/src/adapters/recovery/recovery_stage_metadata.rs similarity index 100% rename from src/adapters/recovery_stage_metadata.rs rename to src/adapters/recovery/recovery_stage_metadata.rs diff --git a/src/adapters/recovery_stage_metadata_error.rs b/src/adapters/recovery/recovery_stage_metadata_error.rs similarity index 100% rename from src/adapters/recovery_stage_metadata_error.rs rename to src/adapters/recovery/recovery_stage_metadata_error.rs diff --git a/src/adapters/recovery_stage_parent.rs b/src/adapters/recovery/recovery_stage_parent.rs similarity index 100% rename from src/adapters/recovery_stage_parent.rs rename to src/adapters/recovery/recovery_stage_parent.rs diff --git a/src/adapters/recovery_stage_pool_outcome.rs b/src/adapters/recovery/recovery_stage_pool_outcome.rs similarity index 100% rename from src/adapters/recovery_stage_pool_outcome.rs rename to src/adapters/recovery/recovery_stage_pool_outcome.rs diff --git a/src/adapters/recovery_stage_synchronization_outcome.rs b/src/adapters/recovery/recovery_stage_synchronization_outcome.rs similarity index 100% rename from src/adapters/recovery_stage_synchronization_outcome.rs rename to src/adapters/recovery/recovery_stage_synchronization_outcome.rs diff --git a/src/adapters/staged_segment.rs b/src/adapters/staged_segment.rs index f1ab83e..4bfaadf 100644 --- a/src/adapters/staged_segment.rs +++ b/src/adapters/staged_segment.rs @@ -2,7 +2,7 @@ use std::collections::HashSet; -use super::recovery_segment_resume_state::RecoverySegmentResumeState; +use super::recovery::recovery_segment_resume_state::RecoverySegmentResumeState; use super::segment_digest_builder::SegmentDigestBuilder; use super::{ AdmittedSegmentRecord, RecoverySegmentResumeRequest, SealedSegment, SegmentDurabilityPhase, diff --git a/src/adapters/store_migration/filesystem_inventory_catalogs.rs b/src/adapters/store_migration/filesystem_inventory_catalogs.rs index 147a608..ebed9d8 100644 --- a/src/adapters/store_migration/filesystem_inventory_catalogs.rs +++ b/src/adapters/store_migration/filesystem_inventory_catalogs.rs @@ -16,9 +16,10 @@ use super::filesystem_inventory_names; use super::filesystem_inventory_segments::FilesystemMigrationSegmentInventory; use super::migration_catalog_admission::{self, MigrationSegmentLoadError}; use crate::CatalogLength; +use crate::adapters::recovery::recovery_pool_name; use crate::adapters::{ CatalogRestartArtifact, CatalogRestartError, CatalogRestartPhase, ChecksummedCatalog, - RecoveryEntryName, SegmentDigest, SegmentReadPolicy, physical_pool_name, recovery_pool_name, + RecoveryEntryName, SegmentDigest, SegmentReadPolicy, physical_pool_name, }; const POOL: MigrationInventoryPool = MigrationInventoryPool::Catalogs; diff --git a/src/adapters/store_migration/filesystem_inventory_segments.rs b/src/adapters/store_migration/filesystem_inventory_segments.rs index 1964c22..9aa7722 100644 --- a/src/adapters/store_migration/filesystem_inventory_segments.rs +++ b/src/adapters/store_migration/filesystem_inventory_segments.rs @@ -12,10 +12,11 @@ use super::filesystem_inventory_file::{ self, FilesystemInventoryFileError, FilesystemInventoryFilePolicy, }; use super::filesystem_inventory_names; +use crate::adapters::recovery::recovery_pool_name; use crate::adapters::segment_header::MAXIMUM_SEGMENT_LENGTH; use crate::adapters::{ AdmittedSegment, CatalogRestartArtifact, CatalogRestartError, CatalogRestartPhase, - RecoveryEntryName, SegmentDigest, SegmentReadPolicy, physical_pool_name, recovery_pool_name, + RecoveryEntryName, SegmentDigest, SegmentReadPolicy, physical_pool_name, }; const POOL: MigrationInventoryPool = MigrationInventoryPool::Segments; diff --git a/tests/adapters_layout_contract.rs b/tests/adapters_layout_contract.rs new file mode 100644 index 0000000..3fda099 --- /dev/null +++ b/tests/adapters_layout_contract.rs @@ -0,0 +1,37 @@ +//! The adapters module root stays a scannable manifest, not a 500-line ceiling risk. + +const ADAPTERS_ROOT: &str = include_str!("../src/adapters/mod.rs"); + +/// `docs/Rust Standards.md` reviews any file above 300 lines and refuses any +/// above 500; the adapters root sat at 498 before its re-export surface moved. +#[test] +fn adapters_root_stays_under_the_review_threshold() { + let lines = ADAPTERS_ROOT.lines().count(); + assert!( + lines < 300, + "src/adapters/mod.rs has {lines} lines; the review threshold is 300" + ); +} + +/// The root declares modules and re-exports one surface; item definitions +/// belong in their own files. +#[test] +fn adapters_root_declares_modules_and_reexports_only() { + for line in ADAPTERS_ROOT.lines() { + let trimmed = line.trim_start(); + let allowed = trimmed.is_empty() + || trimmed.starts_with("//!") + || trimmed.starts_with("//") + || trimmed.starts_with('#') + || trimmed.starts_with("mod ") + || trimmed.starts_with("pub mod ") + || trimmed.starts_with("pub(super) mod ") + || trimmed.starts_with("pub use ") + || trimmed.starts_with("use ") + || trimmed.starts_with("pub(super) use ") + || trimmed == "};" + || trimmed.ends_with(',') + || trimmed.ends_with("::{"); + assert!(allowed, "unexpected item in src/adapters/mod.rs: {line}"); + } +} From 06805c70036f75cd18113f44c1dfa409fd324111 Mon Sep 17 00:00:00 2001 From: James Ross Date: Mon, 7 Sep 2026 10:16:57 -0700 Subject: [PATCH 071/111] Fix: repair the broken intra-doc link and build documentation in CI cargo doc --no-deps failed at HEAD: filesystem_migration_authority.rs linked [`FilesystemMigrationAuthorityError::IntentChanged`] while the type is in scope only under the alias `Error`, and #![deny(warnings)] turns the rustdoc broken-link lint into a hard error. No CI job ran cargo doc, so the branch stayed green while the crate could not be documented. The link now names [`Error::IntentChanged`]. The Rust quality gates gain a "Build documentation" step running `cargo doc --workspace --no-deps --locked` after the doctests, and the local gate chain for this pass runs the same command, so the doc build is the regression test for this class. Self-review finding A1 (P1). Refs #78 --- .github/workflows/ci.yml | 3 +++ CHANGELOG.md | 4 ++++ .../store_migration/filesystem_migration_authority.rs | 2 +- 3 files changed, 8 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3135ee7..3aaf351 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -69,6 +69,9 @@ jobs: - name: Test documentation run: cargo test --workspace --doc --locked + - name: Build documentation + run: cargo doc --workspace --no-deps --locked + - name: Check MSRV contract run: cargo +1.96.0 check --workspace --all-targets --all-features --locked diff --git a/CHANGELOG.md b/CHANGELOG.md index 32ba343..127a732 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,10 @@ after its public API and format compatibility policies are established. ### Changed +- The Rust quality gates now build the crate documentation with + `cargo doc --workspace --no-deps --locked`, so a broken intra-doc link under + `#![deny(warnings)]` fails CI instead of only failing anyone who documents + the crate locally. - The adapters module root now declares modules only: its public re-export surface lives in `adapters/exports.rs` and the recovery-stage adapters live under `adapters/recovery/` behind one facade. No public path changed; the diff --git a/src/adapters/store_migration/filesystem_migration_authority.rs b/src/adapters/store_migration/filesystem_migration_authority.rs index 4ddb05a..f6ca8d3 100644 --- a/src/adapters/store_migration/filesystem_migration_authority.rs +++ b/src/adapters/store_migration/filesystem_migration_authority.rs @@ -109,7 +109,7 @@ impl FilesystemStoreMigrationAuthority { /// # Errors /// /// Returns the exact observation refusal or - /// [`FilesystemMigrationAuthorityError::IntentChanged`] with both intent + /// [`Error::IntentChanged`] with both intent /// digests when current authority no longer reproduces `expected`. pub fn verify_current(&self, expected: &CanonicalStoreMigrationIntent) -> Result<(), Error> { let observed = self.observe_intent()?; From 0c0eea146870f4619983390c0947e080268c110f Mon Sep 17 00:00:00 2001 From: James Ross Date: Mon, 7 Sep 2026 10:22:56 -0700 Subject: [PATCH 072/111] Fix: admit the version-one root namespace before recovery pins any pool FilesystemRecoveryInventoryReader::from_root, which every version-one recovery adapter (discard, completion, resume, next-head finalization) passes through, pinned staging, segments, and catalogs without looking at the root namespace. admit_published had one production caller, the migration authority. A migrated version-two root, whose FORMAT, migration records, reader fence, and retention, gc, and recovery directories version-one admission refuses, therefore received recovery authority and could rewrite its pools. admit_recoverable requires every root entry to be one of the six lawful version-one root names: writer.lock, staging, segments, catalogs, HEAD, and head.next. Every entry is optional, because recovery lawfully opens a store that crashed before first publication or mid-publication; entry kinds stay with the pinning and classification code that already reports the exact namespace and operation. from_root runs the admission first and maps refusal to RecoveryInventoryError at Root / OpenNamespace, which the discarder surfaces as its Namespace variant. Regression laws: the inventory reader and the stage discarder each refuse a completely migrated root through their from_root path, on every platform. Self-review finding A3 (P1). Refs #78 --- CHANGELOG.md | 5 ++ docs/formats/segment-store-v2/recovery.md | 2 +- .../filesystem_initialization_namespace.rs | 22 +++++++++ .../filesystem_recovery_inventory_reader.rs | 14 ++++-- src/adapters/retention.rs | 2 + .../filesystem_recovery_admission_tests.rs | 46 +++++++++++++++++++ 6 files changed, 87 insertions(+), 4 deletions(-) create mode 100644 src/adapters/retention/filesystem_recovery_admission_tests.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index 127a732..0309b22 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,11 @@ after its public API and format compatibility policies are established. ### Changed +- Version-one recovery adapters admit the root namespace before pinning any + pool: every root entry must be one of `writer.lock`, `staging`, `segments`, + `catalogs`, `HEAD`, or `head.next`, so recovery discard, completion, resume, + and finalization refuse a migrated version-two root instead of rewriting its + version-one pools. - The Rust quality gates now build the crate documentation with `cargo doc --workspace --no-deps --locked`, so a broken intra-doc link under `#![deny(warnings)]` fails CI instead of only failing anyone who documents diff --git a/docs/formats/segment-store-v2/recovery.md b/docs/formats/segment-store-v2/recovery.md index 0b48e83..958dd58 100644 --- a/docs/formats/segment-store-v2/recovery.md +++ b/docs/formats/segment-store-v2/recovery.md @@ -61,7 +61,7 @@ corpus `definition.tsv` bytes. The format-marker digest is BLAKE3-256 of `StoreMigrationStorage` names all 21 durability capabilities; `execute_store_migration` verifies current authority first and returns only after final synchronization. `FilesystemStoreMigrationInventoryReader` inventories version-1 bytes under retained writer authority. The fresh writer executes once; partial-prefix -recovery is absent and version-1 reopen refuses a migrated root. +recovery is absent; version-1 reopen and recovery both refuse a migrated root. ## Reader fence diff --git a/src/adapters/filesystem_initialization_namespace.rs b/src/adapters/filesystem_initialization_namespace.rs index 2f927ac..054f9f7 100644 --- a/src/adapters/filesystem_initialization_namespace.rs +++ b/src/adapters/filesystem_initialization_namespace.rs @@ -18,6 +18,15 @@ const PUBLISHED_NAMES: [&str; 5] = [ CATALOGS_NAME, HEAD_NAME, ]; +const NEXT_HEAD_NAME: &str = "head.next"; +const RECOVERABLE_NAMES: [&str; 6] = [ + LOCK_NAME, + STAGING_NAME, + SEGMENTS_NAME, + CATALOGS_NAME, + HEAD_NAME, + NEXT_HEAD_NAME, +]; const READER_LOCK_NAME: &str = "reader.lock"; const MARKER_NAME: &str = "FORMAT"; const INTENT_NAME: &str = "migration.intent"; @@ -57,6 +66,19 @@ pub(super) fn admit_published(directory: &Dir) -> io::Result<()> { admit_membership(directory, &PUBLISHED_NAMES) } +/// Admits a version-1 root at any recovery-lawful point of its lifecycle. +/// +/// Recovery may open a store that crashed before its first publication or +/// mid-publication, so every entry is optional and `head.next` may be present. +/// What is not optional is that every present entry be one of the six +/// version-1 root names: any version-2 or foreign entry means this is not a +/// version-1 root and version-1 recovery must refuse before touching a pool. +/// Entry kinds are verified by the recovery pinning and classification code, +/// which reports the exact namespace and operation. +pub(super) fn admit_recoverable(directory: &Dir) -> io::Result<()> { + admit_membership(directory, &RECOVERABLE_NAMES) +} + /// Admits the exact completely migrated version-2 root namespace. /// /// Every version-1 published entry, the persistent reader fence, the format diff --git a/src/adapters/filesystem_recovery_inventory_reader.rs b/src/adapters/filesystem_recovery_inventory_reader.rs index f269125..392522f 100644 --- a/src/adapters/filesystem_recovery_inventory_reader.rs +++ b/src/adapters/filesystem_recovery_inventory_reader.rs @@ -11,9 +11,10 @@ use super::{ FilesystemRecoveryStageError, RecoveryEntryName, RecoveryInventory, RecoveryInventoryError, RecoveryInventoryLimit, RecoveryInventoryOperation, RecoveryInventoryStorage, RecoveryNamespace, RecoveryStage, RecoveryStageCompletionPool, RecoveryStageEvidence, - RecoveryStageNamespacePhase, RecoveryStageParent, filesystem_platform_profile, - filesystem_recovery_inventory_scan, filesystem_recovery_namespace::PinnedRecoveryDirectory, - filesystem_recovery_stage, read_recovery_inventory, + RecoveryStageNamespacePhase, RecoveryStageParent, filesystem_initialization_namespace, + filesystem_platform_profile, filesystem_recovery_inventory_scan, + filesystem_recovery_namespace::PinnedRecoveryDirectory, filesystem_recovery_stage, + read_recovery_inventory, }; const STAGING_NAME: &str = "staging"; @@ -70,6 +71,13 @@ impl FilesystemRecoveryInventoryReader { } pub(super) fn from_root(root: Dir) -> Result { + filesystem_initialization_namespace::admit_recoverable(&root).map_err(|source| { + RecoveryInventoryError::io( + RecoveryNamespace::Root, + RecoveryInventoryOperation::OpenNamespace, + source, + ) + })?; let staging = PinnedRecoveryDirectory::open(&root, RecoveryNamespace::Staging, STAGING_NAME)?; let segments = diff --git a/src/adapters/retention.rs b/src/adapters/retention.rs index 4dc84eb..2fd8ded 100644 --- a/src/adapters/retention.rs +++ b/src/adapters/retention.rs @@ -13,6 +13,8 @@ mod closure_error_display; mod closure_member; mod closure_profile_error; mod closure_verifier; +#[cfg(test)] +mod filesystem_recovery_admission_tests; mod filesystem_retention_authority; mod filesystem_retention_authority_error; #[cfg(test)] diff --git a/src/adapters/retention/filesystem_recovery_admission_tests.rs b/src/adapters/retention/filesystem_recovery_admission_tests.rs new file mode 100644 index 0000000..e433a55 --- /dev/null +++ b/src/adapters/retention/filesystem_recovery_admission_tests.rs @@ -0,0 +1,46 @@ +//! Version-one recovery adapters refuse a migrated version-two root. + +use std::error::Error; + +use super::filesystem_retention_test_fixture::migrated_store; +use crate::adapters::{ + FilesystemRecoveryInventoryReader, FilesystemRecoveryStageDiscardOpenError, + FilesystemRecoveryStageDiscarder, RecoveryInventoryError, RecoveryInventoryOperation, + RecoveryNamespace, +}; + +#[test] +fn recovery_inventory_reader_refuses_a_migrated_root() -> Result<(), Box> { + let sandbox = migrated_store("recovery-admission-inventory-migrated")?; + + let error = FilesystemRecoveryInventoryReader::open_unchecked_for_tests(sandbox.path()) + .err() + .ok_or("version-one recovery inventory opened a migrated root")?; + + assert!(matches!( + error, + RecoveryInventoryError::Io { + namespace: RecoveryNamespace::Root, + operation: RecoveryInventoryOperation::OpenNamespace, + .. + } + )); + sandbox.remove()?; + Ok(()) +} + +#[test] +fn recovery_stage_discarder_refuses_a_migrated_root() -> Result<(), Box> { + let sandbox = migrated_store("recovery-admission-discarder-migrated")?; + + let error = FilesystemRecoveryStageDiscarder::open_unchecked_for_tests(sandbox.path()) + .err() + .ok_or("version-one recovery discarder opened a migrated root")?; + + assert!(matches!( + error, + FilesystemRecoveryStageDiscardOpenError::Namespace { .. } + )); + sandbox.remove()?; + Ok(()) +} From 89cdb4083b3e6a9c6808784e70063d53e4442f78 Mon Sep 17 00:00:00 2001 From: James Ross Date: Mon, 7 Sep 2026 10:29:22 -0700 Subject: [PATCH 073/111] Fix: refuse version-two residue in recovery instead of exact root membership 0c0eea14 admitted the version-one root by exact membership over six names. That refused the link-target directories the symbolic-link laws plant beside the protocol names, and it duplicated work the recovery inventory already does: recovery name classification refuses every unknown root entry. Seven laws failed; the gate chain that landed the commit relied on shell errexit, which this tool's shell does not honor, so the failure was not seen. admit_recoverable now refuses exactly what version-one classification cannot express, a root that has left version one: FORMAT and its stage, reader.lock, both migration records and their stages, and the retention, gc, and recovery directories. Everything else stays with classification and pinning, which report the exact namespace and operation. Both admission laws and all seven symbolic-link laws pass. Every gate in this pass now runs through an explicit-exit script that fails on the first red command; the suite, doctests, doc build, clippy under both feature sets, documentation and source-structure checks, conformance, the golden worldline, and the crash matrix are green at this commit. Self-review finding A3 (P1), correction. Refs #78 --- CHANGELOG.md | 11 ++--- .../filesystem_initialization_namespace.rs | 43 +++++++++++-------- 2 files changed, 32 insertions(+), 22 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0309b22..6e1eb9f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,11 +10,12 @@ after its public API and format compatibility policies are established. ### Changed -- Version-one recovery adapters admit the root namespace before pinning any - pool: every root entry must be one of `writer.lock`, `staging`, `segments`, - `catalogs`, `HEAD`, or `head.next`, so recovery discard, completion, resume, - and finalization refuse a migrated version-two root instead of rewriting its - version-one pools. +- Version-one recovery adapters refuse version-two residue at the store root + before pinning any pool: a format marker, reader fence, migration record or + stage, or a `retention`, `gc`, or `recovery` directory means recovery + discard, completion, resume, and finalization refuse instead of rewriting a + migrated store's version-one pools. Unknown entries continue to be refused + by recovery name classification. - The Rust quality gates now build the crate documentation with `cargo doc --workspace --no-deps --locked`, so a broken intra-doc link under `#![deny(warnings)]` fails CI instead of only failing anyone who documents diff --git a/src/adapters/filesystem_initialization_namespace.rs b/src/adapters/filesystem_initialization_namespace.rs index 054f9f7..29d2dbe 100644 --- a/src/adapters/filesystem_initialization_namespace.rs +++ b/src/adapters/filesystem_initialization_namespace.rs @@ -18,14 +18,17 @@ const PUBLISHED_NAMES: [&str; 5] = [ CATALOGS_NAME, HEAD_NAME, ]; -const NEXT_HEAD_NAME: &str = "head.next"; -const RECOVERABLE_NAMES: [&str; 6] = [ - LOCK_NAME, - STAGING_NAME, - SEGMENTS_NAME, - CATALOGS_NAME, - HEAD_NAME, - NEXT_HEAD_NAME, +const VERSION_TWO_MARKERS: [&str; 10] = [ + "reader.lock", + "FORMAT", + "FORMAT.next", + "migration.intent", + "migration.intent.next", + "migration.receipt", + "migration.receipt.next", + "retention", + "gc", + "recovery", ]; const READER_LOCK_NAME: &str = "reader.lock"; const MARKER_NAME: &str = "FORMAT"; @@ -66,17 +69,23 @@ pub(super) fn admit_published(directory: &Dir) -> io::Result<()> { admit_membership(directory, &PUBLISHED_NAMES) } -/// Admits a version-1 root at any recovery-lawful point of its lifecycle. +/// Refuses version-two residue before version-one recovery touches a pool. /// -/// Recovery may open a store that crashed before its first publication or -/// mid-publication, so every entry is optional and `head.next` may be present. -/// What is not optional is that every present entry be one of the six -/// version-1 root names: any version-2 or foreign entry means this is not a -/// version-1 root and version-1 recovery must refuse before touching a pool. -/// Entry kinds are verified by the recovery pinning and classification code, -/// which reports the exact namespace and operation. +/// Recovery may open a store at any lawful point of its version-one lifecycle, +/// including before first publication and with `head.next` retained, and the +/// recovery inventory already classifies every unknown root entry as +/// unexpected. What that classification cannot express is that a root has left +/// version one entirely: a format marker, reader fence, migration record or +/// stage, or a `retention`, `gc`, or `recovery` directory means version-one +/// recovery must refuse before pinning anything. pub(super) fn admit_recoverable(directory: &Dir) -> io::Result<()> { - admit_membership(directory, &RECOVERABLE_NAMES) + for entry in directory.entries()? { + let name = entry?.file_name(); + if is_canonical(&name, &VERSION_TWO_MARKERS) { + return Err(ambiguous_namespace()); + } + } + Ok(()) } /// Admits the exact completely migrated version-2 root namespace. From cc23b339cc418fe2ef9e95ad8450fe39974edc1a Mon Sep 17 00:00:00 2001 From: James Ross Date: Mon, 7 Sep 2026 10:37:57 -0700 Subject: [PATCH 074/111] Fix: bind the catalog head on the already-committed path too MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit require_current_catalog ran only when the disposition was Publish. The publication receipt copies the closure's catalog generation and digest for both dispositions, so an already-committed retry whose closure had been verified against another store's CatalogSnapshot returned a receipt citing a catalog this store's HEAD does not name — exactly what filesystem_retention_catalog.rs says must not happen. The binding now runs immediately after the disposition is decided and before either branch, so AlreadyCommitted and Publish both require this store's HEAD to name the closure's catalog. Regression law: publish generation one, replace the catalog HEAD with a same-generation head naming a different catalog, resubmit the identical preparation, and require CatalogDisagreed with an unchanged retention witness. Self-review finding A4 (P2). Refs #78 --- CHANGELOG.md | 4 +++ .../filesystem_retention_catalog_tests.rs | 30 +++++++++++++++++++ .../retention/filesystem_retention_storage.rs | 2 +- 3 files changed, 35 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6e1eb9f..19cd040 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,10 @@ after its public API and format compatibility policies are established. ### Changed +- Retention current-state verification binds this store's catalog `HEAD` to + the closure's catalog coordinates for every disposition, so an + already-committed retry no longer returns a receipt citing a catalog this + store does not name. - Version-one recovery adapters refuse version-two residue at the store root before pinning any pool: a format marker, reader fence, migration record or stage, or a `retention`, `gc`, or `recovery` directory means recovery diff --git a/src/adapters/retention/filesystem_retention_catalog_tests.rs b/src/adapters/retention/filesystem_retention_catalog_tests.rs index 2eff165..7c4f26f 100644 --- a/src/adapters/retention/filesystem_retention_catalog_tests.rs +++ b/src/adapters/retention/filesystem_retention_catalog_tests.rs @@ -8,6 +8,7 @@ use super::filesystem_retention_test_fixture::{ ROOT_HEX, fixture, initial_preparation, open_authority, refusal, retention_witness, }; use super::{RetentionCurrentStateRefusal, RetentionPublicationStorage}; +use crate::execute_retention_publication; /// A generation-one version-one head that names a different catalog digest. const FOREIGN_CATALOG_HEAD_HEX: &str = @@ -38,3 +39,32 @@ fn closure_verified_against_another_catalog_refuses_before_staging() -> Result<( sandbox.remove()?; Ok(()) } + +#[test] +fn committed_retry_over_a_foreign_catalog_refuses_before_reporting_committed() +-> Result<(), Box> { + let (sandbox, mut authority) = + open_authority("filesystem-retention-committed-foreign-catalog")?; + let root_bytes = fixture(ROOT_HEX)?; + let _published = + execute_retention_publication(&mut authority, &initial_preparation(&root_bytes)?)?; + fs::write( + sandbox.path().join("HEAD"), + fixture(FOREIGN_CATALOG_HEAD_HEX)?, + )?; + let before = retention_witness(sandbox.path())?; + let retry = initial_preparation(&root_bytes)?; + + let error = RetentionPublicationStorage::verify_current(&mut authority, &retry) + .err() + .ok_or("already-committed retry was reported over a foreign catalog head")?; + + assert!(matches!( + refusal(&error), + Some(RetentionCurrentStateRefusal::CatalogDisagreed { .. }) + )); + assert_eq!(retention_witness(sandbox.path())?, before); + drop(authority); + sandbox.remove()?; + Ok(()) +} diff --git a/src/adapters/retention/filesystem_retention_storage.rs b/src/adapters/retention/filesystem_retention_storage.rs index 05d85b1..dc61a13 100644 --- a/src/adapters/retention/filesystem_retention_storage.rs +++ b/src/adapters/retention/filesystem_retention_storage.rs @@ -32,8 +32,8 @@ impl RetentionPublicationStorage for FilesystemRetentionPublicationAuthority { return Err(RetentionCurrentStateRefusal::HeadAbsentWithArtifacts.into_io()); } let disposition = filesystem_retention_current::disposition(preparation, current.as_ref())?; + filesystem_retention_catalog::require_current_catalog(&self.root, preparation)?; if disposition == RetentionTransitionDisposition::Publish { - filesystem_retention_catalog::require_current_catalog(&self.root, preparation)?; filesystem_retention_namespace::admit_expectation( &self.roots, preparation.candidate(), From 5cc1553f652145b5ecf7dafbf462dbb7be97ef3c Mon Sep 17 00:00:00 2001 From: James Ross Date: Mon, 7 Sep 2026 10:44:37 -0700 Subject: [PATCH 075/111] Fix: reopen the predecessor root before publishing a successor MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit For a Current(_) expectation the Publish path checked only that the namespace directory existed; nothing reopened the predecessor root file the candidate names, so a successor could be committed over a namespace whose predecessor root had been lost or corrupted, leaving a durable chain no reader could reopen and contradicting admit_expectation's own promise that an unavailable predecessor refuses. verify_predecessor requires the observed manifest to select the candidate's namespace, the candidate to name that selection as its predecessor, and the selection's root pool entry to reopen — bounded by the root format's maximum encoded length, derived from the typed header, namespace, anchor-count, and trailer limits and pinned by a law — and decode to exactly the manifest's generation and digest. Refusals are PredecessorMismatch, PredecessorRootAbsent, and PredecessorRootChanged. It runs on the Publish path after the expectation and capacity checks. Regression laws: an absent generation-one root file and a generation-one root with one flipped byte each refuse a generation-two successor with an unchanged retention witness. Self-review finding A5 (P2). Refs #78 --- CHANGELOG.md | 4 + docs/formats/segment-store-v2/requirements.md | 2 +- .../retention/filesystem_retention_current.rs | 52 +++++++++++++ .../filesystem_retention_expectation_tests.rs | 73 ++++++++++++++++++- .../retention/filesystem_retention_refusal.rs | 9 +++ .../retention/filesystem_retention_storage.rs | 10 +++ src/adapters/retention/root_header_decoder.rs | 6 ++ 7 files changed, 154 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 19cd040..cdef263 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,10 @@ after its public API and format compatibility policies are established. ### Changed +- A successor retention publication now reopens the predecessor root the + current manifest selects, bounded by the root format's maximum encoded + length, and requires it to decode to exactly that generation and digest; a + namespace directory alone no longer stands in for an available predecessor. - Retention current-state verification binds this store's catalog `HEAD` to the closure's catalog coordinates for every disposition, so an already-committed retry no longer returns a receipt citing a catalog this diff --git a/docs/formats/segment-store-v2/requirements.md b/docs/formats/segment-store-v2/requirements.md index b81d7a2..15ca92b 100644 --- a/docs/formats/segment-store-v2/requirements.md +++ b/docs/formats/segment-store-v2/requirements.md @@ -12,7 +12,7 @@ case is not evidence. | `KEEP-RETENTION-001` | `RetentionNamespace`, `RootGeneration`, `LivenessGeneration`, profile coordinates, limits, anchors, and digests are validated typed values | `tests/retention_values.rs`, `tests/retention_root_encoding.rs`, and typed verified anchor-set evidence in `tests/retention_root_decoding.rs` | Implemented | | `KEEP-RETENTION-002` | Root, manifest, and head codecs implement the exact canonical grammars and fixed bounds | independent golden corpus plus `tests/retention_root_encoding.rs`, `tests/retention_root_decoding.rs`, `tests/retention_manifest_codec.rs`, and `tests/retention_head_codec.rs` | Implemented | | `KEEP-RETENTION-003` | Every structural field, truncation boundary, ordering law, duplicate, overflow, flag, reserved byte, digest, checksum, and trailing byte has a precise refusal | seeded `retention_format` fuzz target plus the root, manifest, and head corruption matrix; mutation coverage remains | In progress in #19 | -| `KEEP-RETENTION-004` | Retain and release compare expected and observed generations and publish exact successors only | unforgeable readiness and preflight proofs in `tests/retention_transition.rs` and `tests/retention_preflight.rs`; exact successor preparation and complete receipt evidence in `tests/retention_publication_preparation.rs` and `tests/retention_publication_execution.rs`; writer-locked initial filesystem publication in `filesystem_retention_storage_tests`; observed-head successor publication, exact predecessor binding, and absent-head refusal in `filesystem_retention_successor_tests`; the store's catalog head must name the closure's catalog generation and digest before any forward write in `filesystem_retention_catalog_tests`; a head whose predecessor disagrees with its manifest refuses in `filesystem_retention_current_tests` | Implemented | +| `KEEP-RETENTION-004` | Retain and release compare expected and observed generations and publish exact successors only | unforgeable readiness and preflight proofs in `tests/retention_transition.rs` and `tests/retention_preflight.rs`; exact successor preparation and complete receipt evidence in `tests/retention_publication_preparation.rs` and `tests/retention_publication_execution.rs`; writer-locked initial filesystem publication in `filesystem_retention_storage_tests`; observed-head successor publication, exact predecessor binding, and absent-head refusal in `filesystem_retention_successor_tests`; the store's catalog head must name the closure's catalog generation and digest before any forward write in `filesystem_retention_catalog_tests`; a head whose predecessor disagrees with its manifest refuses in `filesystem_retention_current_tests`; a successor reopens and decodes the manifest-selected predecessor root and refuses an absent or changed one in `filesystem_retention_expectation_tests` | Implemented | | `KEEP-RETENTION-005` | Closure derivation is deterministic, bounded, cycle-safe, fail-closed, and verifies complete blob reconstruction | exact accounting, reconstruction, adversarial-catalog, and exhaustive model laws in `tests/retention_closure.rs`; corrupt members refuse through the inherited segment-record admission laws and seeded `segment_format` fuzz target routed by `closure-corruption.md` | Implemented | | `KEEP-RETENTION-006` | Publication follows the exact ordered durability protocol, including new namespace-directory admission and retention of fixed-stage evidence until head commit, and returns only after cleanup synchronization | typed vocabulary and blocking port in `tests/retention_publication_phase.rs` and `tests/retention_publication_storage.rs`; ordered execution, conditional namespace sync, and all 17 exact storage-fault boundaries in `tests/retention_publication_execution.rs`; production 17-phase forward filesystem execution, exclusive staging, byte-equal inode-substitution refusal, and retained-stage recovery refusal in `filesystem_retention_storage_tests`; orphan namespace directories count against the 4,096 ceiling and refuse a new namespace before any stage is written in `filesystem_retention_capacity_tests`; crash injection remains | In progress in #19 | | `KEEP-RETENTION-007` | Restart resolves every fixed-stage crash prefix to one documented lawful state or typed ambiguity | recovery-required refusals before any mutation in `filesystem_retention_expectation_tests`: an absent head over populated pools, a non-initial head prepared against an absent head, an orphan directory for a namespace expected absent, and an absent directory for a namespace expected current; debug and release crash matrix remains | In progress in #19 | diff --git a/src/adapters/retention/filesystem_retention_current.rs b/src/adapters/retention/filesystem_retention_current.rs index 78dccdc..f349cd9 100644 --- a/src/adapters/retention/filesystem_retention_current.rs +++ b/src/adapters/retention/filesystem_retention_current.rs @@ -6,6 +6,7 @@ use cap_fs_ext::{DirExt, FollowSymlinks, OpenOptionsFollowExt, OpenOptionsSyncEx use cap_std::fs::{Dir, OpenOptions}; use super::filesystem_retention_pool_name as pool_name; +use super::root_header_decoder; use super::{ AdmittedRetentionManifest, AdmittedRetentionRoot, ChecksummedRetentionHead, RetentionCurrentStateRefusal, RetentionPublicationPreparation, RetentionTransitionDisposition, @@ -161,6 +162,57 @@ pub(super) fn verify_committed( } } +/// Reopens the predecessor root a `Current(_)` successor claims to advance. +/// +/// The observed manifest must select the candidate's namespace, the candidate +/// must name that selection as its predecessor, and the selection's root pool +/// entry must reopen and decode to exactly that generation and digest. A +/// namespace directory alone is not proof the predecessor is available. +pub(super) fn verify_predecessor( + roots: &Dir, + current: &ObservedRetentionState, + candidate: &AdmittedRetentionRoot<'_>, +) -> io::Result<()> { + let manifest = AdmittedRetentionManifest::decode(current.manifest_bytes()) + .map_err(|source| RetentionCurrentStateRefusal::ManifestRefused { source }.into_io())?; + let namespace = candidate.root().namespace().digest(); + let entries = manifest.manifest().entries(); + let entry = entries + .binary_search_by_key(&namespace, |entry| entry.namespace()) + .ok() + .and_then(|index| entries.get(index).copied()) + .ok_or_else(|| RetentionCurrentStateRefusal::CommittedSelectionMissing.into_io())?; + if candidate.root().predecessor() != Some(entry.root_digest()) { + return Err(RetentionCurrentStateRefusal::PredecessorMismatch.into_io()); + } + let directory = roots + .open_dir_nofollow(pool_name::namespace(namespace)) + .map_err(|_source| RetentionCurrentStateRefusal::CommittedNamespaceUnavailable.into_io())?; + let name = pool_name::root(entry.root_generation(), entry.root_digest()); + let length = match directory.symlink_metadata(&name) { + Ok(metadata) => usize::try_from(metadata.len()) + .map_err(|_source| RetentionCurrentStateRefusal::RecordLengthOverflow.into_io())?, + Err(source) if source.kind() == io::ErrorKind::NotFound => { + return Err(RetentionCurrentStateRefusal::PredecessorRootAbsent.into_io()); + } + Err(source) => return Err(source), + }; + if length > root_header_decoder::MAXIMUM_ENCODED_LENGTH { + return Err(RetentionCurrentStateRefusal::PredecessorRootAbsent.into_io()); + } + let bytes = read_exact_optional(&directory, &name, length)? + .ok_or_else(|| RetentionCurrentStateRefusal::PredecessorRootAbsent.into_io())?; + let predecessor = AdmittedRetentionRoot::decode(&bytes) + .map_err(|_source| RetentionCurrentStateRefusal::PredecessorRootChanged.into_io())?; + if predecessor.digest() == entry.root_digest() + && predecessor.root().generation() == entry.root_generation() + { + Ok(()) + } else { + Err(RetentionCurrentStateRefusal::PredecessorRootChanged.into_io()) + } +} + /// The empty retention state admits only a generation-one head with no predecessor. fn require_initial_publication( preparation: &RetentionPublicationPreparation<'_>, diff --git a/src/adapters/retention/filesystem_retention_expectation_tests.rs b/src/adapters/retention/filesystem_retention_expectation_tests.rs index c19515e..6d9b47d 100644 --- a/src/adapters/retention/filesystem_retention_expectation_tests.rs +++ b/src/adapters/retention/filesystem_retention_expectation_tests.rs @@ -13,7 +13,7 @@ use super::{ AdmittedRetentionManifest, AdmittedRetentionRoot, RetentionCurrentStateRefusal, RetentionPublicationStorage, }; -use crate::execute_retention_publication; +use crate::{RetentionNamespace, RetentionRoot, execute_retention_publication}; #[test] fn absent_head_with_retention_artifacts_refuses_as_recovery() -> Result<(), Box> { @@ -101,3 +101,74 @@ fn current_expectation_refuses_when_the_namespace_directory_is_absent() -> Resul sandbox.remove()?; Ok(()) } + +#[test] +fn successor_refuses_when_the_predecessor_root_file_is_absent() -> Result<(), Box> { + let (sandbox, mut authority) = open_authority("filesystem-retention-predecessor-absent")?; + let root_bytes = fixture(ROOT_HEX)?; + let _published = + execute_retention_publication(&mut authority, &initial_preparation(&root_bytes)?)?; + let current = authority + .observe_current()? + .ok_or("published retention head was not observed")?; + let current_root = AdmittedRetentionRoot::decode(&root_bytes)?; + let current_manifest = AdmittedRetentionManifest::decode(current.manifest_bytes())?; + fs::remove_file(root_pool_path(sandbox.path(), ¤t_root))?; + let candidate = successor_root(¤t_root)?; + let preparation = successor_preparation(¤t_root, ¤t_manifest, candidate.encoded())?; + let before = retention_witness(sandbox.path())?; + + let error = RetentionPublicationStorage::verify_current(&mut authority, &preparation) + .err() + .ok_or("successor was admitted over an absent predecessor root file")?; + + assert!(matches!( + refusal(&error), + Some(RetentionCurrentStateRefusal::PredecessorRootAbsent) + )); + assert_eq!(retention_witness(sandbox.path())?, before); + drop(authority); + sandbox.remove()?; + Ok(()) +} + +#[test] +fn successor_refuses_when_the_predecessor_root_bytes_changed() -> Result<(), Box> { + let (sandbox, mut authority) = open_authority("filesystem-retention-predecessor-changed")?; + let root_bytes = fixture(ROOT_HEX)?; + let _published = + execute_retention_publication(&mut authority, &initial_preparation(&root_bytes)?)?; + let current = authority + .observe_current()? + .ok_or("published retention head was not observed")?; + let current_root = AdmittedRetentionRoot::decode(&root_bytes)?; + let current_manifest = AdmittedRetentionManifest::decode(current.manifest_bytes())?; + let pool_entry = root_pool_path(sandbox.path(), ¤t_root); + let mut changed = fs::read(&pool_entry)?; + *changed.last_mut().ok_or("empty root pool entry")? ^= 0x01; + fs::write(&pool_entry, &changed)?; + let candidate = successor_root(¤t_root)?; + let preparation = successor_preparation(¤t_root, ¤t_manifest, candidate.encoded())?; + + let error = RetentionPublicationStorage::verify_current(&mut authority, &preparation) + .err() + .ok_or("successor was admitted over changed predecessor root bytes")?; + + assert!(matches!( + refusal(&error), + Some(RetentionCurrentStateRefusal::PredecessorRootChanged) + )); + drop(authority); + sandbox.remove()?; + Ok(()) +} + +#[test] +fn predecessor_read_bound_derives_from_the_typed_root_limits() -> Result<(), Box> { + let anchors = usize::try_from(RetentionRoot::MAXIMUM_ANCHOR_COUNT)?; + let namespace = usize::from(RetentionNamespace::MAXIMUM_BYTE_LENGTH); + let derived = 192_usize + namespace + anchors * 119 + 64; + + assert_eq!(super::root_header_decoder::MAXIMUM_ENCODED_LENGTH, derived); + Ok(()) +} diff --git a/src/adapters/retention/filesystem_retention_refusal.rs b/src/adapters/retention/filesystem_retention_refusal.rs index 1ff0960..fd22601 100644 --- a/src/adapters/retention/filesystem_retention_refusal.rs +++ b/src/adapters/retention/filesystem_retention_refusal.rs @@ -75,6 +75,12 @@ pub enum RetentionCurrentStateRefusal { CommittedRootAbsent, /// The committed root pool entry holds different bytes. CommittedRootChanged, + /// The candidate does not name the manifest's current root as its predecessor. + PredecessorMismatch, + /// The predecessor root pool entry is absent or exceeds the format bound. + PredecessorRootAbsent, + /// The predecessor root pool entry does not decode to the manifest's selection. + PredecessorRootChanged, /// A record's kind or length disagreed with its declaration. RecordKindOrLength, /// A record carried bytes beyond its declared length. @@ -113,6 +119,9 @@ impl fmt::Display for RetentionCurrentStateRefusal { Self::CommittedNamespaceUnavailable => formatter.write_str("committed root namespace directory is unavailable"), Self::CommittedRootAbsent => formatter.write_str("committed root pool entry is absent"), Self::CommittedRootChanged => formatter.write_str("committed root pool entry bytes disagreed"), + Self::PredecessorMismatch => formatter.write_str("candidate does not name the current root as its predecessor"), + Self::PredecessorRootAbsent => formatter.write_str("predecessor root pool entry is absent or exceeds the format bound"), + Self::PredecessorRootChanged => formatter.write_str("predecessor root pool entry does not decode to the manifest's selection"), Self::RecordKindOrLength => formatter.write_str("retention record kind or length disagreed"), Self::RecordTrailingBytes => formatter.write_str("retention record carried trailing bytes"), Self::RecordLengthOverflow => formatter.write_str("retention record length exceeded the addressable range"), diff --git a/src/adapters/retention/filesystem_retention_storage.rs b/src/adapters/retention/filesystem_retention_storage.rs index dc61a13..01a3fc3 100644 --- a/src/adapters/retention/filesystem_retention_storage.rs +++ b/src/adapters/retention/filesystem_retention_storage.rs @@ -16,6 +16,7 @@ use super::{ RetentionCurrentStateRefusal, RetentionNamespaceAdmission, RetentionPublicationPreparation, RetentionPublicationStorage, RetentionTransitionDisposition, }; +use crate::RetentionGenerationExpectation; use crate::adapters::filesystem_catalog_artifact::synchronize_directory; impl RetentionPublicationStorage for FilesystemRetentionPublicationAuthority { @@ -44,6 +45,15 @@ impl RetentionPublicationStorage for FilesystemRetentionPublicationAuthority { &self.roots, preparation.candidate(), )?; + if let (RetentionGenerationExpectation::Current(_), Some(current)) = + (preparation.expected(), current.as_ref()) + { + filesystem_retention_current::verify_predecessor( + &self.roots, + current, + preparation.candidate(), + )?; + } } if disposition == RetentionTransitionDisposition::AlreadyCommitted { let current = current diff --git a/src/adapters/retention/root_header_decoder.rs b/src/adapters/retention/root_header_decoder.rs index 0de5091..7e0f5e9 100644 --- a/src/adapters/retention/root_header_decoder.rs +++ b/src/adapters/retention/root_header_decoder.rs @@ -9,6 +9,12 @@ use super::root_field_decoder::{ pub(super) const HEADER_LENGTH: usize = 192; const ANCHOR_WIDTH: usize = 119; const TRAILER_LENGTH: usize = 64; +/// Longest canonical root: the header, a namespace at +/// `RetentionNamespace::MAXIMUM_BYTE_LENGTH` (255), `RetentionRoot:: +/// MAXIMUM_ANCHOR_COUNT` (65,536) anchors, and the trailer. Pinned against the +/// typed constants by `filesystem_retention_expectation_tests`. +pub(super) const MAXIMUM_ENCODED_LENGTH: usize = + HEADER_LENGTH + 255 + 65_536 * ANCHOR_WIDTH + TRAILER_LENGTH; pub(super) struct DecodedRootHeader { pub(super) generation: u64, From 1b9dd1044e08d343ed8c7d5154e76208883ad901 Mon Sep 17 00:00:00 2001 From: James Ross Date: Mon, 7 Sep 2026 10:52:35 -0700 Subject: [PATCH 076/111] Fix: refuse a version-two reopen whose root identity is not the intent's Version-two reopen probed the root's device, mount, and file identity and discarded the result. Nothing compared it with the coordinates the migration intent binds, although the migration authority compares exactly those before mutation, so a store relocated by cp -a or restored to another volume received retention writer authority over a root whose intent describes a different one. filesystem_version_two_records::admit now returns the intent's bound root coordinates, and require_root_identity compares device, mount, and file with the probed identity, refusing on the first disagreement with FilesystemPlatformAdmissionError::RootIdentityChanged { coordinate, expected, observed } using the migration path's StoreRootIdentityCoordinate vocabulary. Decision recorded in the plan: relocation is refused, matching migration-time behaviour; the volatile-coordinate question (S3/N2) remains a recovery design item. Regression law: a pure comparison over synthetic identities admits an exact match and refuses each coordinate independently with the exact coordinate, expected, and observed values; the exact-migrated-store law still admits. Self-review finding A2 (P2). Refs #78 --- CHANGELOG.md | 5 ++ .../filesystem_platform_admission_error.rs | 15 +++++- .../filesystem_version_two_admission.rs | 52 ++++++++++++++++--- .../filesystem_version_two_records.rs | 41 +++++++++++++-- .../filesystem_version_two_admission_tests.rs | 35 ++++++++++++- 5 files changed, 137 insertions(+), 11 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index cdef263..0640779 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,11 @@ after its public API and format compatibility policies are established. ### Changed +- Version-two reopen compares the reopened root's device, mount, and file + identity with the coordinates bound into `migration.intent` and refuses a + relocated or restored store with + `FilesystemPlatformAdmissionError::RootIdentityChanged`, matching the + comparison the migration authority makes before mutation. - A successor retention publication now reopens the predecessor root the current manifest selects, bounded by the root format's maximum encoded length, and requires it to decode to exactly that generation and digest; a diff --git a/src/adapters/filesystem_platform_admission_error.rs b/src/adapters/filesystem_platform_admission_error.rs index 035eb36..b26d788 100644 --- a/src/adapters/filesystem_platform_admission_error.rs +++ b/src/adapters/filesystem_platform_admission_error.rs @@ -4,7 +4,7 @@ use std::error::Error; use std::fmt; use std::io; -use super::WriterLockAcquireError; +use super::{StoreRootIdentityCoordinate, WriterLockAcquireError}; /// Failure to reacquire writer authority over one published filesystem store. #[derive(Debug)] @@ -29,6 +29,15 @@ pub enum FilesystemPlatformAdmissionError { /// Preserved record-admission failure. source: io::Error, }, + /// The reopened root's physical identity is not the one the migration intent bound. + RootIdentityChanged { + /// The coordinate that disagreed. + coordinate: StoreRootIdentityCoordinate, + /// The value bound into `migration.intent`. + expected: u64, + /// The value observed on reopen. + observed: u64, + }, } impl fmt::Display for FilesystemPlatformAdmissionError { @@ -38,6 +47,9 @@ impl fmt::Display for FilesystemPlatformAdmissionError { Self::WriterLock { .. } => "published store writer-lock acquisition failed", Self::Namespace { .. } => "published store namespace admission failed", Self::MigrationRecord { .. } => "version-two migration record admission failed", + Self::RootIdentityChanged { .. } => { + "reopened root identity disagrees with the migration intent" + } }) } } @@ -48,6 +60,7 @@ impl Error for FilesystemPlatformAdmissionError { Self::Platform { source } | Self::Namespace { source } | Self::MigrationRecord { source } => Some(source), + Self::RootIdentityChanged { .. } => None, Self::WriterLock { source } => Some(source), } } diff --git a/src/adapters/filesystem_version_two_admission.rs b/src/adapters/filesystem_version_two_admission.rs index 635987b..61f52fc 100644 --- a/src/adapters/filesystem_version_two_admission.rs +++ b/src/adapters/filesystem_version_two_admission.rs @@ -7,9 +7,11 @@ use cap_std::ambient_authority; use cap_std::fs::Dir; use super::filesystem_root_identity::FilesystemRootIdentity; +pub(super) use super::filesystem_version_two_records::BoundRootIdentity; use super::{ - FilesystemPlatformAdmissionError, FilesystemWriterLock, filesystem_initialization_namespace, - filesystem_platform_profile, filesystem_version_two_records, + FilesystemPlatformAdmissionError, FilesystemWriterLock, StoreRootIdentityCoordinate, + filesystem_initialization_namespace, filesystem_platform_profile, + filesystem_version_two_records, }; /// Exclusive writer authority over a completely migrated version-two root. @@ -66,11 +68,49 @@ impl FilesystemVersionTwoAdmission { .map_err(|source| FilesystemPlatformAdmissionError::Namespace { source })?; filesystem_initialization_namespace::admit_version_two(&directory) .map_err(|source| FilesystemPlatformAdmissionError::Namespace { source })?; - let _root_identity: FilesystemRootIdentity = - filesystem_platform_profile::root_identity(&directory) - .map_err(|source| FilesystemPlatformAdmissionError::Platform { source })?; - filesystem_version_two_records::admit(&directory) + let observed = filesystem_platform_profile::root_identity(&directory) + .map_err(|source| FilesystemPlatformAdmissionError::Platform { source })?; + let bound = filesystem_version_two_records::admit(&directory) .map_err(|source| FilesystemPlatformAdmissionError::MigrationRecord { source })?; + require_root_identity(bound, observed)?; Ok(Self { lock }) } } + +/// Requires the reopened root to be the physical root the migration intent bound. +/// +/// Device, mount, and file coordinates are compared exactly, as the migration +/// authority compares them before mutation. A relocated or restored store +/// refuses rather than receiving retention authority over a root whose intent +/// describes a different volume. +pub(super) fn require_root_identity( + bound: BoundRootIdentity, + observed: FilesystemRootIdentity, +) -> Result<(), FilesystemPlatformAdmissionError> { + for (coordinate, expected, actual) in [ + ( + StoreRootIdentityCoordinate::Device, + bound.device(), + observed.device(), + ), + ( + StoreRootIdentityCoordinate::Mount, + bound.mount(), + observed.mount(), + ), + ( + StoreRootIdentityCoordinate::File, + bound.file(), + observed.file(), + ), + ] { + if expected != actual { + return Err(FilesystemPlatformAdmissionError::RootIdentityChanged { + coordinate, + expected, + observed: actual, + }); + } + } + Ok(()) +} diff --git a/src/adapters/filesystem_version_two_records.rs b/src/adapters/filesystem_version_two_records.rs index f3b8306..e31b068 100644 --- a/src/adapters/filesystem_version_two_records.rs +++ b/src/adapters/filesystem_version_two_records.rs @@ -15,14 +15,45 @@ const RECEIPT_NAME: &str = "migration.receipt"; const MARKER_LENGTH: usize = 96; const RECORD_LENGTH: usize = 256; +/// Root identity coordinates bound into an admitted `migration.intent`. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(super) struct BoundRootIdentity { + device: u64, + mount: u64, + file: u64, +} + +impl BoundRootIdentity { + pub(super) const fn new(device: u64, mount: u64, file: u64) -> Self { + Self { + device, + mount, + file, + } + } + + pub(super) const fn device(self) -> u64 { + self.device + } + + pub(super) const fn mount(self) -> u64 { + self.mount + } + + pub(super) const fn file(self) -> u64 { + self.file + } +} + /// Reads and jointly admits `FORMAT`, `migration.intent`, and `migration.receipt`. /// /// Each record is reopened without following links, bounded to its exact /// canonical length, and decoded. The receipt is admitted only against the /// decoded intent and marker, so a record set that is individually /// well-formed but mutually inconsistent refuses. Writer authority over a -/// version-two root must not be returned before this admission succeeds. -pub(super) fn admit(root: &Dir) -> io::Result<()> { +/// version-two root must not be returned before this admission succeeds. The +/// intent's bound root coordinates are returned for identity comparison. +pub(super) fn admit(root: &Dir) -> io::Result { let marker_bytes = read_exact(root, MARKER_NAME, MARKER_LENGTH)?; let intent_bytes = read_exact(root, INTENT_NAME, RECORD_LENGTH)?; let receipt_bytes = read_exact(root, RECEIPT_NAME, RECORD_LENGTH)?; @@ -32,7 +63,11 @@ pub(super) fn admit(root: &Dir) -> io::Result<()> { .map_err(|source| invalid_data(INTENT_NAME, &source))?; let _receipt = AdmittedStoreMigrationReceipt::decode(&receipt_bytes, &intent, &marker) .map_err(|source| invalid_data(RECEIPT_NAME, &source))?; - Ok(()) + Ok(BoundRootIdentity::new( + intent.root_device_identity().get(), + intent.root_mount_identity().get(), + intent.root_file_identity().get(), + )) } fn read_exact(root: &Dir, name: &str, length: usize) -> io::Result> { diff --git a/src/adapters/retention/filesystem_version_two_admission_tests.rs b/src/adapters/retention/filesystem_version_two_admission_tests.rs index 80b12b4..dcc90a9 100644 --- a/src/adapters/retention/filesystem_version_two_admission_tests.rs +++ b/src/adapters/retention/filesystem_version_two_admission_tests.rs @@ -4,7 +4,11 @@ use std::error::Error; use std::fs; use super::filesystem_retention_test_fixture::migrated_store; -use crate::adapters::{FilesystemPlatformAdmissionError, FilesystemVersionTwoAdmission}; +use crate::adapters::filesystem_root_identity::FilesystemRootIdentity; +use crate::adapters::filesystem_version_two_admission::{BoundRootIdentity, require_root_identity}; +use crate::adapters::{ + FilesystemPlatformAdmissionError, FilesystemVersionTwoAdmission, StoreRootIdentityCoordinate, +}; #[test] fn version_two_reopen_refuses_a_corrupt_format_marker() -> Result<(), Box> { @@ -111,3 +115,32 @@ fn production_version_two_reopen_admits_an_exact_migrated_store() -> Result<(), sandbox.remove()?; Ok(()) } + +#[test] +fn reopened_root_identity_must_match_the_intent_coordinates() { + let bound = BoundRootIdentity::new(1, 2, 3); + + assert!(require_root_identity(bound, FilesystemRootIdentity::new(1, 2, 3)).is_ok()); + assert!(matches!( + require_root_identity(bound, FilesystemRootIdentity::new(1, 2, 4)), + Err(FilesystemPlatformAdmissionError::RootIdentityChanged { + coordinate: StoreRootIdentityCoordinate::File, + expected: 3, + observed: 4, + }) + )); + assert!(matches!( + require_root_identity(bound, FilesystemRootIdentity::new(9, 2, 3)), + Err(FilesystemPlatformAdmissionError::RootIdentityChanged { + coordinate: StoreRootIdentityCoordinate::Device, + .. + }) + )); + assert!(matches!( + require_root_identity(bound, FilesystemRootIdentity::new(1, 7, 3)), + Err(FilesystemPlatformAdmissionError::RootIdentityChanged { + coordinate: StoreRootIdentityCoordinate::Mount, + .. + }) + )); +} From ac35bf8caf2abe7e544e3c52e62ba6254163a93c Mon Sep 17 00:00:00 2001 From: James Ross Date: Mon, 7 Sep 2026 10:59:42 -0700 Subject: [PATCH 077/111] Fix: admit the nested version-two protocol directories on reopen MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit admit_version_two checked only the twelve root-level names, and open_version_two tolerates absent nested directories on the stated grounds that namespace admission covers them — but admission never required recovery/dispositions or the retention pools, and never bounded the contents of gc or recovery, all of which the migration writer's verify_prefix_directories requires at the end of migration. A migrated root that lost recovery/dispositions or gained a stray gc or recovery entry received writer authority; a missing retention pool surfaced later as a retention Directory error instead of a Namespace refusal. Admission now opens retention, gc, and recovery without following links and requires: both retention pools present (its head and stages belong to retention publication), gc empty until KEEP-GC-001 implements its records, and recovery exactly an empty dispositions directory. Regression laws: a missing dispositions directory, a stray gc entry, a stray recovery entry, and a missing retention roots pool each refuse as Namespace. Self-review finding A6 (P2). Refs #78 --- CHANGELOG.md | 5 ++ .../filesystem_initialization_namespace.rs | 28 +++++++++- .../filesystem_version_two_admission_tests.rs | 51 +++++++++++++++++++ 3 files changed, 83 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0640779..6e6f1e8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,11 @@ after its public API and format compatibility policies are established. ### Changed +- Version-two namespace admission now descends into the protocol directories: + `retention` must hold both immutable pools, `gc` must be empty, and + `recovery` must hold exactly an empty `dispositions`, matching what the + migration writer verifies at completion, so post-migration drift refuses at + admission instead of surfacing later as a pinning failure. - Version-two reopen compares the reopened root's device, mount, and file identity with the coordinates bound into `migration.intent` and refuses a relocated or restored store with diff --git a/src/adapters/filesystem_initialization_namespace.rs b/src/adapters/filesystem_initialization_namespace.rs index 29d2dbe..d1be904 100644 --- a/src/adapters/filesystem_initialization_namespace.rs +++ b/src/adapters/filesystem_initialization_namespace.rs @@ -3,6 +3,7 @@ use std::ffi::OsStr; use std::io; +use cap_fs_ext::DirExt; use cap_std::fs::Dir; const LOCK_NAME: &str = "writer.lock"; @@ -37,6 +38,9 @@ const RECEIPT_NAME: &str = "migration.receipt"; const RETENTION_NAME: &str = "retention"; const GC_NAME: &str = "gc"; const RECOVERY_NAME: &str = "recovery"; +const ROOTS_NAME: &str = "roots"; +const MANIFESTS_NAME: &str = "manifests"; +const DISPOSITIONS_NAME: &str = "dispositions"; const VERSION_TWO_NAMES: [&str; 12] = [ LOCK_NAME, STAGING_NAME, @@ -106,7 +110,29 @@ pub(super) fn admit_version_two(directory: &Dir) -> io::Result<()> { admit_required_directory(directory, RETENTION_NAME)?; admit_required_directory(directory, GC_NAME)?; admit_required_directory(directory, RECOVERY_NAME)?; - admit_membership(directory, &VERSION_TWO_NAMES) + admit_membership(directory, &VERSION_TWO_NAMES)?; + admit_version_two_protocol_directories(directory) +} + +/// Admits the nested version-2 protocol directories the migration writer left. +/// +/// `retention` must carry both immutable pools (its head and stages belong to +/// retention publication); `gc` must be empty until `KEEP-GC-001` implements +/// its records; `recovery` must hold exactly an empty `dispositions`. This is +/// the same membership `verify_prefix_directories` requires at the end of +/// migration, so a root that drifted after migration refuses here rather than +/// as a later pinning failure. +fn admit_version_two_protocol_directories(directory: &Dir) -> io::Result<()> { + let retention = directory.open_dir_nofollow(RETENTION_NAME)?; + admit_required_directory(&retention, ROOTS_NAME)?; + admit_required_directory(&retention, MANIFESTS_NAME)?; + let gc = directory.open_dir_nofollow(GC_NAME)?; + admit_membership(&gc, &[])?; + let recovery = directory.open_dir_nofollow(RECOVERY_NAME)?; + admit_required_directory(&recovery, DISPOSITIONS_NAME)?; + admit_membership(&recovery, &[DISPOSITIONS_NAME])?; + let dispositions = recovery.open_dir_nofollow(DISPOSITIONS_NAME)?; + admit_membership(&dispositions, &[]) } fn admit_optional_file(directory: &Dir, name: &str) -> io::Result<()> { diff --git a/src/adapters/retention/filesystem_version_two_admission_tests.rs b/src/adapters/retention/filesystem_version_two_admission_tests.rs index dcc90a9..c3784c6 100644 --- a/src/adapters/retention/filesystem_version_two_admission_tests.rs +++ b/src/adapters/retention/filesystem_version_two_admission_tests.rs @@ -144,3 +144,54 @@ fn reopened_root_identity_must_match_the_intent_coordinates() { }) )); } + +fn refuses_namespace( + name: &str, + mutate: impl FnOnce(&std::path::Path) -> Result<(), Box>, +) -> Result<(), Box> { + let sandbox = migrated_store(name)?; + mutate(sandbox.path())?; + + let error = FilesystemVersionTwoAdmission::reopen_unchecked_for_tests(sandbox.path()) + .err() + .ok_or_else(|| format!("{name}: version-two root was unexpectedly admitted"))?; + + assert!( + matches!(error, FilesystemPlatformAdmissionError::Namespace { .. }), + "{name}: expected a Namespace refusal, got {error:?}" + ); + sandbox.remove()?; + Ok(()) +} + +#[test] +fn version_two_reopen_refuses_a_missing_dispositions_directory() -> Result<(), Box> { + refuses_namespace("version-two-admission-no-dispositions", |root| { + fs::remove_dir(root.join("recovery").join("dispositions"))?; + Ok(()) + }) +} + +#[test] +fn version_two_reopen_refuses_a_stray_gc_entry() -> Result<(), Box> { + refuses_namespace("version-two-admission-gc-junk", |root| { + fs::write(root.join("gc").join("junk"), b"")?; + Ok(()) + }) +} + +#[test] +fn version_two_reopen_refuses_a_stray_recovery_entry() -> Result<(), Box> { + refuses_namespace("version-two-admission-recovery-junk", |root| { + fs::write(root.join("recovery").join("junk"), b"")?; + Ok(()) + }) +} + +#[test] +fn version_two_reopen_refuses_a_missing_retention_pool() -> Result<(), Box> { + refuses_namespace("version-two-admission-no-roots", |root| { + fs::remove_dir(root.join("retention").join("roots"))?; + Ok(()) + }) +} From d2278224add4baef495f895f77adfcc87a0f0056 Mon Sep 17 00:00:00 2001 From: James Ross Date: Mon, 7 Sep 2026 11:02:22 -0700 Subject: [PATCH 078/111] Docs: state that a stage left by a failed write is recovery evidence The self-review asked whether FilesystemRetentionStage::create and its migration twin should unlink a partially written stage when write_all or flush fails, since the next verify_current refuses on the retained stage until recovery exists. The decision, recorded in the plan, follows the doctrine filesystem_segment_stage.rs already states: a dropped stage deliberately leaves its bytes and name for explicit recovery. Fixed-record stages follow the same law; a failed write is crash residue like any other, classified by the same recovery table, and never unlinked. requirements.md's nonclaims now say so, and a law pins the observable contract for manifest.next as the existing law does for root.next: a retained stage refuses the next publication at current-state verification with an unchanged retention witness. No code change. Self-review finding A7 (P2), resolved by decision. Refs #78 --- CHANGELOG.md | 3 +++ docs/formats/segment-store-v2/requirements.md | 4 ++- .../filesystem_retention_storage_tests.rs | 26 +++++++++++++++++++ 3 files changed, 32 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6e6f1e8..da03b66 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,9 @@ after its public API and format compatibility policies are established. ### Changed +- A retention stage left behind by a failed write is documented as recovery + evidence: it is never unlinked, and the next publication refuses until + recovery classifies it, exactly as the segment-stage doctrine already states. - Version-two namespace admission now descends into the protocol directories: `retention` must hold both immutable pools, `gc` must be empty, and `recovery` must hold exactly an empty `dispositions`, matching what the diff --git a/docs/formats/segment-store-v2/requirements.md b/docs/formats/segment-store-v2/requirements.md index 15ca92b..a5cda67 100644 --- a/docs/formats/segment-store-v2/requirements.md +++ b/docs/formats/segment-store-v2/requirements.md @@ -61,6 +61,8 @@ case is not evidence. production-admitted; partial-prefix recovery and crash evidence remain mandatory. This applies to retention publication exactly as it applies to migration: the filesystem publication writer refuses every retained stage - instead of continuing it. + instead of continuing it. A stage left behind by a failed write is recovery + evidence like any crash residue; it is never unlinked, and the next + publication refuses until recovery classifies it. - Benchmarks are required before performance-sensitive retention or migration optimization. diff --git a/src/adapters/retention/filesystem_retention_storage_tests.rs b/src/adapters/retention/filesystem_retention_storage_tests.rs index 67b24fc..368a058 100644 --- a/src/adapters/retention/filesystem_retention_storage_tests.rs +++ b/src/adapters/retention/filesystem_retention_storage_tests.rs @@ -152,3 +152,29 @@ fn migrated_witness(root: &Path) -> io::Result)>> { witness.sort(); Ok(witness) } + +#[test] +fn retained_manifest_stage_refuses_publication_before_recovery() -> Result<(), Box> { + let (sandbox, mut authority) = + open_authority("filesystem-retention-recovery-required-manifest")?; + let root_bytes = fixture(ROOT_HEX)?; + let preparation = initial_preparation(&root_bytes)?; + fs::write( + sandbox.path().join("retention").join("manifest.next"), + b"partial bytes left by a failed write", + )?; + let before = retention_witness(sandbox.path())?; + + let error = execute_retention_publication(&mut authority, &preparation) + .err() + .ok_or("retained manifest stage was unexpectedly published over")?; + + let RetentionPublicationError::CurrentVerification { source } = error else { + return Err("retained stage refused outside current-state verification".into()); + }; + assert_eq!(source.kind(), io::ErrorKind::InvalidData); + assert_eq!(retention_witness(sandbox.path())?, before); + drop(authority); + sandbox.remove()?; + Ok(()) +} From 5c7f68612c0f78b48fe8faeadb01af1c7e76cfbd Mon Sep 17 00:00:00 2001 From: James Ross Date: Mon, 7 Sep 2026 11:19:24 -0700 Subject: [PATCH 079/111] Fix: let the admission bypass tolerate a kernel without STATX_MNT_ID FilesystemPlatformAdmission::unchecked, the test and repository-task bypass, probed root identity through the production path, which refuses unless statx reports STATX_MNT_ID. On a kernel older than 5.8 every test and the crash matrix therefore failed at the bypass instead of exercising the protocol, while the bypass exists precisely to skip platform admission. The Linux probe now takes a MountIdentityPolicy. Production root_identity keeps Required and refuses exactly as before. The bypass calls root_identity_lenient, which records an unreported mount identity as zero; the Lenient variant and its match arm exist only under test or the repository-tasks feature, so a plain build carries no unused policy. BASIC_STATS remains required on both paths. The pure selection is pinned by a platform-neutral law in filesystem_platform_profile_policy_tests.rs; no deterministic RED exists on CI kernels, which report the mount identity. The version-two test bypass keeps the strict comparison, because the intent it compares against was recorded by a strict production probe. Self-review finding A9 (P2). Refs #78 --- CHANGELOG.md | 4 ++ src/adapters/filesystem_platform_admission.rs | 5 +- src/adapters/filesystem_platform_profile.rs | 70 +++++++++++++++++-- ...ilesystem_platform_profile_policy_tests.rs | 27 +++++++ 4 files changed, 99 insertions(+), 7 deletions(-) create mode 100644 src/adapters/filesystem_platform_profile_policy_tests.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index da03b66..f46a14a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,10 @@ after its public API and format compatibility policies are established. ### Changed +- The test and repository-task admission bypass now probes root identity + through a lenient path that records an unreported `STATX_MNT_ID` as zero, so + the suite and crash matrix run on kernels older than 5.8; every production + probe still refuses without a reported mount identity. - A retention stage left behind by a failed write is documented as recovery evidence: it is never unlinked, and the next publication refuses until recovery classifies it, exactly as the segment-stage doctrine already states. diff --git a/src/adapters/filesystem_platform_admission.rs b/src/adapters/filesystem_platform_admission.rs index 8d31009..35fce4c 100644 --- a/src/adapters/filesystem_platform_admission.rs +++ b/src/adapters/filesystem_platform_admission.rs @@ -44,10 +44,13 @@ impl FilesystemPlatformAdmission { (self.lock, self.root_identity) } + /// Grants authority without platform admission for tests and repository + /// tasks; the identity probe tolerates a kernel that reports no mount + /// identity so the bypass does not require `STATX_MNT_ID`. #[cfg(any(test, feature = "repository-tasks"))] fn unchecked(lock: FilesystemWriterLock) -> std::io::Result { let directory = lock.clone_directory()?; - let root_identity = super::filesystem_platform_profile::root_identity(&directory)?; + let root_identity = super::filesystem_platform_profile::root_identity_lenient(&directory)?; Ok(Self::initialized(lock, root_identity)) } } diff --git a/src/adapters/filesystem_platform_profile.rs b/src/adapters/filesystem_platform_profile.rs index f0e0ad8..aea9e2c 100644 --- a/src/adapters/filesystem_platform_profile.rs +++ b/src/adapters/filesystem_platform_profile.rs @@ -149,26 +149,75 @@ fn linux_directory_properties(file: &std::fs::File) -> io::Result Option { + match (policy, reported) { + (_, true) => Some(mount_id), + (MountIdentityPolicy::Required, false) => None, + #[cfg(any(test, feature = "repository-tasks"))] + (MountIdentityPolicy::Lenient, false) => Some(0), + } +} + #[cfg(target_os = "linux")] pub(super) fn root_identity(directory: &Dir) -> io::Result { let file = directory.try_clone()?.into_std_file(); - linux_file_identity(&file) + linux_file_identity(&file, MountIdentityPolicy::Required) +} + +/// Probes root identity for the test and repository-task admission bypass. +/// +/// Identical to [`root_identity`] except that an unreported mount identity is +/// recorded as zero instead of refusing; production admission never uses it. +#[cfg(all(target_os = "linux", any(test, feature = "repository-tasks")))] +pub(super) fn root_identity_lenient(directory: &Dir) -> io::Result { + let file = directory.try_clone()?.into_std_file(); + linux_file_identity(&file, MountIdentityPolicy::Lenient) } #[cfg(target_os = "linux")] -fn linux_file_identity(file: &std::fs::File) -> io::Result { +fn linux_file_identity( + file: &std::fs::File, + policy: MountIdentityPolicy, +) -> io::Result { use rustix::fs::{AtFlags, StatxFlags, statx}; - let required = StatxFlags::BASIC_STATS | StatxFlags::MNT_ID; - let status = statx(file, ".", AtFlags::empty(), required)?; + let requested = StatxFlags::BASIC_STATS | StatxFlags::MNT_ID; + let status = statx(file, ".", AtFlags::empty(), requested)?; let observed = StatxFlags::from_bits_retain(status.stx_mask); - if !observed.contains(required) { + if !observed.contains(StatxFlags::BASIC_STATS) { return Err(unsupported_linux_profile()); } + let reported = observed.contains(StatxFlags::MNT_ID); + let mount_id = admit_mount_identity(policy, reported, status.stx_mnt_id) + .ok_or_else(unsupported_linux_profile)?; Ok(linux_root_identity( status.stx_dev_major, status.stx_dev_minor, - status.stx_mnt_id, + mount_id, status.stx_ino, )) } @@ -196,6 +245,11 @@ pub(super) fn root_identity(directory: &Dir) -> io::Result io::Result { + root_identity(directory) +} + #[cfg(all(not(target_os = "linux"), not(any(test, feature = "repository-tasks"))))] pub(super) fn root_identity(_directory: &Dir) -> io::Result { Err(io::Error::new( @@ -251,3 +305,7 @@ fn unsupported_linux_profile() -> io::Error { #[cfg(all(test, target_os = "linux"))] #[path = "filesystem_platform_profile_tests.rs"] mod tests; + +#[cfg(test)] +#[path = "filesystem_platform_profile_policy_tests.rs"] +mod policy_tests; diff --git a/src/adapters/filesystem_platform_profile_policy_tests.rs b/src/adapters/filesystem_platform_profile_policy_tests.rs new file mode 100644 index 0000000..9bcd63f --- /dev/null +++ b/src/adapters/filesystem_platform_profile_policy_tests.rs @@ -0,0 +1,27 @@ +//! Mount-identity policy laws shared by every platform's test build. + +use super::{MountIdentityPolicy, admit_mount_identity}; + +#[test] +fn production_identity_requires_a_reported_mount_identity() { + assert_eq!( + admit_mount_identity(MountIdentityPolicy::Required, true, 7), + Some(7) + ); + assert_eq!( + admit_mount_identity(MountIdentityPolicy::Required, false, 7), + None + ); +} + +#[test] +fn bypass_identity_records_an_unreported_mount_identity_as_zero() { + assert_eq!( + admit_mount_identity(MountIdentityPolicy::Lenient, true, 7), + Some(7) + ); + assert_eq!( + admit_mount_identity(MountIdentityPolicy::Lenient, false, 7), + Some(0) + ); +} From 1efdee20cf3c93cc9340a70a14db8e4fcafae545 Mon Sep 17 00:00:00 2001 From: James Ross Date: Mon, 7 Sep 2026 11:25:43 -0700 Subject: [PATCH 080/111] Docs: restructure the unreleased changelog into one change per bullet The [Unreleased] section carried one 65-line bullet that accreted a sentence per commit across the retention publication, version-two admission, and migration work, listed Changed before Added, and had no Fixed heading, so the review corrections of this pass were indistinguishable from the features they corrected. The section now follows the Keep a Changelog order. Added names the new authority, admission type, typed refusal, migration phases, and the retention_format and migration_format fuzz targets, one change per bullet. Changed keeps the version-one refusal of migrated roots, the documentation gate, the adapters split, and the direct restart read. Fixed gathers every review correction under a sentence stating that none shipped in a release. The historical Changed and Added bullets are moved, not rewritten. The plan also asked for a reject_trailing_bytes behaviour bullet; none is warranted. Trailing-byte rejection predates this branch (95c3d3ad), the streaming commit only wrapped it, and its restoration changed no refusal. Self-review finding R2 (P2). Refs #78 --- CHANGELOG.md | 579 ++++++++++++++++++++++++++------------------------- 1 file changed, 294 insertions(+), 285 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f46a14a..c390868 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,39 +8,248 @@ after its public API and format compatibility policies are established. ## [Unreleased] +### Added + +- `FilesystemRetentionPublicationAuthority` executes the 17 ordered retention + publication phases against a completely migrated version-2 root. It stages + `root.next`, `manifest.next`, and `head.next` exclusively, verifies device + and inode identity at every transition, hard-links both immutable pool + entries without replacement, atomically replaces `retention/HEAD`, and + removes retained stages only after its canonical target verifies. A + successor is admitted only when the prepared head names the observed + manifest as its exact predecessor at the next liveness generation; a + superseded candidate, an exact already-committed retry, and every retained + stage refuse or return with zero retention mutation. +- `FilesystemVersionTwoAdmission::reopen` grants version-two writer authority + as its own type, so no version-one publisher can consume it. It reopens + `FORMAT`, `migration.intent`, and `migration.receipt` without following + links, bounds each to its canonical length, admits the receipt only against + the decoded intent and marker, and on Linux admits `retention`, + `retention/roots`, `retention/manifests`, `gc`, `recovery`, and + `recovery/dispositions` against the root's filesystem, mount, and inode + flags exactly as the version-1 protocol directories are admitted. + `FilesystemPlatformAdmissionError::MigrationRecord` names a record refusal. +- `observe_current` returns the published retention head and its + cross-verified pool manifest. +- `RetentionCurrentStateRefusal` travels as the source of every `InvalidData` + that filesystem current-state verification returns, so a superseded + candidate, a stale committed retry, an absent head over populated pools, and + each corruption or decode refusal are distinguishable to callers and + preserve their underlying decode errors. +- Version-2 format marker, typed canonical intent and receipt construction, + and record admission bind exact catalog, predecessor, root, definition, + store, empty-state, checksum, digest, and synchronization-mask coordinates. +- `StoreMigrationPhase` freezes the 21 migration transitions with explicit + storage and verification-first execution. The filesystem migration + authority derives and revalidates one canonical intent from exact Linux + root, namespace, head, catalog, and inventory coordinates; its streamed + inventory is bounded and completely admits every immutable-pool artifact + under the writer lock. +- Retention preflight combines expected-generation planning with + deterministic closure verification, and authority-revalidated 17-phase + orchestration returns its receipt only after durable cleanup. +- The `retention_format` and `migration_format` fuzz targets drive the + retention root, manifest, and head decoders and all three migration record + decoders. +- Specified `keep.segment-store/v2` retention values, root generations, + liveness manifests, reader snapshots, one-way staged migration, exact crash + boundaries, and reserved GC/disposition records. Validated public + `RetentionNamespace`, namespace-digest, `RootGeneration`, + `LivenessGeneration`, `RetentionAnchor`, realization profile, closure limits, + and semantic root values now establish the core boundary. The canonical root + encoder reproduces the independent version-2 golden bytes, and the decoder + verifies framing, checksum, root digest, anchor-set digest, nested identities, + resource bounds, canonical anchor order, and semantic invariants before + admission. Validated global manifest values and their canonical encoder and + decoder now reproduce the independent manifest fixture and enforce liveness + history, namespace uniqueness, bounds, ordering, and all three integrity + layers. Typed manifest lengths and semantic global heads now reproduce and + admit the exact 144-byte head fixture with fixed framing, checksum-first + semantic admission, and explicit generation-history laws. Storage-independent + transition planning now compares absent or exact-generation expectations, + admits only same-namespace exact successors, preserves expected and observed + stale coordinates, and distinguishes byte-identical already-committed + replay. Deterministic storage-independent closure verification now derives + unique catalog members, enforces exact node, depth, encoded-byte, and + physical-byte accounting, replays the registered storage profile, + authenticates each complete retained blob, and emits a catalog-bound + canonical closure digest. Version-1 immutable bytes remain authoritative; + production version-2 writing remains unavailable until issue #19's + executable evidence is complete. +- Accepted ADR-0009 defines caller-supplied retention namespaces, + `BlobId`/`LayoutId` reconstruction anchors, fail-closed canonical closure, + generation-checked retention publication, immutable liveness snapshots, + release nonclaims, and GC evidence boundaries. This records the M4 design + contract; it does not claim that retention transitions or GC are + implemented. +- Checked catalog generations; canonical catalog and publication-head codecs; + exact logical-record-to-segment admission with one bounded physical lookup + plan, one scan per referenced segment, and refusal of every unreferenced + caller-supplied segment during construction or admission; deterministic + successor proofs; immutable reader snapshots; seeded parser fuzzing; and + `BTreeMap` transition-model evidence for `keep.segment-store/v1`. +- Blocking `FilesystemCatalogPublisher` publication under a persistent + kernel-managed writer lock and required `FilesystemPlatformAdmission`, with + pinned directory capabilities, + no-replacement immutable-pool links, complete post-link verification, + explicit file and directory synchronization, transitive `head.next` + verification, atomic `HEAD` replacement, and stale or recovery-required + refusal before mutation. New filesystem segment publication consumes the + sealed stage through its creating publisher, checks process-local publisher + authority, and closes the writable handle before any immutable-pool link; + publisher teardown closes every retained writable handle before releasing + writer authority. + Retry of an already-current complete candidate re-synchronizes the root and + returns an explicit `CatalogPublicationOutcome::AlreadyPublished` receipt + without repeating publication mutations. Retained `head.next` or + `current.cat`, an unselected `current.seg`, and every fixed-name stage on an + already-current retry now refuse at current-state verification before any + publication mutation. An absent `HEAD` with any retained segment-pool or + catalog-pool entry also refuses before mutation. +- Bounded `FilesystemCatalogSnapshot` restart loading that follows only exact + checksummed head, catalog, and segment coordinates; refuses symbolic links, + nonregular files, malformed or conflicting bytes, dangling entries, and + resource-limit violations; and retains immutable bytes for pinned logical + reads. +- Public, allocation-free `SegmentHeader` admission and emission for the exact + `keep.segment-store/v1` 64-byte header, with field-complete typed refusals + and golden-corpus evidence. +- Public, allocation-free `SegmentRecordHeader` admission and emission for the + exact 112-byte chunk and flat-layout record grammar, with typed logical + identities, checked length derivation, and field-complete corruption laws. +- Borrowed `ChecksummedSegmentRecord` and `AdmittedSegmentRecord` states for + bounded complete-record framing, checksum verification, logical + content-identity admission, and allocation-free chunk preparation. +- Public, allocation-free `SegmentSeal` admission and emission for the exact + 128-byte immutable-segment terminator, with checked physical coordinates, + domain-separated digest verification, and seal-checksum corruption laws. +- Borrowed `AdmittedSegment` reading with explicit record and layout resource + limits, exact nested framing and identity admission, physical-order record + iteration, trailing-byte refusal, and duplicate-identity index reservation + bounded by both the configured count and physical record-header capacity. +- Consuming `StagedSegment` transitions and immutable `SealedSegment` receipts + for exact append-only record writing, streaming seal construction, explicit + prefix/sealed flush-and-sync order, phase-typed I/O refusals, and a fallibly + reserved membership index for sublinear duplicate admission. +- Writer-authorized `FilesystemSegmentStage` creation for the fixed + `current.seg` staging name, with a lifetime that retains the + `FilesystemCatalogPublisher` lock, atomic no-replacement admission, + preserved existing evidence, zero-origin writing, and no implicit cleanup + from `Drop`. +- Rust cargo-fuzz coverage for the public segment header, record header, + complete record, seal, and complete-segment parser boundaries, seeded from + the canonical version-1 segment fixtures through `cargo xtask`. +- ADR-0005 and the implementation-independent `keep.segment-store/v1` + protocol: exact immutable segment, catalog-generation, and publication-head + grammars; canonical ordering, bounds, and domain-separated checksums; + one-writer/many-reader publication with explicit flush, synchronization, + atomic replacement, and directory-synchronization order; stable + `KEEP-CRASH-001`–`035` transitions; typed recovery classifications; and + golden physical artifacts. Directory-synchronization crash classes admit + both the lawful pre-sync and durable namespace states, and recovery admits + only the exact verified stage/pool digest duplicate created by interrupted + hard-link publication. Fresh-store initialization is writer-locked, + idempotent across every partial canonical namespace set, and admitted only + after root synchronization. Explicit recovery can complete a durable + fixed-name stage into its immutable pool and durably clear the stage without + promoting a publication head. Explicit discard receipts now follow + synchronization of the stage's actual parent: `staging` for segment and + catalog stages, or the store root for `head.next`. Segment and catalog + production are implemented; crash recovery remains assigned to issue #17. + The golden corpus now includes a generation-2 catalog/head pair whose + predecessor field is the exact generation-1 catalog digest. +- A deterministic, bounded, license-safe streaming CAS benchmark corpus and + release-only `cargo xtask benchmark-baseline` workflow covering all required + ingestion, edit, deduplication, range-read, verification, and input + partitioning scenarios. The versioned TSV report records exact semantic I/O, + amplification and reuse ratios, p50/p95/p99 wall latency, process CPU time, + throughput, allocations, incremental peak live heap, five chunking-profile + comparisons, compiler/target/Git/host identity bound across execution, + refusal of ambient code-generation settings and external Cargo + configuration, single-writer recoverable artifact publication, and an + explicit refusal to invent regression thresholds before controlled baseline + history exists. +- Validated half-open `ByteRange` coordinates and allocation-free range + planning, plus exact synchronous reference-store range reads that load only + overlapping chunks, authenticate each selected complete chunk before + slicing, reauthenticate before output, and return a receipt whose deliberately + narrow verification scope excludes the complete blob, unrequested chunks, + and storage-profile boundaries. Caller-supplied layouts and records must + resolve to a committed target-layout binding before chunk lookup. +- Expected-`BlobId` staging with typed complete-stream mismatch refusal; the + Golden File Worldline scenario and every claimed-content mutation now run + through public stage, commit, and reconstruct APIs instead of only the + private test model. +- Publication now calculates and validates the final materialized-byte count + before changing visible reference-store state, so intervening capacity + exhaustion cannot expose a partial commit. +- Publication refuses staged work whose destination lacks a required chunk, + preventing cross-store commits from exposing incomplete layouts. +- Bounded canonical layout-record reconstruction with typed pre-output refusal + for malformed records, zero-progress and over-reporting writers, output I/O + failures, conflicting stored chunks, corrupted chunk content, and ordinary + publication attempts that would silently repair missing committed chunks or + missing, incomplete, or wrong-target committed layout indexes. +- Exact synchronous reference-store reconstruction that authenticates every + chunk, the registered storage-profile boundaries, and the complete named + blob before output, reverifies chunks during emission, completes short + writes, retries interruptions, and reports missing or mismatched content + with typed expected and observed identities. +- Capacity-bounded streaming ingestion into a non-durable in-memory reference + adapter, with exact blob, chunk, and layout identity calculation, typed + source and capacity refusals, streaming enforcement of the caller's layout + entry cap, identity-based chunk deduplication, and an explicit + staged-to-visible commit transition. +- An independent field-by-field flat-layout fixture oracle that verifies every + fixed offset, checksum, and `LayoutId` before cross-checking the production + encoder. +- Validated flat-layout admission with explicit entry caps and an exact + canonical version-1 encoder backed by every frozen record and `LayoutId` + witness. +- Bounded flat-layout decoding through explicit parse, validate, and admit + stages with deterministic first-failure errors for every frozen structural + mutation and optional final expected-`LayoutId` verification. +- Generated flat-layout canonicality properties and a continuous + `layout_record` decoder fuzz target seeded through the Rust `xtask` with all + four frozen binary records. +- Canonical `StorageProfileId` text coordinates and explicit admission of the + frozen `fastcdc-64k-v1` profile through `RegisteredStorageProfile`. +- Canonical binary and text `LayoutId` coordinates with typed plan-length and + digest mismatch reporting backed by every coordinate refusal vector. +- The canonical `keep.flat-chunks/v1` durable layout specification, typed + `LayoutId` grammar, checked flat-plan bounds, domain-separated checksum, + exact golden records, field-complete `LayoutId` refusal tables and + cardinality-before-aggregate first-failure plan mutation ledger, and + verified storage-profile boundary replay law. +- Canonical version-1 `ChunkId` calculation in a domain distinct from + `BlobId`, with independent golden vectors. +- A constant-memory `FastCdc` detector for `fastcdc-64k-v1` that preserves + boundaries and chunk identities across arbitrary feed partitioning, batches + contiguous identity-hash updates, and enters an explicit failed state after + a typed refusal. +- Typed `ChunkLength`, `ChunkOffset`, and `ChunkSpan` values, corpus-driven + property and adversarial tests, measured allocation and throughput evidence, + and a fail-closed streaming CDC fuzz target. +- Canonical version-1 `BlobId` calculation over exact logical bytes using a + one-pass, length-committing BLAKE3-256 preimage. +- Strict, allocation-bounded text and fixed-width binary `BlobId` codecs with + typed refusal for malformed and unsupported encodings. +- The implementation-independent Golden File Worldline v1 conformance corpus, + independent vector checker, mutation cases, and bounded reference model. +- A versioned Gear64/FastCDC content-defined chunking profile, canonical + `StorageProfileId`, and language-neutral golden boundary corpus. +- Initial repository foundation. + ### Changed -- The test and repository-task admission bypass now probes root identity - through a lenient path that records an unreported `STATX_MNT_ID` as zero, so - the suite and crash matrix run on kernels older than 5.8; every production - probe still refuses without a reported mount identity. -- A retention stage left behind by a failed write is documented as recovery - evidence: it is never unlinked, and the next publication refuses until - recovery classifies it, exactly as the segment-stage doctrine already states. -- Version-two namespace admission now descends into the protocol directories: - `retention` must hold both immutable pools, `gc` must be empty, and - `recovery` must hold exactly an empty `dispositions`, matching what the - migration writer verifies at completion, so post-migration drift refuses at - admission instead of surfacing later as a pinning failure. -- Version-two reopen compares the reopened root's device, mount, and file - identity with the coordinates bound into `migration.intent` and refuses a - relocated or restored store with - `FilesystemPlatformAdmissionError::RootIdentityChanged`, matching the - comparison the migration authority makes before mutation. -- A successor retention publication now reopens the predecessor root the - current manifest selects, bounded by the root format's maximum encoded - length, and requires it to decode to exactly that generation and digest; a - namespace directory alone no longer stands in for an available predecessor. -- Retention current-state verification binds this store's catalog `HEAD` to - the closure's catalog coordinates for every disposition, so an - already-committed retry no longer returns a receipt citing a catalog this - store does not name. -- Version-one recovery adapters refuse version-two residue at the store root - before pinning any pool: a format marker, reader fence, migration record or - stage, or a `retention`, `gc`, or `recovery` directory means recovery - discard, completion, resume, and finalization refuse instead of rewriting a - migrated store's version-one pools. Unknown entries continue to be refused - by recovery name classification. +- Version-one reopen refuses a migrated root, and `admit_version_two` owns the + separate version-2 namespace boundary. Version-one recovery adapters refuse + version-two residue at the store root before pinning any pool: a format + marker, reader fence, migration record or stage, or a `retention`, `gc`, or + `recovery` directory means recovery discard, completion, resume, and + finalization refuse instead of rewriting a migrated store's version-one + pools. Unknown entries continue to be refused by recovery name + classification. - The Rust quality gates now build the crate documentation with `cargo doc --workspace --no-deps --locked`, so a broken intra-doc link under `#![deny(warnings)]` fails CI instead of only failing anyone who documents @@ -53,71 +262,6 @@ after its public API and format compatibility policies are established. by trailing-byte rejection; the interim chunked transfer layer, which pumped every artifact through an 8 KiB buffer without lowering peak memory, is removed with no change to refusal behaviour. -- Version-2 marker, typed canonical intent/receipt construction, and record admission bind - exact catalog, predecessor, root, definition, store, empty-state, and checksum, - digest, and synchronization-mask coordinates; migration fuzzing drives all - three decoders, streamed inventory is bounded, writer-locked filesystem - inventory completely admits every immutable-pool artifact, filesystem - migration authority derives and revalidates one canonical intent from exact - Linux root, namespace, head, catalog, and inventory coordinates, and - `StoreMigrationPhase` freezes 21 transitions with explicit storage and - verification-first execution. - Retention preflight combines expected-generation planning with deterministic - closure verification; authority-revalidated 17-phase orchestration returns its receipt after durable cleanup. - `FilesystemRetentionPublicationAuthority` executes those 17 phases against a - completely migrated version-2 root: it stages `root.next`, `manifest.next`, - and `head.next` exclusively, verifies device and inode identity at every - transition, hard-links both immutable pool entries without replacement, - atomically replaces `retention/HEAD`, and removes retained stages only after - its canonical target verifies. An exact already-committed candidate returns - its receipt with zero retention mutation, and any retained stage refuses as - recovery-required rather than being continued. Version-1 reopen now refuses a - migrated root, and `admit_version_two` owns the separate version-2 namespace - boundary. `observe_current` returns the published head and its cross-verified - pool manifest, and current-state verification admits a successor only when - the prepared head names the observed manifest as its exact predecessor at the - next liveness generation; a superseded candidate refuses with zero mutation. - `FilesystemVersionTwoAdmission::reopen` reopens `FORMAT`, `migration.intent`, and - `migration.receipt` without following links, bounds each to its canonical - length, and admits the receipt only against the decoded intent and marker - before returning writer authority; `FilesystemPlatformAdmissionError::MigrationRecord` - names that refusal. Version-two writer authority is its own type, so no - version-one publisher can consume it. An already-committed retention retry now reopens the - manifest entry and the root pool bytes the head selects and refuses absent, - changed, or corrupt evidence instead of inferring the commit from head - agreement alone. On Linux, `FilesystemVersionTwoAdmission::reopen` admits `retention`, - `retention/roots`, `retention/manifests`, `gc`, `recovery`, and - `recovery/dispositions` against the root's filesystem, mount, and inode - flags exactly as the version-1 protocol directories are admitted. Retention - publication admits the complete `retention` namespace before any forward - write: only `HEAD`, `roots`, and `manifests` may exist, every namespace - directory is 64 lowercase hex, and every pool entry is a regular - `-` file with its canonical suffix. Existing namespace - directories, including recovery-protected orphans, count against the 4,096 - namespace ceiling, and a candidate whose namespace would be the 4,097th - refuses before its root stage exists. Current-state verification now binds - on-disk state to the claimed expectation: an absent `retention/HEAD` is the - empty state only while both pools are empty and admits only an initial head - with no predecessor; a namespace directory must be absent for an `Absent` - expectation and present for a `Current` one. Every mismatch refuses as - recovery-required before any stage is written. Each retention stage - synchronization now also synchronizes the `retention` directory, so the - `root.next`, `manifest.next`, and `head.next` entries are durable before the - namespace directory, pool links, or head replacement that depend on them. - Every read-side reopen in retention publication and version-two admission - opens with `O_NONBLOCK`, so a FIFO planted at `retention/HEAD`, `FORMAT`, or a - pool name refuses by kind instead of blocking under the writer lock. - Filesystem current-state verification now carries a typed - `RetentionCurrentStateRefusal` as the source of every `InvalidData` it - returns, so a superseded candidate, a stale committed retry, an absent head - over populated pools, and each corruption or decode refusal are - distinguishable to callers and preserve their underlying decode errors. - Before any forward retention write, the authority reopens this store's own - catalog `HEAD` and requires it to name exactly the catalog generation and - digest the candidate closure was verified against, so a preparation built - from another store's `CatalogSnapshot` refuses instead of publishing anchors - whose records these pools may not hold. Observing the current state also - requires the head's predecessor digest to equal its manifest's predecessor. - Repository crash-matrix execution now terminates isolated writer process groups at all 105 canonical before/during/after coordinates, retains open writer and stage authority until termination, executes production @@ -409,193 +553,58 @@ after its public API and format compatibility policies are established. longer depends on the adapter-owned `Display` impl. `Debug` output carries no stability contract; this is not a format change. -### Added +### Fixed -- Specified `keep.segment-store/v2` retention values, root generations, - liveness manifests, reader snapshots, one-way staged migration, exact crash - boundaries, and reserved GC/disposition records. Validated public - `RetentionNamespace`, namespace-digest, `RootGeneration`, - `LivenessGeneration`, `RetentionAnchor`, realization profile, closure limits, - and semantic root values now establish the core boundary. The canonical root - encoder reproduces the independent version-2 golden bytes, and the decoder - verifies framing, checksum, root digest, anchor-set digest, nested identities, - resource bounds, canonical anchor order, and semantic invariants before - admission. Validated global manifest values and their canonical encoder and - decoder now reproduce the independent manifest fixture and enforce liveness - history, namespace uniqueness, bounds, ordering, and all three integrity - layers. Typed manifest lengths and semantic global heads now reproduce and - admit the exact 144-byte head fixture with fixed framing, checksum-first - semantic admission, and explicit generation-history laws. Storage-independent - transition planning now compares absent or exact-generation expectations, - admits only same-namespace exact successors, preserves expected and observed - stale coordinates, and distinguishes byte-identical already-committed - replay. Deterministic storage-independent closure verification now derives - unique catalog members, enforces exact node, depth, encoded-byte, and - physical-byte accounting, replays the registered storage profile, - authenticates each complete retained blob, and emits a catalog-bound - canonical closure digest. Version-1 immutable bytes remain authoritative; - production version-2 writing remains unavailable until issue #19's - executable evidence is complete. -- Accepted ADR-0009 defines caller-supplied retention namespaces, - `BlobId`/`LayoutId` reconstruction anchors, fail-closed canonical closure, - generation-checked retention publication, immutable liveness snapshots, - release nonclaims, and GC evidence boundaries. This records the M4 design - contract; it does not claim that retention transitions or GC are - implemented. -- Checked catalog generations; canonical catalog and publication-head codecs; - exact logical-record-to-segment admission with one bounded physical lookup - plan, one scan per referenced segment, and refusal of every unreferenced - caller-supplied segment during construction or admission; deterministic - successor proofs; immutable reader snapshots; seeded parser fuzzing; and - `BTreeMap` transition-model evidence for `keep.segment-store/v1`. -- Blocking `FilesystemCatalogPublisher` publication under a persistent - kernel-managed writer lock and required `FilesystemPlatformAdmission`, with - pinned directory capabilities, - no-replacement immutable-pool links, complete post-link verification, - explicit file and directory synchronization, transitive `head.next` - verification, atomic `HEAD` replacement, and stale or recovery-required - refusal before mutation. New filesystem segment publication consumes the - sealed stage through its creating publisher, checks process-local publisher - authority, and closes the writable handle before any immutable-pool link; - publisher teardown closes every retained writable handle before releasing - writer authority. - Retry of an already-current complete candidate re-synchronizes the root and - returns an explicit `CatalogPublicationOutcome::AlreadyPublished` receipt - without repeating publication mutations. Retained `head.next` or - `current.cat`, an unselected `current.seg`, and every fixed-name stage on an - already-current retry now refuse at current-state verification before any - publication mutation. An absent `HEAD` with any retained segment-pool or - catalog-pool entry also refuses before mutation. -- Bounded `FilesystemCatalogSnapshot` restart loading that follows only exact - checksummed head, catalog, and segment coordinates; refuses symbolic links, - nonregular files, malformed or conflicting bytes, dangling entries, and - resource-limit violations; and retains immutable bytes for pinned logical - reads. -- Public, allocation-free `SegmentHeader` admission and emission for the exact - `keep.segment-store/v1` 64-byte header, with field-complete typed refusals - and golden-corpus evidence. -- Public, allocation-free `SegmentRecordHeader` admission and emission for the - exact 112-byte chunk and flat-layout record grammar, with typed logical - identities, checked length derivation, and field-complete corruption laws. -- Borrowed `ChecksummedSegmentRecord` and `AdmittedSegmentRecord` states for - bounded complete-record framing, checksum verification, logical - content-identity admission, and allocation-free chunk preparation. -- Public, allocation-free `SegmentSeal` admission and emission for the exact - 128-byte immutable-segment terminator, with checked physical coordinates, - domain-separated digest verification, and seal-checksum corruption laws. -- Borrowed `AdmittedSegment` reading with explicit record and layout resource - limits, exact nested framing and identity admission, physical-order record - iteration, trailing-byte refusal, and duplicate-identity index reservation - bounded by both the configured count and physical record-header capacity. -- Consuming `StagedSegment` transitions and immutable `SealedSegment` receipts - for exact append-only record writing, streaming seal construction, explicit - prefix/sealed flush-and-sync order, phase-typed I/O refusals, and a fallibly - reserved membership index for sublinear duplicate admission. -- Writer-authorized `FilesystemSegmentStage` creation for the fixed - `current.seg` staging name, with a lifetime that retains the - `FilesystemCatalogPublisher` lock, atomic no-replacement admission, - preserved existing evidence, zero-origin writing, and no implicit cleanup - from `Drop`. -- Rust cargo-fuzz coverage for the public segment header, record header, - complete record, seal, and complete-segment parser boundaries, seeded from - the canonical version-1 segment fixtures through `cargo xtask`. -- ADR-0005 and the implementation-independent `keep.segment-store/v1` - protocol: exact immutable segment, catalog-generation, and publication-head - grammars; canonical ordering, bounds, and domain-separated checksums; - one-writer/many-reader publication with explicit flush, synchronization, - atomic replacement, and directory-synchronization order; stable - `KEEP-CRASH-001`–`035` transitions; typed recovery classifications; and - golden physical artifacts. Directory-synchronization crash classes admit - both the lawful pre-sync and durable namespace states, and recovery admits - only the exact verified stage/pool digest duplicate created by interrupted - hard-link publication. Fresh-store initialization is writer-locked, - idempotent across every partial canonical namespace set, and admitted only - after root synchronization. Explicit recovery can complete a durable - fixed-name stage into its immutable pool and durably clear the stage without - promoting a publication head. Explicit discard receipts now follow - synchronization of the stage's actual parent: `staging` for segment and - catalog stages, or the store root for `head.next`. Segment and catalog - production are implemented; crash recovery remains assigned to issue #17. - The golden corpus now includes a generation-2 catalog/head pair whose - predecessor field is the exact generation-1 catalog digest. -- A deterministic, bounded, license-safe streaming CAS benchmark corpus and - release-only `cargo xtask benchmark-baseline` workflow covering all required - ingestion, edit, deduplication, range-read, verification, and input - partitioning scenarios. The versioned TSV report records exact semantic I/O, - amplification and reuse ratios, p50/p95/p99 wall latency, process CPU time, - throughput, allocations, incremental peak live heap, five chunking-profile - comparisons, compiler/target/Git/host identity bound across execution, - refusal of ambient code-generation settings and external Cargo - configuration, single-writer recoverable artifact publication, and an - explicit refusal to invent regression thresholds before controlled baseline - history exists. -- Validated half-open `ByteRange` coordinates and allocation-free range - planning, plus exact synchronous reference-store range reads that load only - overlapping chunks, authenticate each selected complete chunk before - slicing, reauthenticate before output, and return a receipt whose deliberately - narrow verification scope excludes the complete blob, unrequested chunks, - and storage-profile boundaries. Caller-supplied layouts and records must - resolve to a committed target-layout binding before chunk lookup. -- Expected-`BlobId` staging with typed complete-stream mismatch refusal; the - Golden File Worldline scenario and every claimed-content mutation now run - through public stage, commit, and reconstruct APIs instead of only the - private test model. -- Publication now calculates and validates the final materialized-byte count - before changing visible reference-store state, so intervening capacity - exhaustion cannot expose a partial commit. -- Publication refuses staged work whose destination lacks a required chunk, - preventing cross-store commits from exposing incomplete layouts. -- Bounded canonical layout-record reconstruction with typed pre-output refusal - for malformed records, zero-progress and over-reporting writers, output I/O - failures, conflicting stored chunks, corrupted chunk content, and ordinary - publication attempts that would silently repair missing committed chunks or - missing, incomplete, or wrong-target committed layout indexes. -- Exact synchronous reference-store reconstruction that authenticates every - chunk, the registered storage-profile boundaries, and the complete named - blob before output, reverifies chunks during emission, completes short - writes, retries interruptions, and reports missing or mismatched content - with typed expected and observed identities. -- Capacity-bounded streaming ingestion into a non-durable in-memory reference - adapter, with exact blob, chunk, and layout identity calculation, typed - source and capacity refusals, streaming enforcement of the caller's layout - entry cap, identity-based chunk deduplication, and an explicit - staged-to-visible commit transition. -- An independent field-by-field flat-layout fixture oracle that verifies every - fixed offset, checksum, and `LayoutId` before cross-checking the production - encoder. -- Validated flat-layout admission with explicit entry caps and an exact - canonical version-1 encoder backed by every frozen record and `LayoutId` - witness. -- Bounded flat-layout decoding through explicit parse, validate, and admit - stages with deterministic first-failure errors for every frozen structural - mutation and optional final expected-`LayoutId` verification. -- Generated flat-layout canonicality properties and a continuous - `layout_record` decoder fuzz target seeded through the Rust `xtask` with all - four frozen binary records. -- Canonical `StorageProfileId` text coordinates and explicit admission of the - frozen `fastcdc-64k-v1` profile through `RegisteredStorageProfile`. -- Canonical binary and text `LayoutId` coordinates with typed plan-length and - digest mismatch reporting backed by every coordinate refusal vector. -- The canonical `keep.flat-chunks/v1` durable layout specification, typed - `LayoutId` grammar, checked flat-plan bounds, domain-separated checksum, - exact golden records, field-complete `LayoutId` refusal tables and - cardinality-before-aggregate first-failure plan mutation ledger, and - verified storage-profile boundary replay law. -- Canonical version-1 `ChunkId` calculation in a domain distinct from - `BlobId`, with independent golden vectors. -- A constant-memory `FastCdc` detector for `fastcdc-64k-v1` that preserves - boundaries and chunk identities across arbitrary feed partitioning, batches - contiguous identity-hash updates, and enters an explicit failed state after - a typed refusal. -- Typed `ChunkLength`, `ChunkOffset`, and `ChunkSpan` values, corpus-driven - property and adversarial tests, measured allocation and throughput evidence, - and a fail-closed streaming CDC fuzz target. -- Canonical version-1 `BlobId` calculation over exact logical bytes using a - one-pass, length-committing BLAKE3-256 preimage. -- Strict, allocation-bounded text and fixed-width binary `BlobId` codecs with - typed refusal for malformed and unsupported encodings. -- The implementation-independent Golden File Worldline v1 conformance corpus, - independent vector checker, mutation cases, and bounded reference model. -- A versioned Gear64/FastCDC content-defined chunking profile, canonical - `StorageProfileId`, and language-neutral golden boundary corpus. -- Initial repository foundation. +Review corrections to the unreleased retention and migration work above; none +of these shipped in a release. + +- An already-committed retention retry reopens the manifest entry and the root + pool bytes the head selects and refuses absent, changed, or corrupt evidence + instead of inferring the commit from head agreement alone. +- Retention current-state verification binds this store's catalog `HEAD` to + the closure's catalog coordinates for every disposition: a preparation built + from another store's `CatalogSnapshot` refuses before any forward write, and + an already-committed retry no longer returns a receipt citing a catalog this + store does not name. Observing the current state also requires the head's + predecessor digest to equal its manifest's predecessor. +- A successor retention publication reopens the predecessor root the current + manifest selects, bounded by the root format's maximum encoded length, and + requires it to decode to exactly that generation and digest; a namespace + directory alone no longer stands in for an available predecessor. +- Version-two reopen compares the reopened root's device, mount, and file + identity with the coordinates bound into `migration.intent` and refuses a + relocated or restored store with + `FilesystemPlatformAdmissionError::RootIdentityChanged`, matching the + comparison the migration authority makes before mutation. +- Version-two namespace admission descends into the protocol directories: + `retention` must hold both immutable pools, `gc` must be empty, and + `recovery` must hold exactly an empty `dispositions`, matching what the + migration writer verifies at completion, so post-migration drift refuses at + admission instead of surfacing later as a pinning failure. +- Retention publication admits the complete `retention` namespace before any + forward write: only `HEAD`, `roots`, and `manifests` may exist, every + namespace directory is 64 lowercase hex, and every pool entry is a regular + `-` file with its canonical suffix. +- Existing namespace directories, including recovery-protected orphans, count + against the 4,096 namespace ceiling, and a candidate whose namespace would + be the 4,097th refuses before its root stage exists. +- Current-state verification binds on-disk state to the claimed expectation: + an absent `retention/HEAD` is the empty state only while both pools are + empty and admits only an initial head with no predecessor; a namespace + directory must be absent for an `Absent` expectation and present for a + `Current` one. Every mismatch refuses as recovery-required before any stage + is written. +- Each retention stage synchronization also synchronizes the `retention` + directory, so the `root.next`, `manifest.next`, and `head.next` entries are + durable before the namespace directory, pool links, or head replacement that + depend on them. +- Every read-side reopen in retention publication and version-two admission + opens with `O_NONBLOCK`, so a FIFO planted at `retention/HEAD`, `FORMAT`, or + a pool name refuses by kind instead of blocking under the writer lock. +- A retention stage left behind by a failed write is documented as recovery + evidence: it is never unlinked, and the next publication refuses until + recovery classifies it, exactly as the segment-stage doctrine already states. +- The test and repository-task admission bypass probes root identity through a + lenient path that records an unreported `STATX_MNT_ID` as zero, so the suite + and crash matrix run on kernels older than 5.8; every production probe still + refuses without a reported mount identity. From b3d6680594328a2797154aae6859902d4bdcce82 Mon Sep 17 00:00:00 2001 From: James Ross Date: Mon, 7 Sep 2026 11:32:09 -0700 Subject: [PATCH 081/111] Refactor: rewrap the retention refusal catalogue to the standard width The Display impl for RetentionCurrentStateRefusal held 26 arms of up to 207 columns, and one doc-comment link ran to 103. rustfmt cannot break a string literal or a link, so cargo fmt --check passed over lines the Rust standard reviews at 100. Each message is now a line-continued literal, the link is reference-style so cargo doc still resolves it, and no Display text changed. A law in tests/adapters_layout_contract.rs pins every line of the file to at most 100 columns; it failed at the previous head and passes now. Self-review finding R3 (P3). Refs #78 --- .../retention/filesystem_retention_refusal.rs | 115 +++++++++++++----- tests/adapters_layout_contract.rs | 20 ++- 2 files changed, 106 insertions(+), 29 deletions(-) diff --git a/src/adapters/retention/filesystem_retention_refusal.rs b/src/adapters/retention/filesystem_retention_refusal.rs index fd22601..0bd1a6c 100644 --- a/src/adapters/retention/filesystem_retention_refusal.rs +++ b/src/adapters/retention/filesystem_retention_refusal.rs @@ -10,9 +10,11 @@ use crate::{CatalogGeneration, LivenessGeneration, RetentionManifestDigest}; /// Exact reason filesystem current-state verification refused a transition. /// /// Every variant is carried as the source of the `io::Error` that -/// [`RetentionPublicationStorage::verify_current`](super::RetentionPublicationStorage::verify_current) -/// returns, so callers can distinguish a lawful stale state that should be -/// replanned from corruption or ambiguity that must route through recovery. +/// [`RetentionPublicationStorage::verify_current`][verify] returns, so callers +/// can distinguish a lawful stale state that should be replanned from +/// corruption or ambiguity that must route through recovery. +/// +/// [verify]: super::RetentionPublicationStorage::verify_current #[derive(Debug)] #[non_exhaustive] pub enum RetentionCurrentStateRefusal { @@ -99,32 +101,89 @@ impl RetentionCurrentStateRefusal { impl fmt::Display for RetentionCurrentStateRefusal { fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { match self { - Self::RetainedStage => formatter.write_str("retained retention stage requires recovery before publication"), - Self::HeadAbsentWithArtifacts => formatter.write_str("retention head is absent while retention pools hold artifacts; recovery is required"), - Self::ExpectedCurrentOverAbsentHead => formatter.write_str("expected a current retention generation but no head is published"), - Self::NonInitialOverAbsentHead => formatter.write_str("absent retention head admits only an initial publication with no predecessor"), - Self::HeadRefused { .. } => formatter.write_str("current retention head refused admission"), - Self::PreparedHeadRefused { .. } => formatter.write_str("prepared retention head refused admission"), - Self::ManifestAbsent => formatter.write_str("current retention head names an absent manifest"), - Self::ManifestRefused { .. } => formatter.write_str("current retention manifest refused admission"), - Self::ManifestDisagreed => formatter.write_str("current retention manifest disagreed with its head"), - Self::HeadPredecessorDisagreed => formatter.write_str("current retention head and its manifest name different predecessors"), - Self::CatalogDisagreed { expected_generation, .. } => write!(formatter, "closure was verified against catalog generation {} which is not this store's current catalog", expected_generation.get()), - Self::CatalogHeadRefused => formatter.write_str("this store's catalog head refused admission"), - Self::LivenessExhausted => formatter.write_str("current liveness generation cannot advance"), - Self::StaleCommittedRetry => formatter.write_str("already-committed retry is stale: another successor is current"), - Self::Superseded { current_generation, .. } => write!(formatter, "candidate is superseded: the current head is liveness generation {}", current_generation.get()), - Self::CommittedSelectionMissing => formatter.write_str("committed manifest does not select the candidate namespace"), - Self::CommittedSelectionMismatch => formatter.write_str("committed manifest selects a different root for the candidate namespace"), - Self::CommittedNamespaceUnavailable => formatter.write_str("committed root namespace directory is unavailable"), + Self::RetainedStage => { + formatter.write_str("retained retention stage requires recovery before publication") + } + Self::HeadAbsentWithArtifacts => formatter.write_str( + "retention head is absent while retention pools hold artifacts; recovery is \ + required", + ), + Self::ExpectedCurrentOverAbsentHead => formatter + .write_str("expected a current retention generation but no head is published"), + Self::NonInitialOverAbsentHead => formatter.write_str( + "absent retention head admits only an initial publication with no predecessor", + ), + Self::HeadRefused { .. } => { + formatter.write_str("current retention head refused admission") + } + Self::PreparedHeadRefused { .. } => { + formatter.write_str("prepared retention head refused admission") + } + Self::ManifestAbsent => { + formatter.write_str("current retention head names an absent manifest") + } + Self::ManifestRefused { .. } => { + formatter.write_str("current retention manifest refused admission") + } + Self::ManifestDisagreed => { + formatter.write_str("current retention manifest disagreed with its head") + } + Self::HeadPredecessorDisagreed => formatter + .write_str("current retention head and its manifest name different predecessors"), + Self::CatalogDisagreed { + expected_generation, + .. + } => write!( + formatter, + "closure was verified against catalog generation {} which is not this store's \ + current catalog", + expected_generation.get() + ), + Self::CatalogHeadRefused => { + formatter.write_str("this store's catalog head refused admission") + } + Self::LivenessExhausted => { + formatter.write_str("current liveness generation cannot advance") + } + Self::StaleCommittedRetry => formatter + .write_str("already-committed retry is stale: another successor is current"), + Self::Superseded { + current_generation, .. + } => write!( + formatter, + "candidate is superseded: the current head is liveness generation {}", + current_generation.get() + ), + Self::CommittedSelectionMissing => { + formatter.write_str("committed manifest does not select the candidate namespace") + } + Self::CommittedSelectionMismatch => formatter.write_str( + "committed manifest selects a different root for the candidate namespace", + ), + Self::CommittedNamespaceUnavailable => { + formatter.write_str("committed root namespace directory is unavailable") + } Self::CommittedRootAbsent => formatter.write_str("committed root pool entry is absent"), - Self::CommittedRootChanged => formatter.write_str("committed root pool entry bytes disagreed"), - Self::PredecessorMismatch => formatter.write_str("candidate does not name the current root as its predecessor"), - Self::PredecessorRootAbsent => formatter.write_str("predecessor root pool entry is absent or exceeds the format bound"), - Self::PredecessorRootChanged => formatter.write_str("predecessor root pool entry does not decode to the manifest's selection"), - Self::RecordKindOrLength => formatter.write_str("retention record kind or length disagreed"), - Self::RecordTrailingBytes => formatter.write_str("retention record carried trailing bytes"), - Self::RecordLengthOverflow => formatter.write_str("retention record length exceeded the addressable range"), + Self::CommittedRootChanged => { + formatter.write_str("committed root pool entry bytes disagreed") + } + Self::PredecessorMismatch => { + formatter.write_str("candidate does not name the current root as its predecessor") + } + Self::PredecessorRootAbsent => formatter + .write_str("predecessor root pool entry is absent or exceeds the format bound"), + Self::PredecessorRootChanged => formatter.write_str( + "predecessor root pool entry does not decode to the manifest's selection", + ), + Self::RecordKindOrLength => { + formatter.write_str("retention record kind or length disagreed") + } + Self::RecordTrailingBytes => { + formatter.write_str("retention record carried trailing bytes") + } + Self::RecordLengthOverflow => { + formatter.write_str("retention record length exceeded the addressable range") + } } } } diff --git a/tests/adapters_layout_contract.rs b/tests/adapters_layout_contract.rs index 3fda099..3eb14f6 100644 --- a/tests/adapters_layout_contract.rs +++ b/tests/adapters_layout_contract.rs @@ -1,6 +1,9 @@ -//! The adapters module root stays a scannable manifest, not a 500-line ceiling risk. +//! Source-layout laws for the adapters tree: the module root stays a scannable +//! manifest, and files rustfmt cannot rewrap stay within the standard width. const ADAPTERS_ROOT: &str = include_str!("../src/adapters/mod.rs"); +const RETENTION_REFUSAL: &str = + include_str!("../src/adapters/retention/filesystem_retention_refusal.rs"); /// `docs/Rust Standards.md` reviews any file above 300 lines and refuses any /// above 500; the adapters root sat at 498 before its re-export surface moved. @@ -35,3 +38,18 @@ fn adapters_root_declares_modules_and_reexports_only() { assert!(allowed, "unexpected item in src/adapters/mod.rs: {line}"); } } + +/// rustfmt cannot break a string literal, so an overlong `Display` arm hides a +/// 200-column line behind a clean `cargo fmt --check`. The refusal catalogue +/// stays readable at the standard's 100-column width. +#[test] +fn retention_refusal_lines_stay_within_one_hundred_columns() { + for (index, line) in RETENTION_REFUSAL.lines().enumerate() { + let width = line.chars().count(); + assert!( + width <= 100, + "filesystem_retention_refusal.rs:{} is {width} columns wide", + index + 1 + ); + } +} From 5d583f67c5566a9333bf3fb9d939a35489682a98 Mon Sep 17 00:00:00 2001 From: James Ross Date: Mon, 7 Sep 2026 11:35:57 -0700 Subject: [PATCH 082/111] Chore: ignore the local .claude agent journal directory --- .gitignore | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitignore b/.gitignore index 0dcad44..f218b89 100644 --- a/.gitignore +++ b/.gitignore @@ -32,3 +32,6 @@ # Local mktxt repository snapshots (exceed the documentation corpus byte budget) /keep.txt + +# Local agent journals (code-smell and cool-idea logs); never tracked. +.claude/ From de3b2c231b837edf223e901e75ff3752ad5542c7 Mon Sep 17 00:00:00 2001 From: James Ross Date: Mon, 7 Sep 2026 11:45:48 -0700 Subject: [PATCH 083/111] Fix: retain publication state on one attempt that verification owns The retention authority carried seven independent Option fields across the storage-port phases. verify_current set the liveness generation before any check, so a refused verification left a coordinate behind; a stage handle from an interrupted run survived into the next run; and admit_root_namespace accepted an existing directory regardless of the expectation that verify_current had just admitted. PublicationAttempt now owns the expectation, liveness generation, namespace capability, retained pool names, and the three stage handles. verify_current discards any prior attempt first and creates a new one only when the disposition is Publish; synchronize_cleanup discards it after the last phase. Every phase obtains the attempt through one refusal, so no phase can run without an admitted verification. admit_root_namespace consults the attempt's expectation: under Absent an existing directory refuses instead of being admitted as Existing, and under Current an absent directory refuses; both carry the new NamespaceExpectationViolated refusal. Manifest and retained pool names moved onto the attempt with the state they belong to. Three laws in filesystem_retention_attempt_tests.rs failed before this change (observed with the attempt module unregistered, since the shared module cannot land ahead of its consumer under deny(warnings)) and pass now. One existing law, existing_root_stage_is_never_truncated, called write_root_stage with no prior verification, which is now unlawful by design; its setup verifies a clean namespace first and then plants the stage, and its assertions (an AlreadyExists refusal and untouched evidence bytes) are unchanged. Self-review findings R4, R5, R6, R15 (P3). Refs #78 --- CHANGELOG.md | 6 + src/adapters/retention.rs | 3 + .../retention/filesystem_retention_attempt.rs | 149 ++++++++++++++++++ .../filesystem_retention_attempt_tests.rs | 109 +++++++++++++ .../filesystem_retention_authority.rs | 72 ++------- .../retention/filesystem_retention_refusal.rs | 6 + .../retention/filesystem_retention_storage.rs | 127 +++++++++------ .../filesystem_retention_storage_tests.rs | 4 + 8 files changed, 362 insertions(+), 114 deletions(-) create mode 100644 src/adapters/retention/filesystem_retention_attempt.rs create mode 100644 src/adapters/retention/filesystem_retention_attempt_tests.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index c390868..d25d636 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -558,6 +558,12 @@ after its public API and format compatibility policies are established. Review corrections to the unreleased retention and migration work above; none of these shipped in a release. +- Every coordinate a retention publication retains between phases now lives on + one publication attempt that current-state verification creates and the next + verification or cleanup discards: a refused verification admits no later + phase, a stage handle from an interrupted run is never reused, and namespace + admission refuses a directory the claimed expectation excludes with + `RetentionCurrentStateRefusal::NamespaceExpectationViolated`. - An already-committed retention retry reopens the manifest entry and the root pool bytes the head selects and refuses absent, changed, or corrupt evidence instead of inferring the commit from head agreement alone. diff --git a/src/adapters/retention.rs b/src/adapters/retention.rs index 2fd8ded..5a9b9f8 100644 --- a/src/adapters/retention.rs +++ b/src/adapters/retention.rs @@ -15,6 +15,9 @@ mod closure_profile_error; mod closure_verifier; #[cfg(test)] mod filesystem_recovery_admission_tests; +mod filesystem_retention_attempt; +#[cfg(test)] +mod filesystem_retention_attempt_tests; mod filesystem_retention_authority; mod filesystem_retention_authority_error; #[cfg(test)] diff --git a/src/adapters/retention/filesystem_retention_attempt.rs b/src/adapters/retention/filesystem_retention_attempt.rs new file mode 100644 index 0000000..1c3449e --- /dev/null +++ b/src/adapters/retention/filesystem_retention_attempt.rs @@ -0,0 +1,149 @@ +//! This module owns the state of one admitted retention publication attempt. + +use std::io; + +use cap_std::fs::Dir; + +use super::CanonicalRetentionManifest; +use super::filesystem_retention_pool_name as pool_name; +use super::filesystem_retention_stage::{FilesystemRetentionStage, invalid_data}; +use crate::{LivenessGeneration, RetentionGenerationExpectation}; + +/// Everything one publication attempt retains between storage-port phases. +/// +/// An attempt exists only after current-state verification admits a forward +/// publication, and it is discarded when the next verification begins or +/// cleanup completes. No stage handle, pool coordinate, namespace capability, +/// or expectation from a refused or interrupted run can therefore reach a +/// later phase. +pub(super) struct PublicationAttempt { + expected: RetentionGenerationExpectation, + liveness_generation: LivenessGeneration, + namespace: Option, + retained_root: Option, + retained_manifest: Option, + root_stage: Option, + manifest_stage: Option, + head_stage: Option, +} + +/// Returns the admitted attempt or refuses because no verification admitted one. +pub(super) fn require(attempt: Option<&PublicationAttempt>) -> io::Result<&PublicationAttempt> { + attempt.ok_or_else(no_attempt) +} + +/// Mutable form of [`require`] for phases that retain state on the attempt. +pub(super) fn require_mut( + attempt: &mut Option, +) -> io::Result<&mut PublicationAttempt> { + attempt.as_mut().ok_or_else(no_attempt) +} + +fn no_attempt() -> io::Error { + invalid_data("no admitted retention publication attempt") +} + +impl PublicationAttempt { + pub(super) const fn new( + expected: RetentionGenerationExpectation, + liveness_generation: LivenessGeneration, + ) -> Self { + Self { + expected, + liveness_generation, + namespace: None, + retained_root: None, + retained_manifest: None, + root_stage: None, + manifest_stage: None, + head_stage: None, + } + } + + pub(super) const fn expected(&self) -> RetentionGenerationExpectation { + self.expected + } + + /// Names the manifest pool entry at this attempt's liveness generation. + pub(super) fn manifest_name(&self, manifest: &CanonicalRetentionManifest) -> String { + pool_name::manifest(self.liveness_generation, manifest.digest()) + } + + pub(super) fn retain_namespace(&mut self, namespace: Dir) { + self.namespace = Some(namespace); + } + + pub(super) fn namespace(&self) -> io::Result<&Dir> { + self.namespace + .as_ref() + .ok_or_else(|| invalid_data("retention root namespace was not admitted")) + } + + pub(super) fn retain_root_name(&mut self, name: String) { + self.retained_root = Some(name); + } + + pub(super) fn retained_root_name(&self) -> io::Result<&str> { + self.retained_root + .as_deref() + .ok_or_else(|| invalid_data("retention root pool coordinate was not retained")) + } + + pub(super) fn retain_manifest_name(&mut self, name: String) { + self.retained_manifest = Some(name); + } + + pub(super) fn retained_manifest_name(&self) -> io::Result<&str> { + self.retained_manifest + .as_deref() + .ok_or_else(|| invalid_data("retention manifest pool coordinate was not retained")) + } + + pub(super) fn retain_root_stage(&mut self, stage: FilesystemRetentionStage) { + self.root_stage = Some(stage); + } + + pub(super) fn root_stage(&self) -> io::Result<&FilesystemRetentionStage> { + self.root_stage + .as_ref() + .ok_or_else(|| invalid_data("retention root stage was not retained")) + } + + pub(super) fn take_root_stage(&mut self) -> io::Result { + self.root_stage + .take() + .ok_or_else(|| invalid_data("retention root stage was not retained")) + } + + pub(super) fn retain_manifest_stage(&mut self, stage: FilesystemRetentionStage) { + self.manifest_stage = Some(stage); + } + + pub(super) fn manifest_stage(&self) -> io::Result<&FilesystemRetentionStage> { + self.manifest_stage + .as_ref() + .ok_or_else(|| invalid_data("retention manifest stage was not retained")) + } + + pub(super) fn take_manifest_stage(&mut self) -> io::Result { + self.manifest_stage + .take() + .ok_or_else(|| invalid_data("retention manifest stage was not retained")) + } + + pub(super) fn retain_head_stage(&mut self, stage: FilesystemRetentionStage) { + self.head_stage = Some(stage); + } + + pub(super) fn head_stage(&self) -> io::Result<&FilesystemRetentionStage> { + self.head_stage + .as_ref() + .ok_or_else(|| invalid_data("retention head stage was not retained")) + } + + pub(super) fn take_head_stage(&mut self) -> io::Result { + self.head_stage + .take() + .ok_or_else(|| invalid_data("retention head stage was not retained")) + } +} diff --git a/src/adapters/retention/filesystem_retention_attempt_tests.rs b/src/adapters/retention/filesystem_retention_attempt_tests.rs new file mode 100644 index 0000000..b69ef55 --- /dev/null +++ b/src/adapters/retention/filesystem_retention_attempt_tests.rs @@ -0,0 +1,109 @@ +//! Filesystem retention publication attempt-state laws. + +use std::error::Error; +use std::fs; +use std::io; + +use super::filesystem_retention_test_fixture::{ + ROOT_HEX, fixture, initial_preparation, open_authority, refusal, retention_witness, + root_pool_path, +}; +use super::{ + RetentionCurrentStateRefusal, RetentionPublicationStorage, RetentionTransitionDisposition, +}; + +#[test] +fn refused_verification_admits_no_later_phase() -> Result<(), Box> { + let (sandbox, mut authority) = open_authority("filesystem-retention-attempt-refused")?; + let root_bytes = fixture(ROOT_HEX)?; + let preparation = initial_preparation(&root_bytes)?; + fs::write( + sandbox.path().join("retention").join("head.next"), + b"retained", + )?; + let before = retention_witness(sandbox.path())?; + + let error = authority + .verify_current(&preparation) + .err() + .ok_or("retained head stage was admitted")?; + assert!(matches!( + refusal(&error), + Some(RetentionCurrentStateRefusal::RetainedStage) + )); + let error = authority + .write_root_stage(preparation.candidate()) + .err() + .ok_or("root stage was written without an admitted attempt")?; + + assert_eq!(error.kind(), io::ErrorKind::InvalidData); + assert_eq!(retention_witness(sandbox.path())?, before); + drop(authority); + sandbox.remove()?; + Ok(()) +} + +#[test] +fn stale_stage_handle_does_not_survive_a_refused_verification() -> Result<(), Box> { + let (sandbox, mut authority) = open_authority("filesystem-retention-attempt-stale")?; + let root_bytes = fixture(ROOT_HEX)?; + let preparation = initial_preparation(&root_bytes)?; + assert_eq!( + authority.verify_current(&preparation)?, + RetentionTransitionDisposition::Publish + ); + authority.write_root_stage(preparation.candidate())?; + + let error = authority + .verify_current(&preparation) + .err() + .ok_or("retained root stage was admitted")?; + assert!(matches!( + refusal(&error), + Some(RetentionCurrentStateRefusal::RetainedStage) + )); + let error = authority + .synchronize_root_stage() + .err() + .ok_or("stale root stage handle was reused after a refused verification")?; + + assert_eq!(error.kind(), io::ErrorKind::InvalidData); + assert!( + sandbox.path().join("retention").join("root.next").is_file(), + "retained stage evidence must remain for recovery" + ); + drop(authority); + sandbox.remove()?; + Ok(()) +} + +#[test] +fn namespace_admission_refuses_a_directory_the_expectation_excludes() -> Result<(), Box> +{ + let (sandbox, mut authority) = open_authority("filesystem-retention-attempt-namespace")?; + let root_bytes = fixture(ROOT_HEX)?; + let preparation = initial_preparation(&root_bytes)?; + assert_eq!( + authority.verify_current(&preparation)?, + RetentionTransitionDisposition::Publish + ); + authority.write_root_stage(preparation.candidate())?; + let namespace = root_pool_path(sandbox.path(), preparation.candidate()) + .parent() + .ok_or("root pool path has no namespace parent")? + .to_path_buf(); + fs::create_dir(&namespace)?; + + let error = authority + .admit_root_namespace(preparation.candidate()) + .err() + .ok_or("namespace directory the expectation excludes was admitted")?; + + assert!(matches!( + refusal(&error), + Some(RetentionCurrentStateRefusal::NamespaceExpectationViolated) + )); + drop(authority); + sandbox.remove()?; + Ok(()) +} diff --git a/src/adapters/retention/filesystem_retention_authority.rs b/src/adapters/retention/filesystem_retention_authority.rs index e1a09d9..26a0044 100644 --- a/src/adapters/retention/filesystem_retention_authority.rs +++ b/src/adapters/retention/filesystem_retention_authority.rs @@ -5,24 +5,26 @@ use std::io; use cap_fs_ext::DirExt; use cap_std::fs::Dir; +use super::filesystem_retention_attempt::PublicationAttempt; use super::filesystem_retention_authority_error::{ FilesystemRetentionAuthorityError as Error, RetentionAuthorityDirectory as Directory, }; use super::filesystem_retention_current::{self, ObservedRetentionState}; use super::filesystem_retention_pool_name as pool_name; -use super::filesystem_retention_stage::{FilesystemRetentionStage, invalid_data}; use crate::adapters::{FilesystemVersionTwoAdmission, FilesystemWriterLock}; /// Exclusive authority to publish retention transitions on one pinned root. /// -/// The authority retains the admitted writer lock and pinned `retention`, -/// `retention/roots`, and `retention/manifests` capabilities for its entire -/// lifetime. When passed to +/// The authority retains the admitted writer lock and pinned root, +/// `retention`, `retention/roots`, and `retention/manifests` capabilities for +/// its entire lifetime. When passed to /// [`execute_retention_publication`](crate::execute_retention_publication) its /// [`RetentionPublicationStorage`](super::RetentionPublicationStorage) /// implementation executes only the forward publication protocol from an -/// exactly admitted current state. It retains opened stage handles through -/// final verification, performs synchronous capability-relative I/O, and uses +/// exactly admitted current state. Every coordinate one run retains between +/// phases, including opened stage handles, lives on one publication attempt +/// that current-state verification creates and the next verification or +/// cleanup discards. It performs synchronous capability-relative I/O and uses /// neither a network nor an asynchronous runtime. Reopening a retained stage /// prefix remains a separate recovery boundary. #[must_use] @@ -31,13 +33,7 @@ pub struct FilesystemRetentionPublicationAuthority { pub(super) retention: Dir, pub(super) roots: Dir, pub(super) manifests: Dir, - pub(super) namespace: Option, - pub(super) liveness_generation: Option, - pub(super) retained_root: Option, - pub(super) retained_manifest: Option, - pub(super) root_stage: Option, - pub(super) manifest_stage: Option, - pub(super) head_stage: Option, + pub(super) attempt: Option, _lock: FilesystemWriterLock, } @@ -69,13 +65,7 @@ impl FilesystemRetentionPublicationAuthority { retention, roots, manifests, - namespace: None, - liveness_generation: None, - retained_root: None, - retained_manifest: None, - root_stage: None, - manifest_stage: None, - head_stage: None, + attempt: None, _lock: lock, }) } @@ -93,48 +83,6 @@ impl FilesystemRetentionPublicationAuthority { pub fn observe_current(&self) -> io::Result> { filesystem_retention_current::observe(&self.retention, &self.manifests) } - - pub(super) fn namespace(&self) -> io::Result<&Dir> { - self.namespace - .as_ref() - .ok_or_else(|| invalid_data("retention root namespace was not admitted")) - } - - pub(super) fn take_root_stage(&mut self) -> io::Result { - self.root_stage - .take() - .ok_or_else(|| invalid_data("retention root stage was not retained")) - } - - pub(super) fn take_manifest_stage(&mut self) -> io::Result { - self.manifest_stage - .take() - .ok_or_else(|| invalid_data("retention manifest stage was not retained")) - } - - pub(super) fn take_head_stage(&mut self) -> io::Result { - self.head_stage - .take() - .ok_or_else(|| invalid_data("retention head stage was not retained")) - } - - pub(super) fn root_stage(&self) -> io::Result<&FilesystemRetentionStage> { - self.root_stage - .as_ref() - .ok_or_else(|| invalid_data("retention root stage was not retained")) - } - - pub(super) fn manifest_stage(&self) -> io::Result<&FilesystemRetentionStage> { - self.manifest_stage - .as_ref() - .ok_or_else(|| invalid_data("retention manifest stage was not retained")) - } - - pub(super) fn head_stage(&self) -> io::Result<&FilesystemRetentionStage> { - self.head_stage - .as_ref() - .ok_or_else(|| invalid_data("retention head stage was not retained")) - } } fn open_directory(parent: &Dir, name: &str, directory: Directory) -> Result { diff --git a/src/adapters/retention/filesystem_retention_refusal.rs b/src/adapters/retention/filesystem_retention_refusal.rs index 0bd1a6c..663868c 100644 --- a/src/adapters/retention/filesystem_retention_refusal.rs +++ b/src/adapters/retention/filesystem_retention_refusal.rs @@ -83,6 +83,9 @@ pub enum RetentionCurrentStateRefusal { PredecessorRootAbsent, /// The predecessor root pool entry does not decode to the manifest's selection. PredecessorRootChanged, + /// The candidate's namespace directory disagreed with the claimed + /// expectation when it was admitted between phases. + NamespaceExpectationViolated, /// A record's kind or length disagreed with its declaration. RecordKindOrLength, /// A record carried bytes beyond its declared length. @@ -175,6 +178,9 @@ impl fmt::Display for RetentionCurrentStateRefusal { Self::PredecessorRootChanged => formatter.write_str( "predecessor root pool entry does not decode to the manifest's selection", ), + Self::NamespaceExpectationViolated => formatter.write_str( + "namespace directory state disagreed with the claimed generation expectation", + ), Self::RecordKindOrLength => { formatter.write_str("retention record kind or length disagreed") } diff --git a/src/adapters/retention/filesystem_retention_storage.rs b/src/adapters/retention/filesystem_retention_storage.rs index 01a3fc3..d0cd0b5 100644 --- a/src/adapters/retention/filesystem_retention_storage.rs +++ b/src/adapters/retention/filesystem_retention_storage.rs @@ -5,6 +5,7 @@ use std::io; use cap_fs_ext::DirExt; use cap_std::fs::Dir; +use super::filesystem_retention_attempt::{self as attempt, PublicationAttempt}; use super::filesystem_retention_authority::FilesystemRetentionPublicationAuthority; use super::filesystem_retention_catalog; use super::filesystem_retention_current; @@ -24,7 +25,7 @@ impl RetentionPublicationStorage for FilesystemRetentionPublicationAuthority { &mut self, preparation: &RetentionPublicationPreparation<'_>, ) -> io::Result { - self.liveness_generation = Some(preparation.liveness_generation()); + self.attempt = None; require_no_retained_stage(&self.retention)?; let census = filesystem_retention_namespace::admit(&self.retention, &self.roots, &self.manifests)?; @@ -54,6 +55,10 @@ impl RetentionPublicationStorage for FilesystemRetentionPublicationAuthority { preparation.candidate(), )?; } + self.attempt = Some(PublicationAttempt::new( + preparation.expected(), + preparation.liveness_generation(), + )); } if disposition == RetentionTransitionDisposition::AlreadyCommitted { let current = current @@ -69,16 +74,20 @@ impl RetentionPublicationStorage for FilesystemRetentionPublicationAuthority { } fn write_root_stage(&mut self, root: &AdmittedRetentionRoot<'_>) -> io::Result<()> { - self.root_stage = Some(FilesystemRetentionStage::create( + let attempt = attempt::require_mut(&mut self.attempt)?; + let stage = FilesystemRetentionStage::create( &self.retention, pool_name::ROOT_STAGE, root.encoded(), - )?); + )?; + attempt.retain_root_stage(stage); Ok(()) } fn synchronize_root_stage(&mut self) -> io::Result<()> { - self.root_stage()?.synchronize(&self.retention)?; + attempt::require(self.attempt.as_ref())? + .root_stage()? + .synchronize(&self.retention)?; synchronize_directory(&self.retention) } @@ -86,15 +95,28 @@ impl RetentionPublicationStorage for FilesystemRetentionPublicationAuthority { &mut self, root: &AdmittedRetentionRoot<'_>, ) -> io::Result { + let attempt = attempt::require_mut(&mut self.attempt)?; let name = pool_name::namespace(root.root().namespace().digest()); - let admission = match self.roots.create_dir(&name) { - Ok(()) => RetentionNamespaceAdmission::Created, - Err(source) if source.kind() == io::ErrorKind::AlreadyExists => { - RetentionNamespaceAdmission::Existing + let admission = match attempt.expected() { + RetentionGenerationExpectation::Absent => match self.roots.create_dir(&name) { + Ok(()) => RetentionNamespaceAdmission::Created, + Err(source) if source.kind() == io::ErrorKind::AlreadyExists => { + return Err( + RetentionCurrentStateRefusal::NamespaceExpectationViolated.into_io() + ); + } + Err(source) => return Err(source), + }, + RetentionGenerationExpectation::Current(_) => RetentionNamespaceAdmission::Existing, + }; + let namespace = match self.roots.open_dir_nofollow(&name) { + Ok(namespace) => namespace, + Err(source) if source.kind() == io::ErrorKind::NotFound => { + return Err(RetentionCurrentStateRefusal::NamespaceExpectationViolated.into_io()); } Err(source) => return Err(source), }; - self.namespace = Some(self.roots.open_dir_nofollow(&name)?); + attempt.retain_namespace(namespace); Ok(admission) } @@ -103,36 +125,44 @@ impl RetentionPublicationStorage for FilesystemRetentionPublicationAuthority { } fn link_root(&mut self, root: &AdmittedRetentionRoot<'_>) -> io::Result<()> { + let attempt = attempt::require_mut(&mut self.attempt)?; let name = pool_name::root(root.root().generation(), root.digest()); - let namespace = self.namespace()?; - self.root_stage()?.link(&self.retention, namespace, &name)?; - self.retained_root = Some(name); + attempt + .root_stage()? + .link(&self.retention, attempt.namespace()?, &name)?; + attempt.retain_root_name(name); Ok(()) } fn synchronize_root_namespace(&mut self, _root: &AdmittedRetentionRoot<'_>) -> io::Result<()> { - synchronize_directory(self.namespace()?) + synchronize_directory(attempt::require(self.attempt.as_ref())?.namespace()?) } fn write_manifest_stage(&mut self, manifest: &CanonicalRetentionManifest) -> io::Result<()> { - self.manifest_stage = Some(FilesystemRetentionStage::create( + let attempt = attempt::require_mut(&mut self.attempt)?; + let stage = FilesystemRetentionStage::create( &self.retention, pool_name::MANIFEST_STAGE, manifest.encoded(), - )?); + )?; + attempt.retain_manifest_stage(stage); Ok(()) } fn synchronize_manifest_stage(&mut self) -> io::Result<()> { - self.manifest_stage()?.synchronize(&self.retention)?; + attempt::require(self.attempt.as_ref())? + .manifest_stage()? + .synchronize(&self.retention)?; synchronize_directory(&self.retention) } fn link_manifest(&mut self, manifest: &CanonicalRetentionManifest) -> io::Result<()> { - let name = self.manifest_name(manifest)?; - self.manifest_stage()? + let attempt = attempt::require_mut(&mut self.attempt)?; + let name = attempt.manifest_name(manifest); + attempt + .manifest_stage()? .link(&self.retention, &self.manifests, &name)?; - self.retained_manifest = Some(name); + attempt.retain_manifest_name(name); Ok(()) } @@ -141,21 +171,26 @@ impl RetentionPublicationStorage for FilesystemRetentionPublicationAuthority { } fn write_head_stage(&mut self, head: &CanonicalRetentionHead) -> io::Result<()> { - self.head_stage = Some(FilesystemRetentionStage::create( + let attempt = attempt::require_mut(&mut self.attempt)?; + let stage = FilesystemRetentionStage::create( &self.retention, pool_name::HEAD_STAGE, head.encoded(), - )?); + )?; + attempt.retain_head_stage(stage); Ok(()) } fn synchronize_head_stage(&mut self) -> io::Result<()> { - self.head_stage()?.synchronize(&self.retention)?; + attempt::require(self.attempt.as_ref())? + .head_stage()? + .synchronize(&self.retention)?; synchronize_directory(&self.retention) } fn replace_head(&mut self) -> io::Result<()> { - self.take_head_stage()? + attempt::require_mut(&mut self.attempt)? + .take_head_stage()? .replace(&self.retention, pool_name::HEAD) } @@ -164,20 +199,29 @@ impl RetentionPublicationStorage for FilesystemRetentionPublicationAuthority { } fn remove_root_stage(&mut self) -> io::Result<()> { - let stage = self.take_root_stage()?; - let name = self.retained_root_name()?; - let namespace = self.namespace()?; - stage.remove(&self.retention, namespace, &name) + let attempt = attempt::require_mut(&mut self.attempt)?; + let stage = attempt.take_root_stage()?; + stage.remove( + &self.retention, + attempt.namespace()?, + attempt.retained_root_name()?, + ) } fn remove_manifest_stage(&mut self) -> io::Result<()> { - let stage = self.take_manifest_stage()?; - let name = self.retained_manifest_name()?; - stage.remove(&self.retention, &self.manifests, &name) + let attempt = attempt::require_mut(&mut self.attempt)?; + let stage = attempt.take_manifest_stage()?; + stage.remove( + &self.retention, + &self.manifests, + attempt.retained_manifest_name()?, + ) } fn synchronize_cleanup(&mut self) -> io::Result<()> { - synchronize_directory(&self.retention) + synchronize_directory(&self.retention)?; + self.attempt = None; + Ok(()) } } @@ -197,24 +241,3 @@ fn require_no_retained_stage(retention: &Dir) -> io::Result<()> { } Ok(()) } - -impl FilesystemRetentionPublicationAuthority { - fn manifest_name(&self, manifest: &CanonicalRetentionManifest) -> io::Result { - let generation = self - .liveness_generation - .ok_or_else(|| invalid_data("selected liveness generation was not retained"))?; - Ok(pool_name::manifest(generation, manifest.digest())) - } - - fn retained_root_name(&self) -> io::Result { - self.retained_root - .clone() - .ok_or_else(|| invalid_data("retention root pool coordinate was not retained")) - } - - fn retained_manifest_name(&self) -> io::Result { - self.retained_manifest - .clone() - .ok_or_else(|| invalid_data("retention manifest pool coordinate was not retained")) - } -} diff --git a/src/adapters/retention/filesystem_retention_storage_tests.rs b/src/adapters/retention/filesystem_retention_storage_tests.rs index 368a058..5d02378 100644 --- a/src/adapters/retention/filesystem_retention_storage_tests.rs +++ b/src/adapters/retention/filesystem_retention_storage_tests.rs @@ -44,6 +44,10 @@ fn existing_root_stage_is_never_truncated() -> Result<(), Box> { let (sandbox, mut authority) = open_authority("filesystem-retention-exclusive-stage")?; let root_bytes = fixture(ROOT_HEX)?; let preparation = initial_preparation(&root_bytes)?; + assert_eq!( + authority.verify_current(&preparation)?, + super::RetentionTransitionDisposition::Publish + ); let stage = sandbox.path().join("retention").join("root.next"); fs::write(&stage, b"retained partial evidence")?; From 2c2f1ff188d32d12d9745d00eb82b84174679a82 Mon Sep 17 00:00:00 2001 From: James Ross Date: Mon, 7 Sep 2026 11:54:11 -0700 Subject: [PATCH 084/111] Fix: carry typed refusals out of retention namespace admission Namespace admission refused with seven bare InvalidData strings while every other current-state refusal already travelled as a RetentionCurrentStateRefusal source, so a caller could distinguish a superseded candidate from a corrupt head but not an unknown retention entry from a full namespace pool. The admission module now refuses with UnknownRetentionEntry, NonNamespaceEntry, NoncanonicalPoolEntry { pool }, NamespaceCapacity (also covering the two count overflows, which are beyond any ceiling by definition), and NamespaceExpectationViolated for all three expectation mismatches. Seven existing laws across the namespace, capacity, and expectation suites now downcast to the exact variant; they failed before this change and pass now. The downcast exposed that the capacity law never reached the capacity check: 4,096 orphan directories under an absent head refuse as HeadAbsentWithArtifacts first. The law now publishes generation one, fills the pool to 4,095 orphans, and attempts a 4,097th namespace, which refuses as NamespaceCapacity with an unchanged retention witness. Self-review finding R7 (P3). Refs #78 --- CHANGELOG.md | 5 +++ .../filesystem_retention_capacity_tests.rs | 30 ++++++++++---- .../filesystem_retention_expectation_tests.rs | 8 ++++ .../filesystem_retention_namespace.rs | 40 +++++++------------ .../filesystem_retention_namespace_tests.rs | 16 ++++++++ .../retention/filesystem_retention_refusal.rs | 27 ++++++++++++- 6 files changed, 91 insertions(+), 35 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d25d636..67c9fff 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -558,6 +558,11 @@ after its public API and format compatibility policies are established. Review corrections to the unreleased retention and migration work above; none of these shipped in a release. +- Retention namespace admission refuses with typed + `RetentionCurrentStateRefusal` variants (`UnknownRetentionEntry`, + `NonNamespaceEntry`, `NoncanonicalPoolEntry`, `NamespaceCapacity`, + `NamespaceExpectationViolated`) instead of bare `InvalidData` strings, so + callers can tell an unknown entry from a full namespace pool. - Every coordinate a retention publication retains between phases now lives on one publication attempt that current-state verification creates and the next verification or cleanup discards: a refused verification admits no later diff --git a/src/adapters/retention/filesystem_retention_capacity_tests.rs b/src/adapters/retention/filesystem_retention_capacity_tests.rs index 06b3a4b..4fb0d68 100644 --- a/src/adapters/retention/filesystem_retention_capacity_tests.rs +++ b/src/adapters/retention/filesystem_retention_capacity_tests.rs @@ -7,12 +7,12 @@ use std::io; use std::path::Path; use super::filesystem_retention_test_fixture::{ - ROOT_HEX, fixture, initial_preparation, open_authority, retention_witness, - successor_preparation, successor_root, + ROOT_HEX, fixture, initial_preparation, initial_root, new_namespace_preparation, + open_authority, refusal, retention_witness, successor_preparation, successor_root, }; use super::{ - AdmittedRetentionManifest, AdmittedRetentionRoot, RetentionPublicationStorage, - RetentionTransitionDisposition, + AdmittedRetentionManifest, AdmittedRetentionRoot, RetentionCurrentStateRefusal, + RetentionPublicationStorage, RetentionTransitionDisposition, }; use crate::{RetentionManifest, execute_retention_publication}; @@ -20,17 +20,31 @@ use crate::{RetentionManifest, execute_retention_publication}; fn a_full_namespace_pool_refuses_a_new_namespace_before_staging() -> Result<(), Box> { let (sandbox, mut authority) = open_authority("filesystem-retention-capacity-full")?; let root_bytes = fixture(ROOT_HEX)?; - let candidate = AdmittedRetentionRoot::decode(&root_bytes)?; - let own = namespace_hex(&candidate); - create_orphans(sandbox.path(), RetentionManifest::MAXIMUM_ENTRY_COUNT, &own)?; + let _published = + execute_retention_publication(&mut authority, &initial_preparation(&root_bytes)?)?; + let current = authority + .observe_current()? + .ok_or("published retention head was not observed")?; + let current_root = AdmittedRetentionRoot::decode(&root_bytes)?; + let current_manifest = AdmittedRetentionManifest::decode(current.manifest_bytes())?; + create_orphans( + sandbox.path(), + RetentionManifest::MAXIMUM_ENTRY_COUNT - 1, + &namespace_hex(¤t_root), + )?; + let candidate = initial_root(b"the-4097th-namespace", ¤t_root)?; + let preparation = new_namespace_preparation(¤t_manifest, candidate.encoded())?; let before = retention_witness(sandbox.path())?; - let preparation = initial_preparation(&root_bytes)?; let error = RetentionPublicationStorage::verify_current(&mut authority, &preparation) .err() .ok_or("a 4,097th retention namespace was unexpectedly admitted")?; assert_eq!(error.kind(), io::ErrorKind::InvalidData); + assert!(matches!( + refusal(&error), + Some(RetentionCurrentStateRefusal::NamespaceCapacity) + )); assert_eq!(retention_witness(sandbox.path())?, before); drop(authority); sandbox.remove()?; diff --git a/src/adapters/retention/filesystem_retention_expectation_tests.rs b/src/adapters/retention/filesystem_retention_expectation_tests.rs index 6d9b47d..f413a25 100644 --- a/src/adapters/retention/filesystem_retention_expectation_tests.rs +++ b/src/adapters/retention/filesystem_retention_expectation_tests.rs @@ -66,6 +66,10 @@ fn absent_expectation_refuses_an_orphan_directory_for_a_new_namespace() -> Resul .ok_or("orphan namespace directory was unexpectedly admitted for an Absent expectation")?; assert_eq!(error.kind(), io::ErrorKind::InvalidData); + assert!(matches!( + super::filesystem_retention_test_fixture::refusal(&error), + Some(super::RetentionCurrentStateRefusal::NamespaceExpectationViolated) + )); assert_eq!(retention_witness(sandbox.path())?, before); drop(authority); sandbox.remove()?; @@ -97,6 +101,10 @@ fn current_expectation_refuses_when_the_namespace_directory_is_absent() -> Resul .ok_or("successor over an absent namespace directory was unexpectedly admitted")?; assert_eq!(error.kind(), io::ErrorKind::InvalidData); + assert!(matches!( + super::filesystem_retention_test_fixture::refusal(&error), + Some(super::RetentionCurrentStateRefusal::NamespaceExpectationViolated) + )); drop(authority); sandbox.remove()?; Ok(()) diff --git a/src/adapters/retention/filesystem_retention_namespace.rs b/src/adapters/retention/filesystem_retention_namespace.rs index 7c5ca7a..546bbee 100644 --- a/src/adapters/retention/filesystem_retention_namespace.rs +++ b/src/adapters/retention/filesystem_retention_namespace.rs @@ -7,8 +7,8 @@ use cap_fs_ext::DirExt; use cap_std::fs::Dir; use super::AdmittedRetentionRoot; +use super::RetentionCurrentStateRefusal as Refusal; use super::filesystem_retention_pool_name as pool_name; -use super::filesystem_retention_stage::invalid_data; use crate::{RetentionGenerationExpectation, RetentionManifest}; const CANONICAL_ENTRIES: [&str; 3] = [pool_name::HEAD, pool_name::ROOTS, pool_name::MANIFESTS]; @@ -49,7 +49,7 @@ pub(super) fn admit( for entry in retention.entries()? { let name = entry?.file_name(); if !CANONICAL_ENTRIES.iter().any(|canonical| name == *canonical) { - return Err(invalid_data("retention namespace carries an unknown entry")); + return Err(Refusal::UnknownRetentionEntry.into_io()); } } let mut namespace_count = 0_u32; @@ -57,17 +57,15 @@ pub(super) fn admit( let entry = entry?; let name = entry.file_name(); if !is_lower_hex(&name, DIGEST_HEX) || !entry.metadata()?.is_dir() { - return Err(invalid_data( - "retention roots carries a non-namespace entry", - )); + return Err(Refusal::NonNamespaceEntry.into_io()); } namespace_count = namespace_count .checked_add(1) - .ok_or_else(|| invalid_data("retention namespace count overflowed"))?; + .ok_or_else(|| Refusal::NamespaceCapacity.into_io())?; let namespace = roots.open_dir_nofollow(&name)?; - let _roots = admit_pool(&namespace, ROOT_SUFFIX, "retention root pool")?; + let _roots = admit_pool(&namespace, ROOT_SUFFIX, "root pool")?; } - let manifest_count = admit_pool(manifests, MANIFEST_SUFFIX, "retention manifest pool")?; + let manifest_count = admit_pool(manifests, MANIFEST_SUFFIX, "manifest pool")?; Ok(RetentionNamespaceCensus { namespace_count, manifest_count, @@ -93,9 +91,7 @@ pub(super) fn admit_capacity( if census.namespace_count < RetentionManifest::MAXIMUM_ENTRY_COUNT { Ok(()) } else { - Err(invalid_data( - "retention namespace pool is at its maximum count", - )) + Err(Refusal::NamespaceCapacity.into_io()) } } Err(source) => Err(source), @@ -122,31 +118,23 @@ pub(super) fn admit_expectation( match (expected, observed) { (RetentionGenerationExpectation::Absent, None) | (RetentionGenerationExpectation::Current(_), Some(true)) => Ok(()), - (RetentionGenerationExpectation::Absent, Some(_)) => Err(invalid_data( - "namespace directory exists although the namespace is expected absent", - )), - (RetentionGenerationExpectation::Current(_), None) => Err(invalid_data( - "namespace directory is absent although a current generation is expected", - )), - (RetentionGenerationExpectation::Current(_), Some(false)) => { - Err(invalid_data("namespace entry is not a directory")) + (RetentionGenerationExpectation::Absent, Some(_)) + | (RetentionGenerationExpectation::Current(_), None | Some(false)) => { + Err(Refusal::NamespaceExpectationViolated.into_io()) } } } -fn admit_pool(pool: &Dir, suffix: &str, label: &'static str) -> io::Result { +fn admit_pool(directory: &Dir, suffix: &str, pool: &'static str) -> io::Result { let mut count = 0_u32; - for entry in pool.entries()? { + for entry in directory.entries()? { let entry = entry?; if !is_pool_name(&entry.file_name(), suffix) || !entry.metadata()?.is_file() { - return Err(io::Error::new( - io::ErrorKind::InvalidData, - format!("{label} carries a noncanonical entry"), - )); + return Err(Refusal::NoncanonicalPoolEntry { pool }.into_io()); } count = count .checked_add(1) - .ok_or_else(|| invalid_data("retention pool entry count overflowed"))?; + .ok_or_else(|| Refusal::NamespaceCapacity.into_io())?; } Ok(count) } diff --git a/src/adapters/retention/filesystem_retention_namespace_tests.rs b/src/adapters/retention/filesystem_retention_namespace_tests.rs index 2d8d1e8..f1c2966 100644 --- a/src/adapters/retention/filesystem_retention_namespace_tests.rs +++ b/src/adapters/retention/filesystem_retention_namespace_tests.rs @@ -25,6 +25,10 @@ fn unknown_retention_entry_refuses_before_any_stage_is_written() -> Result<(), B .ok_or("unknown retention entry was unexpectedly admitted")?; assert_eq!(error.kind(), io::ErrorKind::InvalidData); + assert!(matches!( + super::filesystem_retention_test_fixture::refusal(&error), + Some(super::RetentionCurrentStateRefusal::UnknownRetentionEntry) + )); assert_eq!(retention_witness(sandbox.path())?, before); drop(authority); sandbox.remove()?; @@ -49,6 +53,10 @@ fn non_digest_root_namespace_directory_refuses() -> Result<(), Box> { .ok_or("non-digest namespace directory was unexpectedly admitted")?; assert_eq!(error.kind(), io::ErrorKind::InvalidData); + assert!(matches!( + super::filesystem_retention_test_fixture::refusal(&error), + Some(super::RetentionCurrentStateRefusal::NonNamespaceEntry) + )); drop(authority); sandbox.remove()?; Ok(()) @@ -73,6 +81,10 @@ fn malformed_manifest_pool_name_refuses() -> Result<(), Box> { .ok_or("malformed manifest pool name was unexpectedly admitted")?; assert_eq!(error.kind(), io::ErrorKind::InvalidData); + assert!(matches!( + super::filesystem_retention_test_fixture::refusal(&error), + Some(super::RetentionCurrentStateRefusal::NoncanonicalPoolEntry { .. }) + )); drop(authority); sandbox.remove()?; Ok(()) @@ -99,6 +111,10 @@ fn uppercase_root_pool_name_refuses() -> Result<(), Box> { .ok_or("uppercase root pool name was unexpectedly admitted")?; assert_eq!(error.kind(), io::ErrorKind::InvalidData); + assert!(matches!( + super::filesystem_retention_test_fixture::refusal(&error), + Some(super::RetentionCurrentStateRefusal::NoncanonicalPoolEntry { .. }) + )); drop(authority); sandbox.remove()?; Ok(()) diff --git a/src/adapters/retention/filesystem_retention_refusal.rs b/src/adapters/retention/filesystem_retention_refusal.rs index 663868c..0f7173c 100644 --- a/src/adapters/retention/filesystem_retention_refusal.rs +++ b/src/adapters/retention/filesystem_retention_refusal.rs @@ -83,8 +83,21 @@ pub enum RetentionCurrentStateRefusal { PredecessorRootAbsent, /// The predecessor root pool entry does not decode to the manifest's selection. PredecessorRootChanged, + /// The `retention` directory carries an entry outside `HEAD`, `roots`, + /// and `manifests`. + UnknownRetentionEntry, + /// A `retention/roots` entry is not a 64-lowercase-hex directory. + NonNamespaceEntry, + /// A pool entry is not a regular `-` file with the + /// pool's canonical suffix. + NoncanonicalPoolEntry { + /// The pool that carries the entry. + pool: &'static str, + }, + /// Admitting the candidate would exceed the namespace or pool ceiling. + NamespaceCapacity, /// The candidate's namespace directory disagreed with the claimed - /// expectation when it was admitted between phases. + /// expectation, at verification or when it was admitted between phases. NamespaceExpectationViolated, /// A record's kind or length disagreed with its declaration. RecordKindOrLength, @@ -178,6 +191,18 @@ impl fmt::Display for RetentionCurrentStateRefusal { Self::PredecessorRootChanged => formatter.write_str( "predecessor root pool entry does not decode to the manifest's selection", ), + Self::UnknownRetentionEntry => { + formatter.write_str("retention namespace carries an unknown entry") + } + Self::NonNamespaceEntry => { + formatter.write_str("retention roots carries a non-namespace entry") + } + Self::NoncanonicalPoolEntry { pool } => { + write!(formatter, "retention {pool} carries a noncanonical entry") + } + Self::NamespaceCapacity => { + formatter.write_str("retention namespace or pool count would exceed its ceiling") + } Self::NamespaceExpectationViolated => formatter.write_str( "namespace directory state disagreed with the claimed generation expectation", ), From 885db63ba857e4025f1cffd35caa98e7493cad19 Mon Sep 17 00:00:00 2001 From: James Ross Date: Mon, 7 Sep 2026 12:02:04 -0700 Subject: [PATCH 085/111] Refactor: take fixed record lengths from the decoders that define them Five readers restated the fixed lengths of records whose decoders already publish them: the retention head (144), the version-one publication head (128), the format marker (96), and the migration intent and receipt (256). A format revision that changed a decoder's ENCODED_LENGTH would have left a reader accepting the old length. Each reader now names the decoder constant. store_migration re-exports the marker, intent, and receipt lengths for the version-two record reader (their decoders widen the constant to pub(in crate::adapters) for that purpose), the migration namespace admission reaches the publication head length through the adapters tree, and require_regular takes a usize and performs the one checked conversion to the metadata's u64. A contract test asserts that no code line in the four record-reading modules carries a bare 144, 128, 96, or 256; it failed before this change and passes now. Self-review finding R8 (P3). Refs #78 --- CHANGELOG.md | 4 ++ .../filesystem_version_two_records.rs | 11 ++--- .../retention/filesystem_retention_catalog.rs | 2 +- .../retention/filesystem_retention_current.rs | 2 +- src/adapters/store_migration.rs | 3 ++ .../filesystem_migration_namespace.rs | 12 ++--- ...ilesystem_migration_namespace_directory.rs | 8 +++- .../store_migration/format_marker_decoder.rs | 2 +- .../migration_intent_format.rs | 2 +- .../migration_receipt_format.rs | 2 +- tests/adapters_layout_contract.rs | 45 ++++++++++++++++++- 11 files changed, 75 insertions(+), 18 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 67c9fff..129c3bd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -242,6 +242,10 @@ after its public API and format compatibility policies are established. ### Changed +- The retention head, catalog head, format marker, migration intent, and + migration receipt readers take their fixed record lengths from the decoders + that define those formats instead of restating the numbers; a contract test + keeps the record-reading modules free of bare length literals. - Version-one reopen refuses a migrated root, and `admit_version_two` owns the separate version-2 namespace boundary. Version-one recovery adapters refuse version-two residue at the store root before pinning any pool: a format diff --git a/src/adapters/filesystem_version_two_records.rs b/src/adapters/filesystem_version_two_records.rs index e31b068..434a15e 100644 --- a/src/adapters/filesystem_version_two_records.rs +++ b/src/adapters/filesystem_version_two_records.rs @@ -5,6 +5,9 @@ use std::io::{self, Read}; use cap_fs_ext::{FollowSymlinks, OpenOptionsFollowExt, OpenOptionsSyncExt}; use cap_std::fs::{Dir, OpenOptions}; +use super::store_migration::{ + FORMAT_MARKER_LENGTH, MIGRATION_INTENT_LENGTH, MIGRATION_RECEIPT_LENGTH, +}; use super::{ AdmittedStoreFormatMarker, AdmittedStoreMigrationIntent, AdmittedStoreMigrationReceipt, }; @@ -12,8 +15,6 @@ use super::{ const MARKER_NAME: &str = "FORMAT"; const INTENT_NAME: &str = "migration.intent"; const RECEIPT_NAME: &str = "migration.receipt"; -const MARKER_LENGTH: usize = 96; -const RECORD_LENGTH: usize = 256; /// Root identity coordinates bound into an admitted `migration.intent`. #[derive(Clone, Copy, Debug, Eq, PartialEq)] @@ -54,9 +55,9 @@ impl BoundRootIdentity { /// version-two root must not be returned before this admission succeeds. The /// intent's bound root coordinates are returned for identity comparison. pub(super) fn admit(root: &Dir) -> io::Result { - let marker_bytes = read_exact(root, MARKER_NAME, MARKER_LENGTH)?; - let intent_bytes = read_exact(root, INTENT_NAME, RECORD_LENGTH)?; - let receipt_bytes = read_exact(root, RECEIPT_NAME, RECORD_LENGTH)?; + let marker_bytes = read_exact(root, MARKER_NAME, FORMAT_MARKER_LENGTH)?; + let intent_bytes = read_exact(root, INTENT_NAME, MIGRATION_INTENT_LENGTH)?; + let receipt_bytes = read_exact(root, RECEIPT_NAME, MIGRATION_RECEIPT_LENGTH)?; let marker = AdmittedStoreFormatMarker::decode(&marker_bytes) .map_err(|source| invalid_data(MARKER_NAME, &source))?; let intent = AdmittedStoreMigrationIntent::decode(&intent_bytes) diff --git a/src/adapters/retention/filesystem_retention_catalog.rs b/src/adapters/retention/filesystem_retention_catalog.rs index 850bdec..c7e8a12 100644 --- a/src/adapters/retention/filesystem_retention_catalog.rs +++ b/src/adapters/retention/filesystem_retention_catalog.rs @@ -9,7 +9,7 @@ use super::{RetentionCurrentStateRefusal, RetentionPublicationPreparation}; use crate::adapters::ChecksummedPublicationHead; const HEAD_NAME: &str = "HEAD"; -const HEAD_LENGTH: usize = 128; +const HEAD_LENGTH: usize = crate::adapters::publication_head_decoder::ENCODED_LENGTH; /// Requires the store's catalog head to name the catalog the closure was verified against. /// diff --git a/src/adapters/retention/filesystem_retention_current.rs b/src/adapters/retention/filesystem_retention_current.rs index f349cd9..ab98c92 100644 --- a/src/adapters/retention/filesystem_retention_current.rs +++ b/src/adapters/retention/filesystem_retention_current.rs @@ -13,7 +13,7 @@ use super::{ }; use crate::RetentionGenerationExpectation; -const HEAD_LENGTH: usize = 144; +const HEAD_LENGTH: usize = super::head_decoder::ENCODED_LENGTH; /// Exact bytes of one published retention head and the manifest it selects. /// diff --git a/src/adapters/store_migration.rs b/src/adapters/store_migration.rs index 2df8d0b..b5fa200 100644 --- a/src/adapters/store_migration.rs +++ b/src/adapters/store_migration.rs @@ -107,6 +107,7 @@ pub use filesystem_migration_authority_error::{ }; pub use format_definition_digest::StoreFormatDefinitionDigest; pub use format_marker_decode_error::StoreFormatMarkerDecodeError; +pub(super) use format_marker_decoder::ENCODED_LENGTH as FORMAT_MARKER_LENGTH; pub use format_marker_digest::StoreFormatMarkerDigest; pub use immutable_pool_inventory_digest::ImmutablePoolInventoryDigest; pub use initial_gc_state_digest::InitialGcStateDigest; @@ -115,6 +116,7 @@ pub use migration_error::StoreMigrationError; pub use migration_execution::execute_store_migration; pub use migration_intent_decode_error::StoreMigrationIntentDecodeError; pub use migration_intent_digest::StoreMigrationIntentDigest; +pub(super) use migration_intent_format::ENCODED_LENGTH as MIGRATION_INTENT_LENGTH; pub use migration_inventory_entry::StoreMigrationInventoryEntry; pub use migration_inventory_entry_count::StoreMigrationInventoryEntryCount; pub use migration_inventory_entry_count_error::StoreMigrationInventoryEntryCountError; @@ -122,6 +124,7 @@ pub use migration_inventory_error::StoreMigrationInventoryError; pub use migration_inventory_hasher::StoreMigrationInventoryHasher; pub use migration_phase::StoreMigrationPhase; pub use migration_receipt_decode_error::StoreMigrationReceiptDecodeError; +pub(super) use migration_receipt_format::ENCODED_LENGTH as MIGRATION_RECEIPT_LENGTH; pub use migration_storage::StoreMigrationStorage; pub use migration_synchronization_mask::MigrationSynchronizationMask; pub use store_identifier::StoreIdentifier; diff --git a/src/adapters/store_migration/filesystem_migration_namespace.rs b/src/adapters/store_migration/filesystem_migration_namespace.rs index 04b9279..9b294eb 100644 --- a/src/adapters/store_migration/filesystem_migration_namespace.rs +++ b/src/adapters/store_migration/filesystem_migration_namespace.rs @@ -10,6 +10,8 @@ use super::filesystem_migration_namespace_directory::{ require_regular, required_directory, }; use super::filesystem_migration_reader_fence; +use super::{FORMAT_MARKER_LENGTH, MIGRATION_INTENT_LENGTH, MIGRATION_RECEIPT_LENGTH}; +use crate::adapters::publication_head_decoder::ENCODED_LENGTH as PUBLICATION_HEAD_LENGTH; const WRITER_LOCK: &str = "writer.lock"; const STAGING: &str = "staging"; @@ -137,13 +139,13 @@ pub(super) fn verify_marker_view(root: &Dir) -> io::Result<()> { pub(super) fn verify_marker_contents(root: &Dir) -> io::Result<()> { verify_prefix_directories(root)?; - require_regular(root, "FORMAT", Some(96)) + require_regular(root, "FORMAT", Some(FORMAT_MARKER_LENGTH)) } pub(super) fn verify_receipt_view(root: &Dir) -> io::Result<()> { verify_prefix_directories(root)?; - require_regular(root, "FORMAT", Some(96))?; - require_regular(root, "migration.receipt", Some(256))?; + require_regular(root, "FORMAT", Some(FORMAT_MARKER_LENGTH))?; + require_regular(root, "migration.receipt", Some(MIGRATION_RECEIPT_LENGTH))?; require_exact_membership(root, &RECEIPT_ROOT) } @@ -177,8 +179,8 @@ fn require_v1_and_intent(root: &Dir) -> io::Result<()> { require_directory(root, STAGING)?; require_directory(root, SEGMENTS)?; require_directory(root, CATALOGS)?; - require_regular(root, HEAD, Some(128))?; - require_regular(root, INTENT, Some(256)) + require_regular(root, HEAD, Some(PUBLICATION_HEAD_LENGTH))?; + require_regular(root, INTENT, Some(MIGRATION_INTENT_LENGTH)) } fn verify_prefix_directories(root: &Dir) -> io::Result<()> { diff --git a/src/adapters/store_migration/filesystem_migration_namespace_directory.rs b/src/adapters/store_migration/filesystem_migration_namespace_directory.rs index adb46d2..4368c85 100644 --- a/src/adapters/store_migration/filesystem_migration_namespace_directory.rs +++ b/src/adapters/store_migration/filesystem_migration_namespace_directory.rs @@ -125,9 +125,13 @@ pub(super) fn require_directory(parent: &Dir, name: &str) -> io::Result<()> { } } -pub(super) fn require_regular(parent: &Dir, name: &str, length: Option) -> io::Result<()> { +pub(super) fn require_regular(parent: &Dir, name: &str, length: Option) -> io::Result<()> { let metadata = parent.symlink_metadata(name)?; - if metadata.is_file() && length.is_none_or(|expected| metadata.len() == expected) { + let expected = length + .map(u64::try_from) + .transpose() + .map_err(|_source| ambiguous("required migration file length exceeded u64"))?; + if metadata.is_file() && expected.is_none_or(|expected| metadata.len() == expected) { Ok(()) } else { Err(ambiguous( diff --git a/src/adapters/store_migration/format_marker_decoder.rs b/src/adapters/store_migration/format_marker_decoder.rs index 75ca8f1..075d034 100644 --- a/src/adapters/store_migration/format_marker_decoder.rs +++ b/src/adapters/store_migration/format_marker_decoder.rs @@ -6,7 +6,7 @@ use super::{ }; use crate::RetentionManifest; -pub(super) const ENCODED_LENGTH: usize = 96; +pub(in crate::adapters) const ENCODED_LENGTH: usize = 96; pub(super) const CHECKSUM_OFFSET: usize = 64; pub(super) const MAGIC: [u8; 16] = *b"KEEP:STORE:V2\0\0\0"; pub(super) const VERSION: u16 = 2; diff --git a/src/adapters/store_migration/migration_intent_format.rs b/src/adapters/store_migration/migration_intent_format.rs index 0077bcb..9f0de80 100644 --- a/src/adapters/store_migration/migration_intent_format.rs +++ b/src/adapters/store_migration/migration_intent_format.rs @@ -7,7 +7,7 @@ use super::{ use crate::{CatalogDigest, CatalogGeneration, CatalogLength}; pub(super) const CHECKSUM_OFFSET: usize = 224; -pub(super) const ENCODED_LENGTH: usize = 256; +pub(in crate::adapters) const ENCODED_LENGTH: usize = 256; pub(super) const MAGIC: [u8; 16] = *b"KEEP:MIG:INT2\0\0\0"; pub(super) const RECORD_LENGTH: u16 = 256; pub(super) const VERSION: u16 = 2; diff --git a/src/adapters/store_migration/migration_receipt_format.rs b/src/adapters/store_migration/migration_receipt_format.rs index 392ff3d..72841b7 100644 --- a/src/adapters/store_migration/migration_receipt_format.rs +++ b/src/adapters/store_migration/migration_receipt_format.rs @@ -1,7 +1,7 @@ //! This boundary module owns shared migration-receipt framing and integrity. pub(super) const CHECKSUM_OFFSET: usize = 224; -pub(super) const ENCODED_LENGTH: usize = 256; +pub(in crate::adapters) const ENCODED_LENGTH: usize = 256; pub(super) const MAGIC: [u8; 16] = *b"KEEP:MIG:REC2\0\0\0"; pub(super) const RECORD_LENGTH: u16 = 256; pub(super) const VERSION: u16 = 2; diff --git a/tests/adapters_layout_contract.rs b/tests/adapters_layout_contract.rs index 3eb14f6..e992147 100644 --- a/tests/adapters_layout_contract.rs +++ b/tests/adapters_layout_contract.rs @@ -1,5 +1,6 @@ //! Source-layout laws for the adapters tree: the module root stays a scannable -//! manifest, and files rustfmt cannot rewrap stay within the standard width. +//! manifest, files rustfmt cannot rewrap stay within the standard width, and +//! record readers take their fixed lengths from the decoders that define them. const ADAPTERS_ROOT: &str = include_str!("../src/adapters/mod.rs"); const RETENTION_REFUSAL: &str = @@ -53,3 +54,45 @@ fn retention_refusal_lines_stay_within_one_hundred_columns() { ); } } + +const RECORD_READERS: [(&str, &str); 4] = [ + ( + "src/adapters/retention/filesystem_retention_current.rs", + include_str!("../src/adapters/retention/filesystem_retention_current.rs"), + ), + ( + "src/adapters/retention/filesystem_retention_catalog.rs", + include_str!("../src/adapters/retention/filesystem_retention_catalog.rs"), + ), + ( + "src/adapters/filesystem_version_two_records.rs", + include_str!("../src/adapters/filesystem_version_two_records.rs"), + ), + ( + "src/adapters/store_migration/filesystem_migration_namespace.rs", + include_str!("../src/adapters/store_migration/filesystem_migration_namespace.rs"), + ), +]; + +/// Fixed record lengths belong to the decoders that define the formats. A +/// reader that restates `144`, `128`, `96`, or `256` can drift from them, so +/// only comments may carry those numbers in the record-reading modules. +#[test] +fn record_readers_take_lengths_from_the_decoders() { + for (path, source) in RECORD_READERS { + for (index, line) in source.lines().enumerate() { + let code = line.trim_start(); + if code.starts_with("//") { + continue; + } + let restated = code + .split(|character: char| !character.is_ascii_alphanumeric() && character != '_') + .any(|token| matches!(token, "144" | "128" | "96" | "256")); + assert!( + !restated, + "{path}:{} restates a record length instead of naming its decoder: {line}", + index + 1 + ); + } + } +} From ac8c22a6dca4020e66c1a5c562390b90f065ed39 Mon Sep 17 00:00:00 2001 From: James Ross Date: Mon, 7 Sep 2026 12:10:14 -0700 Subject: [PATCH 086/111] Refactor: decode the observed retention state once ObservedRetentionState held only bytes, so disposition decoded the head a second time and both verify_committed and verify_predecessor decoded the manifest again, each mapping a decode failure that observe() had already excluded by cross-check. The storage port then matched on a plain RetentionTransitionDisposition and needed an arm for AlreadyCommitted with no observed head, a state disposition can never produce. The observed state now carries the decoded RetentionHead and RetentionManifest next to the exact bytes, exposed through head() and manifest(); the byte accessors remain for callers that re-admit the records. disposition returns ObservedDisposition, whose Committed variant carries the observed state it was derived from, so verify_current matches on it and the unreachable arm is gone. No refusal changed; the retention laws cover the paths. Self-review findings R9 and R12 (P3). Refs #78 --- CHANGELOG.md | 5 ++ .../retention/filesystem_retention_current.rs | 84 +++++++++++++------ .../retention/filesystem_retention_storage.rs | 65 +++++++------- 3 files changed, 96 insertions(+), 58 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 129c3bd..eef8f1e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -242,6 +242,11 @@ after its public API and format compatibility policies are established. ### Changed +- `ObservedRetentionState` carries the decoded head and manifest alongside + their exact bytes, so disposition, already-committed verification, and + predecessor verification decode each record once; the disposition names the + observed state it was derived from, which removes an unreachable + already-committed-without-a-head refusal. - The retention head, catalog head, format marker, migration intent, and migration receipt readers take their fixed record lengths from the decoders that define those formats instead of restating the numbers; a contract test diff --git a/src/adapters/retention/filesystem_retention_current.rs b/src/adapters/retention/filesystem_retention_current.rs index ab98c92..0488dca 100644 --- a/src/adapters/retention/filesystem_retention_current.rs +++ b/src/adapters/retention/filesystem_retention_current.rs @@ -11,26 +11,29 @@ use super::{ AdmittedRetentionManifest, AdmittedRetentionRoot, ChecksummedRetentionHead, RetentionCurrentStateRefusal, RetentionPublicationPreparation, RetentionTransitionDisposition, }; -use crate::RetentionGenerationExpectation; +use crate::{RetentionGenerationExpectation, RetentionHead, RetentionManifest}; const HEAD_LENGTH: usize = super::head_decoder::ENCODED_LENGTH; -/// Exact bytes of one published retention head and the manifest it selects. +/// One published retention head and the manifest it selects, bytes and values. /// /// Both records were reopened without following links, bounded by their -/// declared lengths, decoded, and cross-checked: the manifest's canonical -/// digest equals the digest the head names. Callers decode the bytes with -/// [`ChecksummedRetentionHead`] and [`AdmittedRetentionManifest`] to plan the -/// next transition. +/// declared lengths, decoded exactly once, and cross-checked: the manifest's +/// canonical digest, generation, and predecessor equal what the head names. +/// Callers plan the next transition from the decoded values or re-admit the +/// exact bytes with [`ChecksummedRetentionHead`] and +/// [`AdmittedRetentionManifest`]. #[must_use] #[derive(Debug)] pub struct ObservedRetentionState { head: Box<[u8]>, manifest: Box<[u8]>, + decoded_head: RetentionHead, + decoded_manifest: RetentionManifest, } impl ObservedRetentionState { - /// Returns the exact 144 published head bytes. + /// Returns the exact published head bytes. pub const fn head_bytes(&self) -> &[u8] { &self.head } @@ -39,6 +42,34 @@ impl ObservedRetentionState { pub const fn manifest_bytes(&self) -> &[u8] { &self.manifest } + + /// Returns the decoded head coordinate. + pub const fn head(&self) -> &RetentionHead { + &self.decoded_head + } + + /// Returns the decoded manifest the head selects. + pub const fn manifest(&self) -> &RetentionManifest { + &self.decoded_manifest + } +} + +/// The verified relationship between one preparation and the observed state. +#[derive(Clone, Copy)] +pub(super) enum ObservedDisposition<'state> { + /// The observed head already names the prepared successor. + Committed(&'state ObservedRetentionState), + /// The prepared successor advances the observed state. + Publish, +} + +impl ObservedDisposition<'_> { + pub(super) const fn transition(self) -> RetentionTransitionDisposition { + match self { + Self::Committed(_) => RetentionTransitionDisposition::AlreadyCommitted, + Self::Publish => RetentionTransitionDisposition::Publish, + } + } } /// Reads and cross-verifies `retention/HEAD` and its selected manifest. @@ -71,7 +102,14 @@ pub(super) fn observe( if admitted.manifest().predecessor() != selected.predecessor() { return Err(RetentionCurrentStateRefusal::HeadPredecessorDisagreed.into_io()); } - Ok(Some(ObservedRetentionState { head, manifest })) + let decoded_head = *selected; + let decoded_manifest = admitted.manifest().clone(); + Ok(Some(ObservedRetentionState { + head, + manifest, + decoded_head, + decoded_manifest, + })) } /// Compares one preparation against the observed current state. @@ -80,27 +118,27 @@ pub(super) fn observe( /// equals the prepared successor is `AlreadyCommitted`. Otherwise the head /// must be the exact predecessor the prepared successor names, or the /// candidate is superseded and refuses. -pub(super) fn disposition( +pub(super) fn disposition<'state>( preparation: &RetentionPublicationPreparation<'_>, - current: Option<&ObservedRetentionState>, -) -> io::Result { + current: Option<&'state ObservedRetentionState>, +) -> io::Result> { let Some(current) = current else { return match preparation.expected() { - RetentionGenerationExpectation::Absent => require_initial_publication(preparation), + RetentionGenerationExpectation::Absent => { + require_initial_publication(preparation).map(|()| ObservedDisposition::Publish) + } RetentionGenerationExpectation::Current(_) => { Err(RetentionCurrentStateRefusal::ExpectedCurrentOverAbsentHead.into_io()) } }; }; - let head = ChecksummedRetentionHead::decode(current.head_bytes()) - .map_err(|source| RetentionCurrentStateRefusal::HeadRefused { source }.into_io())?; - let head = head.head(); + let head = current.head(); let committed = ( preparation.liveness_generation(), preparation.manifest_digest(), ); if (head.generation(), head.manifest_digest()) == committed { - return Ok(RetentionTransitionDisposition::AlreadyCommitted); + return Ok(ObservedDisposition::Committed(current)); } let publication = preparation .publication() @@ -114,7 +152,7 @@ pub(super) fn disposition( if prepared.head().predecessor() == Some(head.manifest_digest()) && prepared.head().generation() == expected_generation { - Ok(RetentionTransitionDisposition::Publish) + Ok(ObservedDisposition::Publish) } else { Err(RetentionCurrentStateRefusal::Superseded { current_generation: head.generation(), @@ -135,10 +173,8 @@ pub(super) fn verify_committed( current: &ObservedRetentionState, candidate: &AdmittedRetentionRoot<'_>, ) -> io::Result<()> { - let manifest = AdmittedRetentionManifest::decode(current.manifest_bytes()) - .map_err(|source| RetentionCurrentStateRefusal::ManifestRefused { source }.into_io())?; let namespace = candidate.root().namespace().digest(); - let entries = manifest.manifest().entries(); + let entries = current.manifest().entries(); let entry = entries .binary_search_by_key(&namespace, |entry| entry.namespace()) .ok() @@ -173,10 +209,8 @@ pub(super) fn verify_predecessor( current: &ObservedRetentionState, candidate: &AdmittedRetentionRoot<'_>, ) -> io::Result<()> { - let manifest = AdmittedRetentionManifest::decode(current.manifest_bytes()) - .map_err(|source| RetentionCurrentStateRefusal::ManifestRefused { source }.into_io())?; let namespace = candidate.root().namespace().digest(); - let entries = manifest.manifest().entries(); + let entries = current.manifest().entries(); let entry = entries .binary_search_by_key(&namespace, |entry| entry.namespace()) .ok() @@ -216,7 +250,7 @@ pub(super) fn verify_predecessor( /// The empty retention state admits only a generation-one head with no predecessor. fn require_initial_publication( preparation: &RetentionPublicationPreparation<'_>, -) -> io::Result { +) -> io::Result<()> { let publication = preparation .publication() .ok_or_else(|| RetentionCurrentStateRefusal::StaleCommittedRetry.into_io())?; @@ -225,7 +259,7 @@ fn require_initial_publication( if prepared.head().generation() == crate::LivenessGeneration::INITIAL && prepared.head().predecessor().is_none() { - Ok(RetentionTransitionDisposition::Publish) + Ok(()) } else { Err(RetentionCurrentStateRefusal::NonInitialOverAbsentHead.into_io()) } diff --git a/src/adapters/retention/filesystem_retention_storage.rs b/src/adapters/retention/filesystem_retention_storage.rs index d0cd0b5..1af1d17 100644 --- a/src/adapters/retention/filesystem_retention_storage.rs +++ b/src/adapters/retention/filesystem_retention_storage.rs @@ -8,10 +8,10 @@ use cap_std::fs::Dir; use super::filesystem_retention_attempt::{self as attempt, PublicationAttempt}; use super::filesystem_retention_authority::FilesystemRetentionPublicationAuthority; use super::filesystem_retention_catalog; -use super::filesystem_retention_current; +use super::filesystem_retention_current::{self, ObservedDisposition}; use super::filesystem_retention_namespace; use super::filesystem_retention_pool_name as pool_name; -use super::filesystem_retention_stage::{FilesystemRetentionStage, invalid_data}; +use super::filesystem_retention_stage::FilesystemRetentionStage; use super::{ AdmittedRetentionRoot, CanonicalRetentionHead, CanonicalRetentionManifest, RetentionCurrentStateRefusal, RetentionNamespaceAdmission, RetentionPublicationPreparation, @@ -33,44 +33,43 @@ impl RetentionPublicationStorage for FilesystemRetentionPublicationAuthority { if current.is_none() && !census.is_empty() { return Err(RetentionCurrentStateRefusal::HeadAbsentWithArtifacts.into_io()); } - let disposition = filesystem_retention_current::disposition(preparation, current.as_ref())?; + let observed = filesystem_retention_current::disposition(preparation, current.as_ref())?; filesystem_retention_catalog::require_current_catalog(&self.root, preparation)?; - if disposition == RetentionTransitionDisposition::Publish { - filesystem_retention_namespace::admit_expectation( - &self.roots, - preparation.candidate(), - preparation.expected(), - )?; - filesystem_retention_namespace::admit_capacity( - census, - &self.roots, - preparation.candidate(), - )?; - if let (RetentionGenerationExpectation::Current(_), Some(current)) = - (preparation.expected(), current.as_ref()) - { - filesystem_retention_current::verify_predecessor( + match observed { + ObservedDisposition::Publish => { + filesystem_retention_namespace::admit_expectation( + &self.roots, + preparation.candidate(), + preparation.expected(), + )?; + filesystem_retention_namespace::admit_capacity( + census, + &self.roots, + preparation.candidate(), + )?; + if let (RetentionGenerationExpectation::Current(_), Some(current)) = + (preparation.expected(), current.as_ref()) + { + filesystem_retention_current::verify_predecessor( + &self.roots, + current, + preparation.candidate(), + )?; + } + self.attempt = Some(PublicationAttempt::new( + preparation.expected(), + preparation.liveness_generation(), + )); + } + ObservedDisposition::Committed(current) => { + filesystem_retention_current::verify_committed( &self.roots, current, preparation.candidate(), )?; } - self.attempt = Some(PublicationAttempt::new( - preparation.expected(), - preparation.liveness_generation(), - )); - } - if disposition == RetentionTransitionDisposition::AlreadyCommitted { - let current = current - .as_ref() - .ok_or_else(|| invalid_data("already-committed disposition without a head"))?; - filesystem_retention_current::verify_committed( - &self.roots, - current, - preparation.candidate(), - )?; } - Ok(disposition) + Ok(observed.transition()) } fn write_root_stage(&mut self, root: &AdmittedRetentionRoot<'_>) -> io::Result<()> { From ccf64232bebe7c84cfe1c5d1975efaef74128908 Mon Sep 17 00:00:00 2001 From: James Ross Date: Mon, 7 Sep 2026 12:19:36 -0700 Subject: [PATCH 087/111] Fix: carry decode errors as the source of record-admission refusals Version-two record admission rendered every decode failure into an InvalidData string, so a caller behind FilesystemPlatformAdmissionError:: MigrationRecord could read which record refused only by parsing text and could not reach the decoder's own refusal. The retention catalog-head binding discarded its PublicationHeadDecodeError the same way. VersionTwoRecordRefusal now travels as the source of that InvalidData error: LengthOverflow, KindOrLength, and TrailingBytes name the record, and Marker, Intent, and Receipt carry the exact decode error as their source. It is exported and #[non_exhaustive]. RetentionCurrentStateRefusal::CatalogHeadRefused carries its PublicationHeadDecodeError. The corrupt-marker law now downcasts to VersionTwoRecordRefusal::Marker (it failed before this change), and a new law corrupts this store's catalog HEAD and requires CatalogHeadRefused with a present source. Self-review finding R10 (P3). Refs #78 --- CHANGELOG.md | 5 + src/adapters/exports.rs | 1 + .../filesystem_version_two_record_refusal.rs | 102 ++++++++++++++++++ .../filesystem_version_two_records.rs | 30 +++--- src/adapters/mod.rs | 1 + .../retention/filesystem_retention_catalog.rs | 2 +- .../filesystem_retention_catalog_tests.rs | 35 ++++++ .../retention/filesystem_retention_refusal.rs | 11 +- .../filesystem_version_two_admission_tests.rs | 10 +- src/lib.rs | 6 +- 10 files changed, 177 insertions(+), 26 deletions(-) create mode 100644 src/adapters/filesystem_version_two_record_refusal.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index eef8f1e..0648f6c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -567,6 +567,11 @@ after its public API and format compatibility policies are established. Review corrections to the unreleased retention and migration work above; none of these shipped in a release. +- Version-two record admission and catalog-head binding carry their decode + errors as sources: `VersionTwoRecordRefusal` names which of `FORMAT`, + `migration.intent`, or `migration.receipt` refused and preserves the + decoder's diagnosis, and `CatalogHeadRefused` carries the + `PublicationHeadDecodeError`. - Retention namespace admission refuses with typed `RetentionCurrentStateRefusal` variants (`UnknownRetentionEntry`, `NonNamespaceEntry`, `NoncanonicalPoolEntry`, `NamespaceCapacity`, diff --git a/src/adapters/exports.rs b/src/adapters/exports.rs index 4b18787..e2c8c51 100644 --- a/src/adapters/exports.rs +++ b/src/adapters/exports.rs @@ -56,6 +56,7 @@ pub use super::filesystem_recovery_stage_error::{ }; pub use super::filesystem_segment_stage::FilesystemSegmentStage; pub use super::filesystem_version_two_admission::FilesystemVersionTwoAdmission; +pub use super::filesystem_version_two_record_refusal::VersionTwoRecordRefusal; pub use super::filesystem_writer_lock::FilesystemWriterLock; pub use super::layout_decode_error::LayoutDecodeError; pub use super::layout_decode_policy::LayoutDecodePolicy; diff --git a/src/adapters/filesystem_version_two_record_refusal.rs b/src/adapters/filesystem_version_two_record_refusal.rs new file mode 100644 index 0000000..21711cf --- /dev/null +++ b/src/adapters/filesystem_version_two_record_refusal.rs @@ -0,0 +1,102 @@ +//! This module owns the typed refusal behind version-two record admission. + +use std::error::Error; +use std::fmt; +use std::io; + +use super::{ + StoreFormatMarkerDecodeError, StoreMigrationIntentDecodeError, StoreMigrationReceiptDecodeError, +}; + +/// Why `FORMAT`, `migration.intent`, or `migration.receipt` refused admission. +/// +/// Carried as the source of the `InvalidData` error behind +/// [`FilesystemPlatformAdmissionError::MigrationRecord`](super::FilesystemPlatformAdmissionError::MigrationRecord), +/// so a caller can tell which record refused and recover the decoder's own +/// diagnosis instead of a rendered string. +#[derive(Debug)] +#[non_exhaustive] +pub enum VersionTwoRecordRefusal { + /// The record's canonical length exceeds the platform's addressable range. + LengthOverflow { + /// The record's fixed name. + name: &'static str, + }, + /// The record is not a regular file of its canonical length. + KindOrLength { + /// The record's fixed name. + name: &'static str, + }, + /// The record carried bytes beyond its canonical length. + TrailingBytes { + /// The record's fixed name. + name: &'static str, + }, + /// `FORMAT` did not decode as a canonical version-two marker. + Marker { + /// The exact decode refusal. + source: StoreFormatMarkerDecodeError, + }, + /// `migration.intent` did not decode as a canonical intent. + Intent { + /// The exact decode refusal. + source: StoreMigrationIntentDecodeError, + }, + /// `migration.receipt` did not admit against the decoded intent and marker. + Receipt { + /// The exact admission refusal. + source: StoreMigrationReceiptDecodeError, + }, +} + +impl VersionTwoRecordRefusal { + /// Wraps the refusal as the `InvalidData` error record admission returns. + pub(super) fn into_io(self) -> io::Error { + io::Error::new(io::ErrorKind::InvalidData, self) + } +} + +impl fmt::Display for VersionTwoRecordRefusal { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::LengthOverflow { name } => write!( + formatter, + "version-two record {name} has a canonical length beyond the addressable range" + ), + Self::KindOrLength { name } => { + write!( + formatter, + "version-two record {name} has the wrong kind or length" + ) + } + Self::TrailingBytes { name } => { + write!( + formatter, + "version-two record {name} carried trailing bytes" + ) + } + Self::Marker { .. } => { + formatter.write_str("version-two record FORMAT refused admission") + } + Self::Intent { .. } => { + formatter.write_str("version-two record migration.intent refused admission") + } + Self::Receipt { .. } => { + formatter.write_str("version-two record migration.receipt refused admission") + } + } + } +} + +impl Error for VersionTwoRecordRefusal { + fn source(&self) -> Option<&(dyn Error + 'static)> { + match self { + Self::Marker { source } => Some(source), + Self::Intent { source } => Some(source), + Self::Receipt { source } => Some(source), + Self::LengthOverflow { .. } + | Self::KindOrLength { .. } + | Self::TrailingBytes { .. } => None, + } + } +} diff --git a/src/adapters/filesystem_version_two_records.rs b/src/adapters/filesystem_version_two_records.rs index 434a15e..0b11841 100644 --- a/src/adapters/filesystem_version_two_records.rs +++ b/src/adapters/filesystem_version_two_records.rs @@ -5,6 +5,7 @@ use std::io::{self, Read}; use cap_fs_ext::{FollowSymlinks, OpenOptionsFollowExt, OpenOptionsSyncExt}; use cap_std::fs::{Dir, OpenOptions}; +use super::VersionTwoRecordRefusal as Refusal; use super::store_migration::{ FORMAT_MARKER_LENGTH, MIGRATION_INTENT_LENGTH, MIGRATION_RECEIPT_LENGTH, }; @@ -52,18 +53,20 @@ impl BoundRootIdentity { /// canonical length, and decoded. The receipt is admitted only against the /// decoded intent and marker, so a record set that is individually /// well-formed but mutually inconsistent refuses. Writer authority over a -/// version-two root must not be returned before this admission succeeds. The -/// intent's bound root coordinates are returned for identity comparison. +/// version-two root must not be returned before this admission succeeds. Every +/// refusal is an `InvalidData` error whose source is a +/// [`VersionTwoRecordRefusal`](super::VersionTwoRecordRefusal). The intent's +/// bound root coordinates are returned for identity comparison. pub(super) fn admit(root: &Dir) -> io::Result { let marker_bytes = read_exact(root, MARKER_NAME, FORMAT_MARKER_LENGTH)?; let intent_bytes = read_exact(root, INTENT_NAME, MIGRATION_INTENT_LENGTH)?; let receipt_bytes = read_exact(root, RECEIPT_NAME, MIGRATION_RECEIPT_LENGTH)?; let marker = AdmittedStoreFormatMarker::decode(&marker_bytes) - .map_err(|source| invalid_data(MARKER_NAME, &source))?; + .map_err(|source| Refusal::Marker { source }.into_io())?; let intent = AdmittedStoreMigrationIntent::decode(&intent_bytes) - .map_err(|source| invalid_data(INTENT_NAME, &source))?; + .map_err(|source| Refusal::Intent { source }.into_io())?; let _receipt = AdmittedStoreMigrationReceipt::decode(&receipt_bytes, &intent, &marker) - .map_err(|source| invalid_data(RECEIPT_NAME, &source))?; + .map_err(|source| Refusal::Receipt { source }.into_io())?; Ok(BoundRootIdentity::new( intent.root_device_identity().get(), intent.root_mount_identity().get(), @@ -71,28 +74,21 @@ pub(super) fn admit(root: &Dir) -> io::Result { )) } -fn read_exact(root: &Dir, name: &str, length: usize) -> io::Result> { +fn read_exact(root: &Dir, name: &'static str, length: usize) -> io::Result> { let mut options = OpenOptions::new(); options.read(true).follow(FollowSymlinks::No).nonblock(true); let mut file = root.open_with(name, &options)?; - let expected_length = u64::try_from(length) - .map_err(|_source| invalid_data(name, &"record length exceeded u64"))?; + let expected_length = + u64::try_from(length).map_err(|_source| Refusal::LengthOverflow { name }.into_io())?; let metadata = file.metadata()?; if !metadata.is_file() || metadata.len() != expected_length { - return Err(invalid_data(name, &"record kind or length disagreed")); + return Err(Refusal::KindOrLength { name }.into_io()); } let mut bytes = vec![0_u8; length]; file.read_exact(&mut bytes)?; let mut trailing = [0_u8; 1]; if file.read(&mut trailing)? != 0 { - return Err(invalid_data(name, &"record carried trailing bytes")); + return Err(Refusal::TrailingBytes { name }.into_io()); } Ok(bytes) } - -fn invalid_data(name: &str, source: &dyn std::fmt::Display) -> io::Error { - io::Error::new( - io::ErrorKind::InvalidData, - format!("version-two record {name} refused admission: {source}"), - ) -} diff --git a/src/adapters/mod.rs b/src/adapters/mod.rs index 3a6333f..4927cd1 100644 --- a/src/adapters/mod.rs +++ b/src/adapters/mod.rs @@ -129,6 +129,7 @@ mod filesystem_store_initializer_tests; #[path = "../../tests/segment_filesystem_stage/sandbox.rs"] mod filesystem_test_sandbox; mod filesystem_version_two_admission; +mod filesystem_version_two_record_refusal; mod filesystem_version_two_records; mod filesystem_writer_lock; mod framed_blake3; diff --git a/src/adapters/retention/filesystem_retention_catalog.rs b/src/adapters/retention/filesystem_retention_catalog.rs index c7e8a12..c08ec67 100644 --- a/src/adapters/retention/filesystem_retention_catalog.rs +++ b/src/adapters/retention/filesystem_retention_catalog.rs @@ -32,7 +32,7 @@ pub(super) fn require_current_catalog( .into_io() })?; let head = ChecksummedPublicationHead::decode(&bytes) - .map_err(|_source| RetentionCurrentStateRefusal::CatalogHeadRefused.into_io())?; + .map_err(|source| RetentionCurrentStateRefusal::CatalogHeadRefused { source }.into_io())?; if head.generation() == expected_generation && head.catalog_digest() == closure.catalog_digest() { Ok(()) diff --git a/src/adapters/retention/filesystem_retention_catalog_tests.rs b/src/adapters/retention/filesystem_retention_catalog_tests.rs index 7c4f26f..e55bd68 100644 --- a/src/adapters/retention/filesystem_retention_catalog_tests.rs +++ b/src/adapters/retention/filesystem_retention_catalog_tests.rs @@ -68,3 +68,38 @@ fn committed_retry_over_a_foreign_catalog_refuses_before_reporting_committed() sandbox.remove()?; Ok(()) } + +#[test] +fn a_corrupt_catalog_head_refuses_with_its_decode_error() -> Result<(), Box> { + let (sandbox, mut authority) = super::filesystem_retention_test_fixture::open_authority( + "filesystem-retention-catalog-corrupt-head", + )?; + let root_bytes = super::filesystem_retention_test_fixture::fixture( + super::filesystem_retention_test_fixture::ROOT_HEX, + )?; + let preparation = super::filesystem_retention_test_fixture::initial_preparation(&root_bytes)?; + let head = sandbox.path().join("HEAD"); + let mut bytes = std::fs::read(&head)?; + *bytes + .get_mut(100) + .ok_or("catalog head shorter than 101 bytes")? ^= 0x01; + std::fs::write(&head, &bytes)?; + + let error = RetentionPublicationStorage::verify_current(&mut authority, &preparation) + .err() + .ok_or("corrupt catalog head was unexpectedly admitted")?; + + let refusal = super::filesystem_retention_test_fixture::refusal(&error) + .ok_or("catalog head refusal was not typed")?; + assert!(matches!( + refusal, + RetentionCurrentStateRefusal::CatalogHeadRefused { .. } + )); + assert!( + refusal.source().is_some(), + "the decode error must travel as the refusal's source" + ); + drop(authority); + sandbox.remove()?; + Ok(()) +} diff --git a/src/adapters/retention/filesystem_retention_refusal.rs b/src/adapters/retention/filesystem_retention_refusal.rs index 0f7173c..21d59f8 100644 --- a/src/adapters/retention/filesystem_retention_refusal.rs +++ b/src/adapters/retention/filesystem_retention_refusal.rs @@ -5,6 +5,7 @@ use std::fmt; use std::io; use super::{RetentionHeadDecodeError, RetentionManifestDecodeError}; +use crate::adapters::PublicationHeadDecodeError; use crate::{CatalogGeneration, LivenessGeneration, RetentionManifestDigest}; /// Exact reason filesystem current-state verification refused a transition. @@ -54,8 +55,11 @@ pub enum RetentionCurrentStateRefusal { /// The catalog generation the store's head names, if it decoded. observed_generation: Option, }, - /// The store's catalog head refused admission. - CatalogHeadRefused, + /// This store's catalog `HEAD` did not decode. + CatalogHeadRefused { + /// The exact decode refusal. + source: PublicationHeadDecodeError, + }, /// The current liveness generation has no successor. LivenessExhausted, /// A byte-identical retry found that another successor is current. @@ -155,7 +159,7 @@ impl fmt::Display for RetentionCurrentStateRefusal { current catalog", expected_generation.get() ), - Self::CatalogHeadRefused => { + Self::CatalogHeadRefused { .. } => { formatter.write_str("this store's catalog head refused admission") } Self::LivenessExhausted => { @@ -224,6 +228,7 @@ impl Error for RetentionCurrentStateRefusal { match self { Self::HeadRefused { source } | Self::PreparedHeadRefused { source } => Some(source), Self::ManifestRefused { source } => Some(source), + Self::CatalogHeadRefused { source } => Some(source), _ => None, } } diff --git a/src/adapters/retention/filesystem_version_two_admission_tests.rs b/src/adapters/retention/filesystem_version_two_admission_tests.rs index c3784c6..91c7602 100644 --- a/src/adapters/retention/filesystem_version_two_admission_tests.rs +++ b/src/adapters/retention/filesystem_version_two_admission_tests.rs @@ -8,6 +8,7 @@ use crate::adapters::filesystem_root_identity::FilesystemRootIdentity; use crate::adapters::filesystem_version_two_admission::{BoundRootIdentity, require_root_identity}; use crate::adapters::{ FilesystemPlatformAdmissionError, FilesystemVersionTwoAdmission, StoreRootIdentityCoordinate, + VersionTwoRecordRefusal, }; #[test] @@ -24,9 +25,14 @@ fn version_two_reopen_refuses_a_corrupt_format_marker() -> Result<(), Box()), + Some(VersionTwoRecordRefusal::Marker { .. }) )); sandbox.remove()?; Ok(()) diff --git a/src/lib.rs b/src/lib.rs index 2ce7c18..6611c89 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -123,9 +123,9 @@ pub use adapters::{ StoreMigrationInventoryError, StoreMigrationInventoryHasher, StoreMigrationPhase, StoreMigrationReceiptDecodeError, StoreMigrationStorage, StoreRootDeviceIdentity, StoreRootFileIdentity, StoreRootIdentityCoordinate, StoreRootMountIdentity, - WriterLockAcquireError, WriterLockAcquirePhase, admit_recovery_stage_bytes, - assess_recovery_stage, classify_recovery_catalog_stage, classify_recovery_names, - classify_recovery_next_head_stage, classify_recovery_segment_stage, + VersionTwoRecordRefusal, WriterLockAcquireError, WriterLockAcquirePhase, + admit_recovery_stage_bytes, assess_recovery_stage, classify_recovery_catalog_stage, + classify_recovery_names, classify_recovery_next_head_stage, classify_recovery_segment_stage, execute_recovery_next_head_finalization, execute_recovery_segment_resume, execute_recovery_stage_completion, execute_recovery_stage_discard, execute_store_migration, fingerprint_recovery_stage, initialize_store, plan_recovery_next_head_finalization, From b38e4168c10295fe0def5913d8c1841d24f3efe4 Mon Sep 17 00:00:00 2001 From: James Ross Date: Mon, 7 Sep 2026 12:26:20 -0700 Subject: [PATCH 088/111] Fix: report the stage's real length when trailing bytes are found reject_trailing_bytes reported LengthChanged.observed as the expected length plus the single byte that detected the overrun, so a 6-byte stage declared at 3 bytes was reported as 4. Two unreachable fallback arms reported the expected length itself as the observation. The three disagreement sites now report the file's metadata length through one helper, so observed names what the filesystem holds. The pinned trailing- bytes law expects observed: 6 for the 6-byte fixture; it failed before this change and passes now. Self-review finding A8 (P3). Refs #78 --- CHANGELOG.md | 3 ++ ...lesystem_recovery_stage_materialization.rs | 54 +++++++++++-------- 2 files changed, 34 insertions(+), 23 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0648f6c..e6c1955 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -567,6 +567,9 @@ after its public API and format compatibility policies are established. Review corrections to the unreleased retention and migration work above; none of these shipped in a release. +- `FilesystemRecoveryStageError::LengthChanged` reports the stage's actual + on-disk length in `observed` when trailing bytes are found, instead of the + expected length plus the one byte that detected them. - Version-two record admission and catalog-head binding carry their decode errors as sources: `VersionTwoRecordRefusal` names which of `FORMAT`, `migration.intent`, or `migration.receipt` refused and preserves the diff --git a/src/adapters/filesystem_recovery_stage_materialization.rs b/src/adapters/filesystem_recovery_stage_materialization.rs index c1a98aa..9705158 100644 --- a/src/adapters/filesystem_recovery_stage_materialization.rs +++ b/src/adapters/filesystem_recovery_stage_materialization.rs @@ -56,12 +56,16 @@ fn read_exact( expected: length, source, })?; - let observed = - u64::try_from(observed).map_err(|_source| FilesystemRecoveryStageError::LengthChanged { - stage, - expected: length, - observed: expected, - })?; + let observed = match u64::try_from(observed) { + Ok(observed) => observed, + Err(_source) => { + return Err(FilesystemRecoveryStageError::LengthChanged { + stage, + expected: length, + observed: observed_length(file, stage, length)?, + }); + } + }; if observed < expected { return Err(FilesystemRecoveryStageError::Materialize { stage, @@ -85,25 +89,11 @@ fn reject_trailing_bytes( loop { match file.read(&mut trailing) { Ok(0) => return Ok(()), - Ok(read_bytes) => { - let increment = u64::try_from(read_bytes).map_err(|_source| { - FilesystemRecoveryStageError::LengthChanged { - stage, - expected, - observed: expected.get(), - } - })?; - let observed = expected.get().checked_add(increment).ok_or_else(|| { - FilesystemRecoveryStageError::LengthChanged { - stage, - expected, - observed: expected.get(), - } - })?; + Ok(_trailing_bytes) => { return Err(FilesystemRecoveryStageError::LengthChanged { stage, expected, - observed, + observed: observed_length(file, stage, expected)?, }); } Err(source) if source.kind() == io::ErrorKind::Interrupted => {} @@ -118,6 +108,24 @@ fn reject_trailing_bytes( } } +/// Reports the stage's actual length when its bytes disagree with `expected`. +/// +/// `LengthChanged.observed` names what the filesystem holds now, not a guess +/// derived from the read that detected the disagreement. +fn observed_length( + file: &File, + stage: RecoveryStage, + expected: RecoveryStageLength, +) -> Result { + file.metadata() + .map(|metadata| metadata.len()) + .map_err(|source| FilesystemRecoveryStageError::Materialize { + stage, + expected, + source, + }) +} + pub(super) fn verify_position( file: &mut File, stage: RecoveryStage, @@ -222,7 +230,7 @@ mod tests { FilesystemRecoveryStageError::LengthChanged { stage: RecoveryStage::Segment, expected, - observed: 4, + observed: 6, } if expected.get() == 3 )); drop(file); From c95a0946ffae925a1f9fa000227958b06d3619a3 Mon Sep 17 00:00:00 2001 From: James Ross Date: Mon, 7 Sep 2026 12:31:47 -0700 Subject: [PATCH 089/111] Fix: make the platform admission error non-exhaustive FilesystemPlatformAdmissionError gained MigrationRecord and RootIdentityChanged on this branch, and every future admission boundary (recovery, GC) will add a refusal of its own. Without #[non_exhaustive] each addition would be a breaking change for any caller that matched the enum exhaustively. Inside the crate only matches! uses exist, so nothing changes here. A source contract in tests/version_two_admission_contract.rs pins the attribute; it failed before this change and passes now. Self-review finding X1 (P3). Refs #78 --- CHANGELOG.md | 3 +++ src/adapters/filesystem_platform_admission_error.rs | 1 + tests/version_two_admission_contract.rs | 12 ++++++++++++ 3 files changed, 16 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index e6c1955..649243d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -242,6 +242,9 @@ after its public API and format compatibility policies are established. ### Changed +- `FilesystemPlatformAdmissionError` is `#[non_exhaustive]`, so a future + admission refusal can be added without a breaking change; a source contract + pins the attribute. - `ObservedRetentionState` carries the decoded head and manifest alongside their exact bytes, so disposition, already-committed verification, and predecessor verification decode each record once; the disposition names the diff --git a/src/adapters/filesystem_platform_admission_error.rs b/src/adapters/filesystem_platform_admission_error.rs index b26d788..d824d82 100644 --- a/src/adapters/filesystem_platform_admission_error.rs +++ b/src/adapters/filesystem_platform_admission_error.rs @@ -8,6 +8,7 @@ use super::{StoreRootIdentityCoordinate, WriterLockAcquireError}; /// Failure to reacquire writer authority over one published filesystem store. #[derive(Debug)] +#[non_exhaustive] pub enum FilesystemPlatformAdmissionError { /// The store root does not satisfy the production platform profile. Platform { diff --git a/tests/version_two_admission_contract.rs b/tests/version_two_admission_contract.rs index 1c33945..5ae4a9d 100644 --- a/tests/version_two_admission_contract.rs +++ b/tests/version_two_admission_contract.rs @@ -8,6 +8,8 @@ const VERSION_TWO_ADMISSION: &str = include_str!("../src/adapters/filesystem_version_two_admission.rs"); const MIGRATION_AUTHORITY: &str = include_str!("../src/adapters/store_migration/filesystem_migration_authority.rs"); +const ADMISSION_ERROR: &str = + include_str!("../src/adapters/filesystem_platform_admission_error.rs"); #[test] fn retention_publication_consumes_only_version_two_authority() { @@ -31,3 +33,13 @@ fn version_two_reopen_produces_only_version_two_authority() { assert!(VERSION_TWO_ADMISSION.contains("filesystem_version_two_records::admit")); assert!(VERSION_TWO_ADMISSION.contains("filesystem_platform_profile::open_version_two")); } + +/// Admission refusals grow with the platform surface (`MigrationRecord` and +/// `RootIdentityChanged` arrived after the first release candidate), so the +/// public error is non-exhaustive and a new refusal is not a breaking change. +#[test] +fn admission_error_is_non_exhaustive() { + assert!( + ADMISSION_ERROR.contains("#[non_exhaustive]\npub enum FilesystemPlatformAdmissionError") + ); +} From 47e137e479ec7a26b055b81b192d1737e140130c Mon Sep 17 00:00:00 2001 From: James Ross Date: Mon, 7 Sep 2026 12:40:24 -0700 Subject: [PATCH 090/111] Docs: state the version-two implementation boundary in the formats index The formats registry described keep.segment-store/v2 as "retention transition implementation planned", and the version-two overview said no version-2 writer was available and version 1 remained the only admitted production store. Both predate the migration authority, FilesystemVersionTwoAdmission, and FilesystemRetentionPublicationAuthority on this branch. Both pages now name what is implemented with executable evidence and what remains planned in issue #19, and the overview points at the requirements ledger as the authority on which requirements are proven. Nonclaims in retention.md and requirements.md are unchanged. Self-review finding X2 (P3). Refs #78 --- CHANGELOG.md | 4 ++++ docs/formats/README.md | 2 +- docs/formats/segment-store-v2/README.md | 10 ++++++---- 3 files changed, 11 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 649243d..17a7a60 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -242,6 +242,10 @@ after its public API and format compatibility policies are established. ### Changed +- The formats index and the version-two overview state what is implemented + (one-way migration, version-two reopen, forward retention publication) and + what remains planned in issue #19 (retention recovery, reader fencing, + collection), instead of describing the whole version as planned. - `FilesystemPlatformAdmissionError` is `#[non_exhaustive]`, so a future admission refusal can be added without a breaking change; a source contract pins the attribute. diff --git a/docs/formats/README.md b/docs/formats/README.md index 6952726..17cdb32 100644 --- a/docs/formats/README.md +++ b/docs/formats/README.md @@ -9,7 +9,7 @@ admitted merely because one Rust type can serialize and deserialize it. | --- | --- | --- | --- | | [Flat Chunk Layout v1](flat-chunk-layout-v1/README.md) | `keep.flat-chunks/v1` | Implemented through verified reconstruction in issues #10 and #13 | [Golden corpus](../../conformance/layout/v1/README.md) | | [Durable Segment Store v1](segment-store-v1/README.md) | `keep.segment-store/v1` | Implemented through initialization, publication, restart, and recovery in issues #14–#17 | [Golden corpus](../../conformance/segment-store/v1/README.md) | -| [Durable Segment Store v2](segment-store-v2/README.md) | `keep.segment-store/v2` | Retention transition implementation planned in issue #19 | [Golden corpus](../../conformance/segment-store/v2/README.md) | +| [Durable Segment Store v2](segment-store-v2/README.md) | `keep.segment-store/v2` | One-way migration, version-two reopen, and forward retention publication implemented; retention recovery, reader fencing, and collection planned in issue #19 | [Golden corpus](../../conformance/segment-store/v2/README.md) | The registry records protocol specifications, including formats whose implementation is still planned. Each format page states its exact proof diff --git a/docs/formats/segment-store-v2/README.md b/docs/formats/segment-store-v2/README.md index 7d76e94..04cb7b3 100644 --- a/docs/formats/segment-store-v2/README.md +++ b/docs/formats/segment-store-v2/README.md @@ -7,10 +7,12 @@ reader fences, migration evidence, and reserved GC and recovery-disposition namespaces. ADR-0009 owns the cross-cutting retention and liveness decision. These pages -own its durable representation. Issue #19 must supply the production retention -implementation and executable evidence before any version-2 writer is -available. Until that implementation lands, version 1 remains the only -admitted production store. +own its durable representation. The one-way migration, version-two reopen, and +forward retention publication are implemented with executable evidence; +recovery of retained retention stages, reader fencing, and collection remain +planned in issue #19, and the [requirements ledger](requirements.md) records +exactly which requirements are proven. A version-1 store remains admitted until +its owner migrates it. ## Core laws From 8ed1b4e24b3fb84d264b845a2bf59279b1a96b75 Mon Sep 17 00:00:00 2001 From: James Ross Date: Mon, 7 Sep 2026 12:48:24 -0700 Subject: [PATCH 091/111] Fix: create test FIFOs only through mknodat and gate the laws to Linux The retention FIFO laws fell back to spawning mkfifo(1) on hosts where rustix compiles mknodat out. A spawned child briefly holds copies of every open descriptor in the process, and a flock lives as long as any copy of its descriptor, so while one FIFO law spawned, another law's writer lock stayed held across its drop-and-reopen and refused with WriterLock { source: Busy }. Measured locally: 3 of 6 parallel lib runs failed, 0 of 4 single-threaded, every failure a FIFO law. The fallback is deleted and the module is compiled only on Linux, where CI runs it through mknodat. A contract test walks src/ and refuses any module that names process::Command or Command::new; it failed before this change and passes now. No production path ever spawned a process. Self-review finding R18 (P4), pulled forward because it explained the flake. Refs #78 --- CHANGELOG.md | 5 ++++ src/adapters/retention.rs | 2 +- .../filesystem_retention_fifo_tests.rs | 21 +++++-------- tests/adapters_layout_contract.rs | 30 ++++++++++++++++++- 4 files changed, 42 insertions(+), 16 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 17a7a60..fa3a2bb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -242,6 +242,11 @@ after its public API and format compatibility policies are established. ### Changed +- The retention FIFO laws run only on Linux through `mknodat`; the `mkfifo(1)` + fallback for other hosts is removed, and a contract test keeps every module + under `src/` free of process spawns. The fallback's spawned child briefly + held copies of other tests' lock descriptors, which surfaced locally as an + intermittent `WriterLock { source: Busy }` at reopen. - The formats index and the version-two overview state what is implemented (one-way migration, version-two reopen, forward retention publication) and what remains planned in issue #19 (retention recovery, reader fencing, diff --git a/src/adapters/retention.rs b/src/adapters/retention.rs index 5a9b9f8..334f250 100644 --- a/src/adapters/retention.rs +++ b/src/adapters/retention.rs @@ -30,7 +30,7 @@ mod filesystem_retention_current; mod filesystem_retention_current_tests; #[cfg(test)] mod filesystem_retention_expectation_tests; -#[cfg(test)] +#[cfg(all(test, target_os = "linux"))] mod filesystem_retention_fifo_tests; mod filesystem_retention_namespace; #[cfg(test)] diff --git a/src/adapters/retention/filesystem_retention_fifo_tests.rs b/src/adapters/retention/filesystem_retention_fifo_tests.rs index fccc9d1..e91a160 100644 --- a/src/adapters/retention/filesystem_retention_fifo_tests.rs +++ b/src/adapters/retention/filesystem_retention_fifo_tests.rs @@ -1,4 +1,6 @@ //! Filesystem retention non-regular-file laws: a FIFO at a protocol name refuses, never blocks. +//! +//! Linux only: the fixture creates the FIFO through `mknodat`. use std::error::Error; use std::path::Path; @@ -71,10 +73,11 @@ fn a_fifo_at_a_manifest_pool_name_refuses_instead_of_blocking() -> Result<(), Bo /// Creates a FIFO at `path` for the law under test. /// -/// rustix compiles `mknodat` out on Apple targets, so the fixture falls back -/// to `mkfifo(1)` there. This is test scaffolding only; no storage path -/// spawns a process. -#[cfg(target_os = "linux")] +/// These laws run only on Linux, where rustix exposes `mknodat`; rustix +/// compiles it out on Apple targets, and spawning `mkfifo(1)` instead is not +/// acceptable scaffolding: a spawned child briefly holds copies of every open +/// descriptor, which kept another law's `flock` alive across its +/// drop-and-reopen. CI runs these laws. fn make_fifo(path: &Path) -> Result<(), Box> { use rustix::fs::{CWD, FileType, Mode, mknodat}; @@ -82,16 +85,6 @@ fn make_fifo(path: &Path) -> Result<(), Box> { Ok(()) } -#[cfg(not(target_os = "linux"))] -fn make_fifo(path: &Path) -> Result<(), Box> { - let status = std::process::Command::new("mkfifo").arg(path).status()?; - if status.success() { - Ok(()) - } else { - Err(format!("mkfifo exited with {status}").into()) - } -} - /// Runs `operation` on its own thread and refuses the test if it does not finish. fn completes_within( operation: impl FnOnce() -> Result<(), E> + Send + 'static, diff --git a/tests/adapters_layout_contract.rs b/tests/adapters_layout_contract.rs index e992147..eca5799 100644 --- a/tests/adapters_layout_contract.rs +++ b/tests/adapters_layout_contract.rs @@ -1,6 +1,7 @@ //! Source-layout laws for the adapters tree: the module root stays a scannable //! manifest, files rustfmt cannot rewrap stay within the standard width, and -//! record readers take their fixed lengths from the decoders that define them. +//! record readers take their fixed lengths from the decoders that define them, +//! and no module under `src/` spawns a process. const ADAPTERS_ROOT: &str = include_str!("../src/adapters/mod.rs"); const RETENTION_REFUSAL: &str = @@ -96,3 +97,30 @@ fn record_readers_take_lengths_from_the_decoders() { } } } + +/// No source module spawns a process, test scaffolding included. A spawned +/// child briefly holds copies of every open descriptor, so a `mkfifo(1)` +/// fallback in one law kept another law's `flock` alive across its +/// drop-and-reopen and surfaced as an intermittent `WriterLock { Busy }`. +/// Laws that need a device node use the kernel API and are gated to Linux. +#[test] +fn no_source_module_spawns_a_process() -> Result<(), Box> { + let root = std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("src"); + let mut pending = vec![root]; + while let Some(directory) = pending.pop() { + for entry in std::fs::read_dir(&directory)? { + let path = entry?.path(); + if path.is_dir() { + pending.push(path); + } else if path.extension().is_some_and(|extension| extension == "rs") { + let source = std::fs::read_to_string(&path)?; + assert!( + !source.contains("process::Command") && !source.contains("Command::new("), + "{} spawns a process", + path.display() + ); + } + } + } + Ok(()) +} From ba10582e402b3f2b6141831586753d448f013afa Mon Sep 17 00:00:00 2001 From: James Ross Date: Mon, 7 Sep 2026 14:49:05 -0700 Subject: [PATCH 092/111] Refactor: share exact-record primitives, starting with the retention stage The retention stage, the migration fixed-record stage, the retention current-state reader, and the version-two record reader each carried their own copy of the same primitive: open read-only without following links and without blocking, check kind and length, read exactly the expected bytes, refuse trailing bytes, and for stages reverify device and inode identity on both sides of the read. Four copies of a security-relevant read path drift independently; the FIFO refusal, for one, had to be added to each. filesystem_exact_record now owns EntryIdentity, a typed ExactRecordRefusal, verify_named, require_absent, and link_without_replacement; the bounded readers arrive with their consumers in later commits, since deny(warnings) refuses a primitive with no consumer. Refusals carry no protocol vocabulary; each consumer maps them onto its own exact messages or typed variants, so no observable refusal changes. The retention stage is ported first, its messages preserved byte for byte. Two laws pin the primitives (different bytes and a byte-equal substitute refuse; absence and no-replacement linking hold), and a contract test keeps ported modules from reopening records themselves. Self-review finding D1 (P3), commit 1 of 6. Refs #78 --- CHANGELOG.md | 5 + src/adapters/filesystem_exact_record.rs | 194 ++++++++++++++++++ src/adapters/filesystem_exact_record_tests.rs | 74 +++++++ src/adapters/mod.rs | 1 + .../retention/filesystem_retention_stage.rs | 111 +++------- tests/adapters_layout_contract.rs | 25 +++ 6 files changed, 332 insertions(+), 78 deletions(-) create mode 100644 src/adapters/filesystem_exact_record.rs create mode 100644 src/adapters/filesystem_exact_record_tests.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index fa3a2bb..c11ea4e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -242,6 +242,11 @@ after its public API and format compatibility policies are established. ### Changed +- Stage publishers share one exact-record module for no-follow non-blocking + opens, exact-length verification, trailing-byte refusal, device-and-inode + reverification, absence checks, and no-replacement links; the retention + stage is the first consumer, and a contract test keeps ported modules from + reimplementing those primitives. - The retention FIFO laws run only on Linux through `mknodat`; the `mkfifo(1)` fallback for other hosts is removed, and a contract test keeps every module under `src/` free of process spawns. The fallback's spawned child briefly diff --git a/src/adapters/filesystem_exact_record.rs b/src/adapters/filesystem_exact_record.rs new file mode 100644 index 0000000..6d246b0 --- /dev/null +++ b/src/adapters/filesystem_exact_record.rs @@ -0,0 +1,194 @@ +//! This module owns the exact-record filesystem primitives shared by stage +//! publishers and fixed-record readers. +//! +//! Every open here is read-only, follows no links, and does not block, so a +//! FIFO or device planted at a protocol name refuses by kind instead of +//! hanging under the writer lock. Every read is bounded by the caller's exact +//! expected length and refuses trailing bytes. Callers map each +//! [`ExactRecordRefusal`] to their own typed refusal or message, so the +//! primitives carry no protocol vocabulary of their own. + +use std::error::Error; +use std::fmt; +use std::io::{self, Read}; + +use cap_fs_ext::{FollowSymlinks, MetadataExt, OpenOptionsFollowExt, OpenOptionsSyncExt}; +use cap_std::fs::{Dir, File, Metadata, OpenOptions}; + +/// Device and inode identity of one directory entry or open handle. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(super) struct EntryIdentity { + device: u64, + inode: u64, +} + +impl EntryIdentity { + /// Reads the identity behind an open file handle. + pub(super) fn of_file(file: &File) -> io::Result { + file.metadata().map(|metadata| Self::from(&metadata)) + } +} + +impl From<&Metadata> for EntryIdentity { + fn from(metadata: &Metadata) -> Self { + Self { + device: metadata.dev(), + inode: metadata.ino(), + } + } +} + +/// Why an exact record refused, independent of which protocol named it. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(super) enum ExactRecordRefusal { + /// The expected length does not fit the filesystem's `u64` length. + LengthOverflow, + /// The entry's kind, length, or device and inode identity disagreed. + KindLengthOrIdentity, + /// The entry's bytes disagreed with the expected record. + Bytes, + /// The entry carried bytes beyond the expected length. + TrailingBytes, + /// An entry that must be absent is still visible. + RemainedVisible, +} + +/// An exact-record failure: the filesystem's own error or a typed refusal. +#[derive(Debug)] +pub(super) enum ExactRecordError { + /// The filesystem refused the operation. + Io(io::Error), + /// The entry exists but is not the expected record. + Refused(ExactRecordRefusal), +} + +impl fmt::Display for ExactRecordRefusal { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter.write_str(match self { + Self::LengthOverflow => "exact record length exceeded the filesystem's range", + Self::KindLengthOrIdentity => "exact record kind, length, or identity disagreed", + Self::Bytes => "exact record bytes disagreed", + Self::TrailingBytes => "exact record carried trailing bytes", + Self::RemainedVisible => "removed exact record remained visible", + }) + } +} + +impl fmt::Display for ExactRecordError { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::Io(source) => write!(formatter, "exact record I/O failed: {source}"), + Self::Refused(refusal) => fmt::Display::fmt(refusal, formatter), + } + } +} + +impl Error for ExactRecordError { + fn source(&self) -> Option<&(dyn Error + 'static)> { + match self { + Self::Io(source) => Some(source), + Self::Refused(_) => None, + } + } +} + +impl From for ExactRecordError { + fn from(source: io::Error) -> Self { + Self::Io(source) + } +} + +impl From for ExactRecordError { + fn from(refusal: ExactRecordRefusal) -> Self { + Self::Refused(refusal) + } +} + +/// Reverifies that `name` is exactly `expected` with `identity`, before and after reading. +/// +/// Both the opened handle and the directory entry are checked for kind, +/// length, and identity on either side of the read, so a replaced or +/// byte-equal substituted entry refuses instead of being admitted. +pub(super) fn verify_named( + directory: &Dir, + name: &str, + expected: &[u8], + identity: EntryIdentity, +) -> Result<(), ExactRecordError> { + let mut file = open_read(directory, name)?; + require_metadata(&file.metadata()?, expected.len(), identity)?; + require_metadata(&directory.symlink_metadata(name)?, expected.len(), identity)?; + let mut observed = vec![0_u8; expected.len()]; + file.read_exact(&mut observed)?; + if observed != expected { + return Err(ExactRecordRefusal::Bytes.into()); + } + require_no_trailing_bytes(&mut file)?; + require_metadata(&file.metadata()?, expected.len(), identity)?; + require_metadata(&directory.symlink_metadata(name)?, expected.len(), identity) +} + +/// Requires that no entry named `name` remains visible. +pub(super) fn require_absent(directory: &Dir, name: &str) -> Result<(), ExactRecordError> { + match directory.symlink_metadata(name) { + Err(source) if source.kind() == io::ErrorKind::NotFound => Ok(()), + Ok(_) => Err(ExactRecordRefusal::RemainedVisible.into()), + Err(source) => Err(source.into()), + } +} + +/// Hard-links `source_name` to `destination_name`, admitting an existing target. +/// +/// An `AlreadyExists` target is left untouched for the caller to verify; this +/// primitive never replaces an entry. +pub(super) fn link_without_replacement( + source_directory: &Dir, + source_name: &str, + destination_directory: &Dir, + destination_name: &str, +) -> io::Result<()> { + match source_directory.hard_link(source_name, destination_directory, destination_name) { + Ok(()) => Ok(()), + Err(source) if source.kind() == io::ErrorKind::AlreadyExists => Ok(()), + Err(source) => Err(source), + } +} + +fn open_read(directory: &Dir, name: &str) -> io::Result { + let mut options = OpenOptions::new(); + options.read(true).follow(FollowSymlinks::No).nonblock(true); + directory.open_with(name, &options) +} + +fn exact_length(length: usize) -> Result { + u64::try_from(length).map_err(|_source| ExactRecordRefusal::LengthOverflow.into()) +} + +fn require_metadata( + metadata: &Metadata, + expected_length: usize, + expected_identity: EntryIdentity, +) -> Result<(), ExactRecordError> { + let expected_length = exact_length(expected_length)?; + if metadata.is_file() + && metadata.len() == expected_length + && EntryIdentity::from(metadata) == expected_identity + { + Ok(()) + } else { + Err(ExactRecordRefusal::KindLengthOrIdentity.into()) + } +} + +fn require_no_trailing_bytes(file: &mut File) -> Result<(), ExactRecordError> { + let mut trailing = [0_u8; 1]; + if file.read(&mut trailing)? == 0 { + Ok(()) + } else { + Err(ExactRecordRefusal::TrailingBytes.into()) + } +} + +#[cfg(test)] +#[path = "filesystem_exact_record_tests.rs"] +mod tests; diff --git a/src/adapters/filesystem_exact_record_tests.rs b/src/adapters/filesystem_exact_record_tests.rs new file mode 100644 index 0000000..40868cc --- /dev/null +++ b/src/adapters/filesystem_exact_record_tests.rs @@ -0,0 +1,74 @@ +//! Exact-record primitive laws: identity reverification, absence, no replacement. + +use std::error::Error; +use std::fs; + +use cap_std::fs::Dir; + +use super::{EntryIdentity, ExactRecordError, ExactRecordRefusal}; +use crate::adapters::filesystem_test_sandbox::TestDirectory; + +fn open(sandbox: &TestDirectory) -> Result> { + Ok(Dir::open_ambient_dir( + sandbox.path(), + cap_std::ambient_authority(), + )?) +} + +fn refusal(error: ExactRecordError) -> Result> { + match error { + ExactRecordError::Refused(refusal) => Ok(refusal), + ExactRecordError::Io(source) => Err(source.into()), + } +} + +#[test] +fn verify_named_refuses_different_bytes_and_a_byte_equal_substitute() -> Result<(), Box> +{ + let sandbox = TestDirectory::create("exact-record-verify")?; + let directory = open(&sandbox)?; + let path = sandbox.path().join("record"); + fs::write(&path, b"exact")?; + let identity = EntryIdentity::of_file(&directory.open("record")?)?; + + super::verify_named(&directory, "record", b"exact", identity)?; + let bytes = super::verify_named(&directory, "record", b"other", identity) + .err() + .ok_or("different bytes admitted")?; + fs::remove_file(&path)?; + fs::write(&path, b"exact")?; + let substitute = super::verify_named(&directory, "record", b"exact", identity) + .err() + .ok_or("byte-equal substitute admitted")?; + + assert_eq!(refusal(bytes)?, ExactRecordRefusal::Bytes); + assert_eq!( + refusal(substitute)?, + ExactRecordRefusal::KindLengthOrIdentity + ); + drop(directory); + sandbox.remove()?; + Ok(()) +} + +#[test] +fn require_absent_refuses_a_visible_entry_and_links_never_replace() -> Result<(), Box> { + let sandbox = TestDirectory::create("exact-record-absent-link")?; + let directory = open(&sandbox)?; + fs::write(sandbox.path().join("stage"), b"stage")?; + fs::write(sandbox.path().join("target"), b"existing")?; + + let visible = super::require_absent(&directory, "stage") + .err() + .ok_or("visible entry admitted as absent")?; + super::link_without_replacement(&directory, "stage", &directory, "target")?; + super::link_without_replacement(&directory, "stage", &directory, "linked")?; + + assert_eq!(refusal(visible)?, ExactRecordRefusal::RemainedVisible); + assert_eq!(fs::read(sandbox.path().join("target"))?, b"existing"); + assert_eq!(fs::read(sandbox.path().join("linked"))?, b"stage"); + super::require_absent(&directory, "absent")?; + drop(directory); + sandbox.remove()?; + Ok(()) +} diff --git a/src/adapters/mod.rs b/src/adapters/mod.rs index 4927cd1..840b9bf 100644 --- a/src/adapters/mod.rs +++ b/src/adapters/mod.rs @@ -77,6 +77,7 @@ mod filesystem_catalog_publisher_tests; mod filesystem_catalog_segment; mod filesystem_catalog_snapshot; mod filesystem_catalog_storage; +mod filesystem_exact_record; mod filesystem_initialization_namespace; mod filesystem_initialization_storage; mod filesystem_platform_admission; diff --git a/src/adapters/retention/filesystem_retention_stage.rs b/src/adapters/retention/filesystem_retention_stage.rs index 1ffe7b3..ed77e62 100644 --- a/src/adapters/retention/filesystem_retention_stage.rs +++ b/src/adapters/retention/filesystem_retention_stage.rs @@ -1,11 +1,13 @@ //! This module owns exact variable-length retention stage publication. -use std::io::{self, Read, Write}; +use std::io::{self, Write}; -use cap_fs_ext::{FollowSymlinks, MetadataExt, OpenOptionsFollowExt, OpenOptionsSyncExt}; -use cap_std::fs::{Dir, File, Metadata, OpenOptions}; +use cap_std::fs::{Dir, File}; use crate::adapters::filesystem_catalog_artifact; +use crate::adapters::filesystem_exact_record::{ + self as exact_record, EntryIdentity, ExactRecordError, ExactRecordRefusal, +}; /// One exclusively created, verified, and retained retention stage file. /// @@ -16,7 +18,7 @@ use crate::adapters::filesystem_catalog_artifact; pub(super) struct FilesystemRetentionStage { name: &'static str, expected: Box<[u8]>, - identity: StageIdentity, + identity: EntryIdentity, file: File, } @@ -24,7 +26,7 @@ impl FilesystemRetentionStage { /// Exclusively creates the named stage and writes its complete bytes. pub(super) fn create(root: &Dir, name: &'static str, expected: &[u8]) -> io::Result { let mut file = filesystem_catalog_artifact::create_exclusive(root, name)?; - let identity = StageIdentity::read_file(&file)?; + let identity = EntryIdentity::of_file(&file)?; file.write_all(expected)?; file.flush()?; Ok(Self { @@ -45,33 +47,29 @@ impl FilesystemRetentionStage { /// Links the verified stage into `target` under `name` without replacement. pub(super) fn link(&self, root: &Dir, target: &Dir, name: &str) -> io::Result<()> { self.verify_stage(root)?; - match root.hard_link(self.name, target, name) { - Ok(()) => {} - Err(source) if source.kind() == io::ErrorKind::AlreadyExists => {} - Err(source) => return Err(source), - } + exact_record::link_without_replacement(root, self.name, target, name)?; self.verify_stage(root)?; - verify_name(target, name, &self.expected, self.identity) + verify_named_record(target, name, &self.expected, self.identity) } /// Removes only the retained stage after confirming its linked target. pub(super) fn remove(self, root: &Dir, target: &Dir, name: &str) -> io::Result<()> { - verify_name(target, name, &self.expected, self.identity)?; + verify_named_record(target, name, &self.expected, self.identity)?; root.remove_file(self.name)?; - require_absent(root, self.name)?; - verify_name(target, name, &self.expected, self.identity) + exact_record::require_absent(root, self.name).map_err(retention_error)?; + verify_named_record(target, name, &self.expected, self.identity) } /// Renames the verified stage onto `name`, replacing it atomically. pub(super) fn replace(self, root: &Dir, name: &str) -> io::Result<()> { self.verify_stage(root)?; root.rename(self.name, root, name)?; - require_absent(root, self.name)?; - verify_name(root, name, &self.expected, self.identity) + exact_record::require_absent(root, self.name).map_err(retention_error)?; + verify_named_record(root, name, &self.expected, self.identity) } fn require_handle(&self) -> io::Result<()> { - if StageIdentity::read_file(&self.file)? == self.identity { + if EntryIdentity::of_file(&self.file)? == self.identity { Ok(()) } else { Err(invalid_data("retention stage handle changed identity")) @@ -80,76 +78,33 @@ impl FilesystemRetentionStage { fn verify_stage(&self, root: &Dir) -> io::Result<()> { self.require_handle()?; - verify_name(root, self.name, &self.expected, self.identity) - } -} - -#[derive(Clone, Copy, Debug, Eq, PartialEq)] -struct StageIdentity { - device: u64, - inode: u64, -} - -impl StageIdentity { - fn read_file(file: &File) -> io::Result { - file.metadata().map(|metadata| Self::from(&metadata)) + verify_named_record(root, self.name, &self.expected, self.identity) } } -impl From<&Metadata> for StageIdentity { - fn from(metadata: &Metadata) -> Self { - Self { - device: metadata.dev(), - inode: metadata.ino(), - } - } -} - -fn verify_name( +fn verify_named_record( directory: &Dir, name: &str, expected: &[u8], - identity: StageIdentity, + identity: EntryIdentity, ) -> io::Result<()> { - let mut options = OpenOptions::new(); - options.read(true).follow(FollowSymlinks::No).nonblock(true); - let mut file = directory.open_with(name, &options)?; - require_metadata(&file.metadata()?, expected.len(), identity)?; - require_metadata(&directory.symlink_metadata(name)?, expected.len(), identity)?; - let mut observed = vec![0_u8; expected.len()]; - file.read_exact(&mut observed)?; - let mut trailing = [0_u8; 1]; - if observed != expected || file.read(&mut trailing)? != 0 { - return Err(invalid_data("retention record bytes disagreed")); - } - require_metadata(&file.metadata()?, expected.len(), identity)?; - require_metadata(&directory.symlink_metadata(name)?, expected.len(), identity) -} - -fn require_metadata( - metadata: &Metadata, - expected_length: usize, - expected_identity: StageIdentity, -) -> io::Result<()> { - let expected_length = u64::try_from(expected_length) - .map_err(|_source| invalid_data("retention record length exceeded u64"))?; - if metadata.is_file() - && metadata.len() == expected_length - && StageIdentity::from(metadata) == expected_identity - { - Ok(()) - } else { - Err(invalid_data( - "retention record kind, length, or identity disagreed", - )) - } + exact_record::verify_named(directory, name, expected, identity).map_err(retention_error) } -fn require_absent(directory: &Dir, name: &str) -> io::Result<()> { - match directory.symlink_metadata(name) { - Err(source) if source.kind() == io::ErrorKind::NotFound => Ok(()), - Ok(_) => Err(invalid_data("removed retention stage remained visible")), - Err(source) => Err(source), +/// Maps a shared exact-record failure onto this protocol's refusal messages. +fn retention_error(error: ExactRecordError) -> io::Error { + match error { + ExactRecordError::Io(source) => source, + ExactRecordError::Refused(refusal) => invalid_data(match refusal { + ExactRecordRefusal::LengthOverflow => "retention record length exceeded u64", + ExactRecordRefusal::KindLengthOrIdentity => { + "retention record kind, length, or identity disagreed" + } + ExactRecordRefusal::Bytes | ExactRecordRefusal::TrailingBytes => { + "retention record bytes disagreed" + } + ExactRecordRefusal::RemainedVisible => "removed retention stage remained visible", + }), } } diff --git a/tests/adapters_layout_contract.rs b/tests/adapters_layout_contract.rs index eca5799..21f880b 100644 --- a/tests/adapters_layout_contract.rs +++ b/tests/adapters_layout_contract.rs @@ -124,3 +124,28 @@ fn no_source_module_spawns_a_process() -> Result<(), Box> } Ok(()) } + +const EXACT_RECORD_CONSUMERS: [(&str, &str); 1] = [( + "src/adapters/retention/filesystem_retention_stage.rs", + include_str!("../src/adapters/retention/filesystem_retention_stage.rs"), +)]; + +/// Modules ported onto `filesystem_exact_record` no longer open, read, or +/// identity-check records themselves; one implementation of the no-follow, +/// non-blocking, exact-length read serves every stage and record reader. +#[test] +fn exact_record_consumers_do_not_open_records_themselves() { + for (path, source) in EXACT_RECORD_CONSUMERS { + for forbidden in [ + "nonblock(true)", + "fn verify_name(", + "fn require_metadata(", + "fn require_absent(", + ] { + assert!( + !source.contains(forbidden), + "{path} reimplements the exact-record primitive `{forbidden}`" + ); + } + } +} From 7be0b7d703802ef9cdba1eab01ecd4db902a1c47 Mon Sep 17 00:00:00 2001 From: James Ross Date: Mon, 7 Sep 2026 15:25:26 -0700 Subject: [PATCH 093/111] Refactor: port the migration fixed-record stage onto the shared exact-record module The migration fixed-record stage carried the second copy of the stage primitives: its own device-and-inode identity type, its own no-follow non-blocking exact read with double metadata reverification, its own absence check, and its own no-replacement hard link. It now consumes filesystem_exact_record like the retention stage, mapping each shared refusal onto its existing messages byte for byte, so no observable refusal changes. The module is added to the exact-record consumer contract, which refuses any reimplementation of the primitives. The migration laws and the crash matrix's migration paths cover the port. Self-review finding D1 (P3), commit 2 of 6. Refs #78 --- CHANGELOG.md | 4 +- .../filesystem_migration_fixed_artifact.rs | 108 ++++++------------ tests/adapters_layout_contract.rs | 14 ++- 3 files changed, 44 insertions(+), 82 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c11ea4e..8b06644 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -245,8 +245,8 @@ after its public API and format compatibility policies are established. - Stage publishers share one exact-record module for no-follow non-blocking opens, exact-length verification, trailing-byte refusal, device-and-inode reverification, absence checks, and no-replacement links; the retention - stage is the first consumer, and a contract test keeps ported modules from - reimplementing those primitives. + stage and the migration fixed-record stage consume it, and a contract test + keeps ported modules from reimplementing those primitives. - The retention FIFO laws run only on Linux through `mknodat`; the `mkfifo(1)` fallback for other hosts is removed, and a contract test keeps every module under `src/` free of process spawns. The fallback's spawned child briefly diff --git a/src/adapters/store_migration/filesystem_migration_fixed_artifact.rs b/src/adapters/store_migration/filesystem_migration_fixed_artifact.rs index 06705fc..c3d2683 100644 --- a/src/adapters/store_migration/filesystem_migration_fixed_artifact.rs +++ b/src/adapters/store_migration/filesystem_migration_fixed_artifact.rs @@ -1,12 +1,14 @@ //! This module owns exact fixed-record migration publication. -use std::io::{self, Read, Write}; +use std::io::{self, Write}; -use cap_fs_ext::{FollowSymlinks, MetadataExt, OpenOptionsFollowExt, OpenOptionsSyncExt}; -use cap_std::fs::{Dir, File, Metadata, OpenOptions}; +use cap_std::fs::{Dir, File}; use super::{format_marker_decoder, migration_intent_format, migration_receipt_format}; use crate::adapters::filesystem_catalog_artifact; +use crate::adapters::filesystem_exact_record::{ + self as exact_record, EntryIdentity, ExactRecordError, ExactRecordRefusal, +}; #[derive(Clone, Copy, Debug, Eq, PartialEq)] pub(super) enum FilesystemMigrationFixedArtifact { @@ -44,7 +46,7 @@ impl FilesystemMigrationFixedArtifact { pub(super) struct FilesystemMigrationFixedStage { artifact: FilesystemMigrationFixedArtifact, expected: Box<[u8]>, - identity: FixedFileIdentity, + identity: EntryIdentity, file: File, } @@ -56,7 +58,7 @@ impl FilesystemMigrationFixedStage { ) -> io::Result { require_length(artifact, expected)?; let mut file = filesystem_catalog_artifact::create_exclusive(root, artifact.stage_name())?; - let identity = FixedFileIdentity::read_file(&file)?; + let identity = EntryIdentity::of_file(&file)?; file.write_all(expected)?; file.flush()?; Ok(Self { @@ -81,22 +83,19 @@ impl FilesystemMigrationFixedStage { ) -> io::Result<()> { self.require_record(artifact, expected)?; self.verify_stage(root)?; - match root.hard_link( + exact_record::link_without_replacement( + root, self.artifact.stage_name(), root, self.artifact.canonical_name(), - ) { - Ok(()) => {} - Err(source) if source.kind() == io::ErrorKind::AlreadyExists => {} - Err(source) => return Err(source), - } + )?; self.verify_linked_names(root) } pub(super) fn remove(self, root: &Dir) -> io::Result { self.verify_linked_names(root)?; root.remove_file(self.artifact.stage_name())?; - require_absent(root, self.artifact.stage_name())?; + exact_record::require_absent(root, self.artifact.stage_name()).map_err(migration_error)?; self.verify_canonical(root)?; Ok(self) } @@ -107,7 +106,7 @@ impl FilesystemMigrationFixedStage { pub(super) fn verify_canonical(&self, root: &Dir) -> io::Result<()> { self.require_handle()?; - verify_name( + verify_named_record( root, self.artifact.canonical_name(), &self.expected, @@ -120,7 +119,7 @@ impl FilesystemMigrationFixedStage { } fn require_handle(&self) -> io::Result<()> { - let observed = FixedFileIdentity::read_file(&self.file)?; + let observed = EntryIdentity::of_file(&self.file)?; if observed == self.identity { Ok(()) } else { @@ -141,7 +140,7 @@ impl FilesystemMigrationFixedStage { } fn verify_stage(&self, root: &Dir) -> io::Result<()> { - verify_name( + verify_named_record( root, self.artifact.stage_name(), &self.expected, @@ -151,7 +150,7 @@ impl FilesystemMigrationFixedStage { fn verify_linked_names(&self, root: &Dir) -> io::Result<()> { self.verify_stage(root)?; - verify_name( + verify_named_record( root, self.artifact.canonical_name(), &self.expected, @@ -160,64 +159,29 @@ impl FilesystemMigrationFixedStage { } } -#[derive(Clone, Copy, Debug, Eq, PartialEq)] -struct FixedFileIdentity { - device: u64, - inode: u64, -} - -impl FixedFileIdentity { - fn read_file(file: &File) -> io::Result { - file.metadata().map(|metadata| Self::from(&metadata)) - } -} - -impl From<&Metadata> for FixedFileIdentity { - fn from(metadata: &Metadata) -> Self { - Self { - device: metadata.dev(), - inode: metadata.ino(), - } - } -} - -fn verify_name( +fn verify_named_record( root: &Dir, name: &str, expected: &[u8], - identity: FixedFileIdentity, + identity: EntryIdentity, ) -> io::Result<()> { - let mut options = OpenOptions::new(); - options.read(true).follow(FollowSymlinks::No).nonblock(true); - let mut file = root.open_with(name, &options)?; - require_metadata(&file.metadata()?, expected.len(), identity)?; - require_metadata(&root.symlink_metadata(name)?, expected.len(), identity)?; - let mut observed = vec![0_u8; expected.len()]; - file.read_exact(&mut observed)?; - let mut trailing = [0_u8; 1]; - if observed != expected || file.read(&mut trailing)? != 0 { - return Err(invalid_data("migration fixed-record bytes disagreed")); - } - require_metadata(&file.metadata()?, expected.len(), identity)?; - require_metadata(&root.symlink_metadata(name)?, expected.len(), identity) + exact_record::verify_named(root, name, expected, identity).map_err(migration_error) } -fn require_metadata( - metadata: &Metadata, - expected_length: usize, - expected_identity: FixedFileIdentity, -) -> io::Result<()> { - let expected_length = u64::try_from(expected_length) - .map_err(|_source| invalid_data("migration fixed-record length exceeded u64"))?; - if metadata.is_file() - && metadata.len() == expected_length - && FixedFileIdentity::from(metadata) == expected_identity - { - Ok(()) - } else { - Err(invalid_data( - "migration fixed-record kind, length, or identity disagreed", - )) +/// Maps a shared exact-record failure onto this protocol's refusal messages. +fn migration_error(error: ExactRecordError) -> io::Error { + match error { + ExactRecordError::Io(source) => source, + ExactRecordError::Refused(refusal) => invalid_data(match refusal { + ExactRecordRefusal::LengthOverflow => "migration fixed-record length exceeded u64", + ExactRecordRefusal::KindLengthOrIdentity => { + "migration fixed-record kind, length, or identity disagreed" + } + ExactRecordRefusal::Bytes | ExactRecordRefusal::TrailingBytes => { + "migration fixed-record bytes disagreed" + } + ExactRecordRefusal::RemainedVisible => "removed migration stage remained visible", + }), } } @@ -229,14 +193,6 @@ fn require_length(artifact: FilesystemMigrationFixedArtifact, expected: &[u8]) - } } -fn require_absent(root: &Dir, name: &str) -> io::Result<()> { - match root.symlink_metadata(name) { - Err(source) if source.kind() == io::ErrorKind::NotFound => Ok(()), - Ok(_) => Err(invalid_data("removed migration stage remained visible")), - Err(source) => Err(source), - } -} - fn invalid_data(message: &'static str) -> io::Error { io::Error::new(io::ErrorKind::InvalidData, message) } diff --git a/tests/adapters_layout_contract.rs b/tests/adapters_layout_contract.rs index 21f880b..10a6b4e 100644 --- a/tests/adapters_layout_contract.rs +++ b/tests/adapters_layout_contract.rs @@ -125,10 +125,16 @@ fn no_source_module_spawns_a_process() -> Result<(), Box> Ok(()) } -const EXACT_RECORD_CONSUMERS: [(&str, &str); 1] = [( - "src/adapters/retention/filesystem_retention_stage.rs", - include_str!("../src/adapters/retention/filesystem_retention_stage.rs"), -)]; +const EXACT_RECORD_CONSUMERS: [(&str, &str); 2] = [ + ( + "src/adapters/retention/filesystem_retention_stage.rs", + include_str!("../src/adapters/retention/filesystem_retention_stage.rs"), + ), + ( + "src/adapters/store_migration/filesystem_migration_fixed_artifact.rs", + include_str!("../src/adapters/store_migration/filesystem_migration_fixed_artifact.rs"), + ), +]; /// Modules ported onto `filesystem_exact_record` no longer open, read, or /// identity-check records themselves; one implementation of the no-follow, From 80908e38f12ea78b3e68bf22dda4b9e06c1d1571 Mon Sep 17 00:00:00 2001 From: James Ross Date: Mon, 7 Sep 2026 15:36:02 -0700 Subject: [PATCH 094/111] Refactor: read optional retention records through the shared exact-record module The retention current-state reader carried the third copy of the bounded read: open without following links or blocking, refuse a non-regular entry or a length disagreement, read exactly the declared bytes, refuse trailing bytes. The head, manifest, committed-root, and predecessor-root reads all go through it, as does the catalog-head binding. filesystem_exact_record gains read_exact_optional and the KindOrLength refusal, pinned by two laws (absent reads as None while an exact record reads its bytes; a short file or a directory refuses before any byte is read). The retention reader is now a wrapper that maps shared refusals onto RecordLengthOverflow, RecordTrailingBytes, and RecordKindOrLength, so every typed refusal callers already downcast to is unchanged. Both stage mappers fold the new variant into their kind-or-length message. The module joins the consumer contract. Self-review finding D1 (P3), commit 3 of 6. Refs #78 --- CHANGELOG.md | 11 ++--- src/adapters/filesystem_exact_record.rs | 29 ++++++++++++ src/adapters/filesystem_exact_record_tests.rs | 41 ++++++++++++++++- .../retention/filesystem_retention_current.rs | 45 ++++++++++--------- .../retention/filesystem_retention_stage.rs | 2 +- .../filesystem_migration_fixed_artifact.rs | 2 +- tests/adapters_layout_contract.rs | 6 ++- 7 files changed, 104 insertions(+), 32 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8b06644..2f0c8a1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -242,11 +242,12 @@ after its public API and format compatibility policies are established. ### Changed -- Stage publishers share one exact-record module for no-follow non-blocking - opens, exact-length verification, trailing-byte refusal, device-and-inode - reverification, absence checks, and no-replacement links; the retention - stage and the migration fixed-record stage consume it, and a contract test - keeps ported modules from reimplementing those primitives. +- Stage publishers and fixed-record readers share one exact-record module for + no-follow non-blocking opens, exact-length reads, trailing-byte refusal, + device-and-inode reverification, absence checks, and no-replacement links; + the retention stage, the migration fixed-record stage, and the retention + current-state reader consume it, and a contract test keeps ported modules + from reimplementing those primitives. - The retention FIFO laws run only on Linux through `mknodat`; the `mkfifo(1)` fallback for other hosts is removed, and a contract test keeps every module under `src/` free of process spawns. The fallback's spawned child briefly diff --git a/src/adapters/filesystem_exact_record.rs b/src/adapters/filesystem_exact_record.rs index 6d246b0..ee12aae 100644 --- a/src/adapters/filesystem_exact_record.rs +++ b/src/adapters/filesystem_exact_record.rs @@ -43,6 +43,8 @@ impl From<&Metadata> for EntryIdentity { pub(super) enum ExactRecordRefusal { /// The expected length does not fit the filesystem's `u64` length. LengthOverflow, + /// The entry is not a regular file of the expected length. + KindOrLength, /// The entry's kind, length, or device and inode identity disagreed. KindLengthOrIdentity, /// The entry's bytes disagreed with the expected record. @@ -66,6 +68,7 @@ impl fmt::Display for ExactRecordRefusal { fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { formatter.write_str(match self { Self::LengthOverflow => "exact record length exceeded the filesystem's range", + Self::KindOrLength => "exact record kind or length disagreed", Self::KindLengthOrIdentity => "exact record kind, length, or identity disagreed", Self::Bytes => "exact record bytes disagreed", Self::TrailingBytes => "exact record carried trailing bytes", @@ -104,6 +107,32 @@ impl From for ExactRecordError { } } +/// Reads exactly `length` bytes of the regular file `name`, or `None` if absent. +/// +/// The open follows no links and does not block. A present entry that is not +/// a regular file of exactly `length` bytes refuses before any byte is read, +/// and bytes beyond `length` refuse after. +pub(super) fn read_exact_optional( + directory: &Dir, + name: &str, + length: usize, +) -> Result>, ExactRecordError> { + let mut file = match open_read(directory, name) { + Ok(file) => file, + Err(source) if source.kind() == io::ErrorKind::NotFound => return Ok(None), + Err(source) => return Err(source.into()), + }; + let expected_length = exact_length(length)?; + let metadata = file.metadata()?; + if !metadata.is_file() || metadata.len() != expected_length { + return Err(ExactRecordRefusal::KindOrLength.into()); + } + let mut bytes = vec![0_u8; length]; + file.read_exact(&mut bytes)?; + require_no_trailing_bytes(&mut file)?; + Ok(Some(bytes)) +} + /// Reverifies that `name` is exactly `expected` with `identity`, before and after reading. /// /// Both the opened handle and the directory entry are checked for kind, diff --git a/src/adapters/filesystem_exact_record_tests.rs b/src/adapters/filesystem_exact_record_tests.rs index 40868cc..07972ff 100644 --- a/src/adapters/filesystem_exact_record_tests.rs +++ b/src/adapters/filesystem_exact_record_tests.rs @@ -1,11 +1,11 @@ -//! Exact-record primitive laws: identity reverification, absence, no replacement. +//! Exact-record primitive laws: bounded reads, identity reverification, absence, no replacement. use std::error::Error; use std::fs; use cap_std::fs::Dir; -use super::{EntryIdentity, ExactRecordError, ExactRecordRefusal}; +use super::{EntryIdentity, ExactRecordError, ExactRecordRefusal, read_exact_optional}; use crate::adapters::filesystem_test_sandbox::TestDirectory; fn open(sandbox: &TestDirectory) -> Result> { @@ -22,6 +22,43 @@ fn refusal(error: ExactRecordError) -> Result } } +#[test] +fn absent_record_reads_as_none_and_exact_record_reads_its_bytes() -> Result<(), Box> { + let sandbox = TestDirectory::create("exact-record-optional")?; + let directory = open(&sandbox)?; + fs::write(sandbox.path().join("record"), b"exact")?; + + assert!(read_exact_optional(&directory, "absent", 5)?.is_none()); + assert_eq!( + read_exact_optional(&directory, "record", 5)?.as_deref(), + Some(b"exact".as_slice()) + ); + drop(directory); + sandbox.remove()?; + Ok(()) +} + +#[test] +fn wrong_length_or_kind_refuses_before_reading() -> Result<(), Box> { + let sandbox = TestDirectory::create("exact-record-kind-or-length")?; + let directory = open(&sandbox)?; + fs::write(sandbox.path().join("short"), b"abc")?; + fs::create_dir(sandbox.path().join("directory"))?; + + let short = read_exact_optional(&directory, "short", 5) + .err() + .ok_or("short record admitted")?; + let kind = read_exact_optional(&directory, "directory", 5) + .err() + .ok_or("directory admitted as a record")?; + + assert_eq!(refusal(short)?, ExactRecordRefusal::KindOrLength); + assert_eq!(refusal(kind)?, ExactRecordRefusal::KindOrLength); + drop(directory); + sandbox.remove()?; + Ok(()) +} + #[test] fn verify_named_refuses_different_bytes_and_a_byte_equal_substitute() -> Result<(), Box> { diff --git a/src/adapters/retention/filesystem_retention_current.rs b/src/adapters/retention/filesystem_retention_current.rs index 0488dca..6c4ecd9 100644 --- a/src/adapters/retention/filesystem_retention_current.rs +++ b/src/adapters/retention/filesystem_retention_current.rs @@ -1,9 +1,9 @@ //! This module owns exact observation of the current filesystem retention state. -use std::io::{self, Read}; +use std::io; -use cap_fs_ext::{DirExt, FollowSymlinks, OpenOptionsFollowExt, OpenOptionsSyncExt}; -use cap_std::fs::{Dir, OpenOptions}; +use cap_fs_ext::DirExt; +use cap_std::fs::Dir; use super::filesystem_retention_pool_name as pool_name; use super::root_header_decoder; @@ -11,6 +11,9 @@ use super::{ AdmittedRetentionManifest, AdmittedRetentionRoot, ChecksummedRetentionHead, RetentionCurrentStateRefusal, RetentionPublicationPreparation, RetentionTransitionDisposition, }; +use crate::adapters::filesystem_exact_record::{ + self as exact_record, ExactRecordError, ExactRecordRefusal, +}; use crate::{RetentionGenerationExpectation, RetentionHead, RetentionManifest}; const HEAD_LENGTH: usize = super::head_decoder::ENCODED_LENGTH; @@ -265,29 +268,27 @@ fn require_initial_publication( } } +/// Reads one optional exact record, mapping shared refusals onto this protocol's. pub(super) fn read_exact_optional( directory: &Dir, name: &str, length: usize, ) -> io::Result>> { - let mut options = OpenOptions::new(); - options.read(true).follow(FollowSymlinks::No).nonblock(true); - let mut file = match directory.open_with(name, &options) { - Ok(file) => file, - Err(source) if source.kind() == io::ErrorKind::NotFound => return Ok(None), - Err(source) => return Err(source), - }; - let expected_length = u64::try_from(length) - .map_err(|_source| RetentionCurrentStateRefusal::RecordLengthOverflow.into_io())?; - let metadata = file.metadata()?; - if !metadata.is_file() || metadata.len() != expected_length { - return Err(RetentionCurrentStateRefusal::RecordKindOrLength.into_io()); - } - let mut bytes = vec![0_u8; length]; - file.read_exact(&mut bytes)?; - let mut trailing = [0_u8; 1]; - if file.read(&mut trailing)? != 0 { - return Err(RetentionCurrentStateRefusal::RecordTrailingBytes.into_io()); + match exact_record::read_exact_optional(directory, name, length) { + Ok(bytes) => Ok(bytes.map(Vec::into_boxed_slice)), + Err(ExactRecordError::Io(source)) => Err(source), + Err(ExactRecordError::Refused(refusal)) => Err(match refusal { + ExactRecordRefusal::LengthOverflow => { + RetentionCurrentStateRefusal::RecordLengthOverflow + } + ExactRecordRefusal::TrailingBytes => RetentionCurrentStateRefusal::RecordTrailingBytes, + ExactRecordRefusal::KindOrLength + | ExactRecordRefusal::KindLengthOrIdentity + | ExactRecordRefusal::Bytes + | ExactRecordRefusal::RemainedVisible => { + RetentionCurrentStateRefusal::RecordKindOrLength + } + } + .into_io()), } - Ok(Some(bytes.into_boxed_slice())) } diff --git a/src/adapters/retention/filesystem_retention_stage.rs b/src/adapters/retention/filesystem_retention_stage.rs index ed77e62..35a2459 100644 --- a/src/adapters/retention/filesystem_retention_stage.rs +++ b/src/adapters/retention/filesystem_retention_stage.rs @@ -97,7 +97,7 @@ fn retention_error(error: ExactRecordError) -> io::Error { ExactRecordError::Io(source) => source, ExactRecordError::Refused(refusal) => invalid_data(match refusal { ExactRecordRefusal::LengthOverflow => "retention record length exceeded u64", - ExactRecordRefusal::KindLengthOrIdentity => { + ExactRecordRefusal::KindOrLength | ExactRecordRefusal::KindLengthOrIdentity => { "retention record kind, length, or identity disagreed" } ExactRecordRefusal::Bytes | ExactRecordRefusal::TrailingBytes => { diff --git a/src/adapters/store_migration/filesystem_migration_fixed_artifact.rs b/src/adapters/store_migration/filesystem_migration_fixed_artifact.rs index c3d2683..50bca1e 100644 --- a/src/adapters/store_migration/filesystem_migration_fixed_artifact.rs +++ b/src/adapters/store_migration/filesystem_migration_fixed_artifact.rs @@ -174,7 +174,7 @@ fn migration_error(error: ExactRecordError) -> io::Error { ExactRecordError::Io(source) => source, ExactRecordError::Refused(refusal) => invalid_data(match refusal { ExactRecordRefusal::LengthOverflow => "migration fixed-record length exceeded u64", - ExactRecordRefusal::KindLengthOrIdentity => { + ExactRecordRefusal::KindOrLength | ExactRecordRefusal::KindLengthOrIdentity => { "migration fixed-record kind, length, or identity disagreed" } ExactRecordRefusal::Bytes | ExactRecordRefusal::TrailingBytes => { diff --git a/tests/adapters_layout_contract.rs b/tests/adapters_layout_contract.rs index 10a6b4e..78c2797 100644 --- a/tests/adapters_layout_contract.rs +++ b/tests/adapters_layout_contract.rs @@ -125,7 +125,11 @@ fn no_source_module_spawns_a_process() -> Result<(), Box> Ok(()) } -const EXACT_RECORD_CONSUMERS: [(&str, &str); 2] = [ +const EXACT_RECORD_CONSUMERS: [(&str, &str); 3] = [ + ( + "src/adapters/retention/filesystem_retention_current.rs", + include_str!("../src/adapters/retention/filesystem_retention_current.rs"), + ), ( "src/adapters/retention/filesystem_retention_stage.rs", include_str!("../src/adapters/retention/filesystem_retention_stage.rs"), From d7510eee02190de763d73d0ea55b024af19b2b54 Mon Sep 17 00:00:00 2001 From: James Ross Date: Mon, 7 Sep 2026 15:45:40 -0700 Subject: [PATCH 095/111] Refactor: read version-two records through the shared exact-record module The version-two record reader carried the fourth copy of the bounded read for FORMAT, migration.intent, and migration.receipt. It now consumes filesystem_exact_record's new read_exact_regular, which keeps absence as the filesystem's own NotFound error because those records are required, and maps each shared refusal onto VersionTwoRecordRefusal by record name: LengthOverflow, KindOrLength, and TrailingBytes are unchanged for callers. One law pins the required read (an exact record reads its bytes; an absent one is NotFound), and the module joins the consumer contract, so all four readers and stages now share one no-follow, non-blocking, exact-length implementation. Self-review finding D1 (P3), commit 4 of 6. Refs #78 --- CHANGELOG.md | 6 ++-- src/adapters/filesystem_exact_record.rs | 19 ++++++++-- src/adapters/filesystem_exact_record_tests.rs | 24 ++++++++++++- .../filesystem_version_two_records.rs | 35 +++++++++---------- tests/adapters_layout_contract.rs | 6 +++- 5 files changed, 64 insertions(+), 26 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2f0c8a1..73d3f24 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -245,9 +245,9 @@ after its public API and format compatibility policies are established. - Stage publishers and fixed-record readers share one exact-record module for no-follow non-blocking opens, exact-length reads, trailing-byte refusal, device-and-inode reverification, absence checks, and no-replacement links; - the retention stage, the migration fixed-record stage, and the retention - current-state reader consume it, and a contract test keeps ported modules - from reimplementing those primitives. + the retention stage, the migration fixed-record stage, the retention + current-state reader, and the version-two record reader consume it, and a + contract test keeps ported modules from reimplementing those primitives. - The retention FIFO laws run only on Linux through `mknodat`; the `mkfifo(1)` fallback for other hosts is removed, and a contract test keeps every module under `src/` free of process spawns. The fallback's spawned child briefly diff --git a/src/adapters/filesystem_exact_record.rs b/src/adapters/filesystem_exact_record.rs index ee12aae..3852380 100644 --- a/src/adapters/filesystem_exact_record.rs +++ b/src/adapters/filesystem_exact_record.rs @@ -122,6 +122,21 @@ pub(super) fn read_exact_optional( Err(source) if source.kind() == io::ErrorKind::NotFound => return Ok(None), Err(source) => return Err(source.into()), }; + read_opened_exactly(&mut file, length).map(Some) +} + +/// Reads exactly `length` bytes of the regular file `name`; absence is the +/// filesystem's own `NotFound` error, since the record is required. +pub(super) fn read_exact_regular( + directory: &Dir, + name: &str, + length: usize, +) -> Result, ExactRecordError> { + let mut file = open_read(directory, name)?; + read_opened_exactly(&mut file, length) +} + +fn read_opened_exactly(file: &mut File, length: usize) -> Result, ExactRecordError> { let expected_length = exact_length(length)?; let metadata = file.metadata()?; if !metadata.is_file() || metadata.len() != expected_length { @@ -129,8 +144,8 @@ pub(super) fn read_exact_optional( } let mut bytes = vec![0_u8; length]; file.read_exact(&mut bytes)?; - require_no_trailing_bytes(&mut file)?; - Ok(Some(bytes)) + require_no_trailing_bytes(file)?; + Ok(bytes) } /// Reverifies that `name` is exactly `expected` with `identity`, before and after reading. diff --git a/src/adapters/filesystem_exact_record_tests.rs b/src/adapters/filesystem_exact_record_tests.rs index 07972ff..0de8ace 100644 --- a/src/adapters/filesystem_exact_record_tests.rs +++ b/src/adapters/filesystem_exact_record_tests.rs @@ -5,7 +5,9 @@ use std::fs; use cap_std::fs::Dir; -use super::{EntryIdentity, ExactRecordError, ExactRecordRefusal, read_exact_optional}; +use super::{ + EntryIdentity, ExactRecordError, ExactRecordRefusal, read_exact_optional, read_exact_regular, +}; use crate::adapters::filesystem_test_sandbox::TestDirectory; fn open(sandbox: &TestDirectory) -> Result> { @@ -38,6 +40,26 @@ fn absent_record_reads_as_none_and_exact_record_reads_its_bytes() -> Result<(), Ok(()) } +#[test] +fn required_record_reports_absence_as_the_filesystem_error() -> Result<(), Box> { + let sandbox = TestDirectory::create("exact-record-required")?; + let directory = open(&sandbox)?; + fs::write(sandbox.path().join("record"), b"exact")?; + + assert_eq!(read_exact_regular(&directory, "record", 5)?, b"exact"); + let absent = read_exact_regular(&directory, "absent", 5) + .err() + .ok_or("absent required record was admitted")?; + + assert!(matches!( + absent, + ExactRecordError::Io(ref source) if source.kind() == std::io::ErrorKind::NotFound + )); + drop(directory); + sandbox.remove()?; + Ok(()) +} + #[test] fn wrong_length_or_kind_refuses_before_reading() -> Result<(), Box> { let sandbox = TestDirectory::create("exact-record-kind-or-length")?; diff --git a/src/adapters/filesystem_version_two_records.rs b/src/adapters/filesystem_version_two_records.rs index 0b11841..de6b4c3 100644 --- a/src/adapters/filesystem_version_two_records.rs +++ b/src/adapters/filesystem_version_two_records.rs @@ -1,11 +1,11 @@ //! This module owns joint admission of the three fixed version-two migration records. -use std::io::{self, Read}; +use std::io; -use cap_fs_ext::{FollowSymlinks, OpenOptionsFollowExt, OpenOptionsSyncExt}; -use cap_std::fs::{Dir, OpenOptions}; +use cap_std::fs::Dir; use super::VersionTwoRecordRefusal as Refusal; +use super::filesystem_exact_record::{self as exact_record, ExactRecordError, ExactRecordRefusal}; use super::store_migration::{ FORMAT_MARKER_LENGTH, MIGRATION_INTENT_LENGTH, MIGRATION_RECEIPT_LENGTH, }; @@ -74,21 +74,18 @@ pub(super) fn admit(root: &Dir) -> io::Result { )) } +/// Reads one required record, mapping shared refusals onto this record's refusal. fn read_exact(root: &Dir, name: &'static str, length: usize) -> io::Result> { - let mut options = OpenOptions::new(); - options.read(true).follow(FollowSymlinks::No).nonblock(true); - let mut file = root.open_with(name, &options)?; - let expected_length = - u64::try_from(length).map_err(|_source| Refusal::LengthOverflow { name }.into_io())?; - let metadata = file.metadata()?; - if !metadata.is_file() || metadata.len() != expected_length { - return Err(Refusal::KindOrLength { name }.into_io()); - } - let mut bytes = vec![0_u8; length]; - file.read_exact(&mut bytes)?; - let mut trailing = [0_u8; 1]; - if file.read(&mut trailing)? != 0 { - return Err(Refusal::TrailingBytes { name }.into_io()); - } - Ok(bytes) + exact_record::read_exact_regular(root, name, length).map_err(|error| match error { + ExactRecordError::Io(source) => source, + ExactRecordError::Refused(refusal) => match refusal { + ExactRecordRefusal::LengthOverflow => Refusal::LengthOverflow { name }, + ExactRecordRefusal::TrailingBytes => Refusal::TrailingBytes { name }, + ExactRecordRefusal::KindOrLength + | ExactRecordRefusal::KindLengthOrIdentity + | ExactRecordRefusal::Bytes + | ExactRecordRefusal::RemainedVisible => Refusal::KindOrLength { name }, + } + .into_io(), + }) } diff --git a/tests/adapters_layout_contract.rs b/tests/adapters_layout_contract.rs index 78c2797..824e7b7 100644 --- a/tests/adapters_layout_contract.rs +++ b/tests/adapters_layout_contract.rs @@ -125,7 +125,11 @@ fn no_source_module_spawns_a_process() -> Result<(), Box> Ok(()) } -const EXACT_RECORD_CONSUMERS: [(&str, &str); 3] = [ +const EXACT_RECORD_CONSUMERS: [(&str, &str); 4] = [ + ( + "src/adapters/filesystem_version_two_records.rs", + include_str!("../src/adapters/filesystem_version_two_records.rs"), + ), ( "src/adapters/retention/filesystem_retention_current.rs", include_str!("../src/adapters/retention/filesystem_retention_current.rs"), From 41e680f83ba424fa1f130893e6fe247f6de83ac2 Mon Sep 17 00:00:00 2001 From: James Ross Date: Mon, 7 Sep 2026 15:55:28 -0700 Subject: [PATCH 096/111] Refactor: link version-one pool entries through the shared exact-record module The version-one segment and catalog publishers linked their sealed stages into the immutable pools through filesystem_catalog_artifact's own copy of the no-replacement hard link. The retention and migration stages already use the shared primitive, so the copy is deleted and both publishers call filesystem_exact_record::link_without_replacement directly. Behaviour is identical: an existing target is left for post-link verification, never replaced. The crash matrix's segment and catalog publication protocols cover the repoint. Self-review finding D1 (P3), commit 5 of 6. Refs #78 --- CHANGELOG.md | 5 +++-- src/adapters/filesystem_catalog_artifact.rs | 13 ------------- src/adapters/filesystem_catalog_catalog.rs | 2 +- src/adapters/filesystem_catalog_segment.rs | 2 +- 4 files changed, 5 insertions(+), 17 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 73d3f24..80014ed 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -246,8 +246,9 @@ after its public API and format compatibility policies are established. no-follow non-blocking opens, exact-length reads, trailing-byte refusal, device-and-inode reverification, absence checks, and no-replacement links; the retention stage, the migration fixed-record stage, the retention - current-state reader, and the version-two record reader consume it, and a - contract test keeps ported modules from reimplementing those primitives. + current-state reader, the version-two record reader, and the version-one + segment and catalog publishers' pool links consume it, and a contract test + keeps ported modules from reimplementing those primitives. - The retention FIFO laws run only on Linux through `mknodat`; the `mkfifo(1)` fallback for other hosts is removed, and a contract test keeps every module under `src/` free of process spawns. The fallback's spawned child briefly diff --git a/src/adapters/filesystem_catalog_artifact.rs b/src/adapters/filesystem_catalog_artifact.rs index 0b9aab6..7adf710 100644 --- a/src/adapters/filesystem_catalog_artifact.rs +++ b/src/adapters/filesystem_catalog_artifact.rs @@ -30,19 +30,6 @@ pub(super) fn synchronize_directory(directory: &Dir) -> io::Result<()> { .sync_all() } -pub(super) fn link_without_replacement( - source_directory: &Dir, - source_name: &str, - destination_directory: &Dir, - destination_name: &str, -) -> io::Result<()> { - match source_directory.hard_link(source_name, destination_directory, destination_name) { - Ok(()) => Ok(()), - Err(source) if source.kind() == io::ErrorKind::AlreadyExists => Ok(()), - Err(source) => Err(source), - } -} - pub(super) fn verify_segment( directory: &Dir, name: &str, diff --git a/src/adapters/filesystem_catalog_catalog.rs b/src/adapters/filesystem_catalog_catalog.rs index 023e98e..4539e90 100644 --- a/src/adapters/filesystem_catalog_catalog.rs +++ b/src/adapters/filesystem_catalog_catalog.rs @@ -61,7 +61,7 @@ pub(super) fn link( filesystem_catalog_publisher::CURRENT_CATALOG, catalog, )?; - filesystem_catalog_artifact::link_without_replacement( + super::filesystem_exact_record::link_without_replacement( &publisher.staging, filesystem_catalog_publisher::CURRENT_CATALOG, &publisher.catalogs, diff --git a/src/adapters/filesystem_catalog_segment.rs b/src/adapters/filesystem_catalog_segment.rs index d98d912..5af91ec 100644 --- a/src/adapters/filesystem_catalog_segment.rs +++ b/src/adapters/filesystem_catalog_segment.rs @@ -17,7 +17,7 @@ pub(super) fn link( segment, publisher.policy.segment_read(), )?; - filesystem_catalog_artifact::link_without_replacement( + super::filesystem_exact_record::link_without_replacement( &publisher.staging, filesystem_catalog_publisher::CURRENT_SEGMENT, &publisher.segments, From 39014f147642d050a289ade017b39eb6404f91bc Mon Sep 17 00:00:00 2001 From: James Ross Date: Mon, 7 Sep 2026 16:13:31 -0700 Subject: [PATCH 097/111] Refactor: collapse the three pinned-directory identity types onto EntryIdentity The recovery namespace, the migration inventory directory, and the migration namespace directory each declared an identical private DirectoryIdentity (device and inode read from a pinned Dir) to detect a replaced protocol directory between phases. The shared exact-record module's EntryIdentity is the same pair, so it gains of_directory and the three copies are deleted; every comparison site is unchanged in meaning. Recovery and migration laws and the crash matrix cover the identity checks. Self-review finding D1 (P3), commit 6 of 6. Refs #78 --- CHANGELOG.md | 5 +-- src/adapters/filesystem_exact_record.rs | 7 ++++ src/adapters/filesystem_recovery_namespace.rs | 34 ++++-------------- .../filesystem_inventory_directory.rs | 36 ++++--------------- ...ilesystem_migration_namespace_directory.rs | 32 +++++------------ 5 files changed, 32 insertions(+), 82 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 80014ed..928909c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -247,8 +247,9 @@ after its public API and format compatibility policies are established. device-and-inode reverification, absence checks, and no-replacement links; the retention stage, the migration fixed-record stage, the retention current-state reader, the version-two record reader, and the version-one - segment and catalog publishers' pool links consume it, and a contract test - keeps ported modules from reimplementing those primitives. + segment and catalog publishers' pool links consume it, the three pinned + directory identity types collapse onto its `EntryIdentity`, and a contract + test keeps ported modules from reimplementing those primitives. - The retention FIFO laws run only on Linux through `mknodat`; the `mkfifo(1)` fallback for other hosts is removed, and a contract test keeps every module under `src/` free of process spawns. The fallback's spawned child briefly diff --git a/src/adapters/filesystem_exact_record.rs b/src/adapters/filesystem_exact_record.rs index 3852380..14beda7 100644 --- a/src/adapters/filesystem_exact_record.rs +++ b/src/adapters/filesystem_exact_record.rs @@ -27,6 +27,13 @@ impl EntryIdentity { pub(super) fn of_file(file: &File) -> io::Result { file.metadata().map(|metadata| Self::from(&metadata)) } + + /// Reads the identity behind a pinned directory capability. + pub(super) fn of_directory(directory: &Dir) -> io::Result { + directory + .dir_metadata() + .map(|metadata| Self::from(&metadata)) + } } impl From<&Metadata> for EntryIdentity { diff --git a/src/adapters/filesystem_recovery_namespace.rs b/src/adapters/filesystem_recovery_namespace.rs index 95dc0ce..6ced5dc 100644 --- a/src/adapters/filesystem_recovery_namespace.rs +++ b/src/adapters/filesystem_recovery_namespace.rs @@ -2,8 +2,9 @@ use std::io; -use cap_fs_ext::MetadataExt; -use cap_std::fs::{Dir, Metadata}; +use cap_std::fs::Dir; + +use super::filesystem_exact_record::EntryIdentity; use super::{ RecoveryInventoryError, RecoveryInventoryOperation, RecoveryNamespace, sync_capable_directory, @@ -12,7 +13,7 @@ use super::{ pub(super) struct PinnedRecoveryDirectory { namespace: RecoveryNamespace, name: &'static str, - identity: DirectoryIdentity, + identity: EntryIdentity, directory: Dir, } @@ -25,7 +26,7 @@ impl PinnedRecoveryDirectory { let directory = sync_capable_directory::open(root, name).map_err(|source| { RecoveryInventoryError::io(namespace, RecoveryInventoryOperation::OpenNamespace, source) })?; - let identity = DirectoryIdentity::read(&directory).map_err(|source| { + let identity = EntryIdentity::of_directory(&directory).map_err(|source| { RecoveryInventoryError::io(namespace, RecoveryInventoryOperation::OpenNamespace, source) })?; Ok(Self { @@ -44,7 +45,7 @@ impl PinnedRecoveryDirectory { source, ) })?; - let observed = DirectoryIdentity::from(&metadata); + let observed = EntryIdentity::from(&metadata); if metadata.is_dir() && observed == self.identity { return Ok(()); } @@ -62,26 +63,3 @@ impl PinnedRecoveryDirectory { &self.directory } } - -#[derive(Clone, Copy, Debug, Eq, PartialEq)] -struct DirectoryIdentity { - device: u64, - inode: u64, -} - -impl DirectoryIdentity { - fn read(directory: &Dir) -> io::Result { - directory - .dir_metadata() - .map(|metadata| Self::from(&metadata)) - } -} - -impl From<&Metadata> for DirectoryIdentity { - fn from(metadata: &Metadata) -> Self { - Self { - device: metadata.dev(), - inode: metadata.ino(), - } - } -} diff --git a/src/adapters/store_migration/filesystem_inventory_directory.rs b/src/adapters/store_migration/filesystem_inventory_directory.rs index c797f72..e48f836 100644 --- a/src/adapters/store_migration/filesystem_inventory_directory.rs +++ b/src/adapters/store_migration/filesystem_inventory_directory.rs @@ -1,7 +1,8 @@ //! This module owns pinned migration pool-directory identity. -use cap_fs_ext::MetadataExt; -use cap_std::fs::{Dir, Metadata}; +use cap_std::fs::Dir; + +use crate::adapters::filesystem_exact_record::EntryIdentity; use super::filesystem_inventory_error::{ FilesystemMigrationInventoryError, FilesystemMigrationInventoryOperation, @@ -12,7 +13,7 @@ use crate::adapters::sync_capable_directory; pub(super) struct PinnedMigrationPoolDirectory { pool: MigrationInventoryPool, name: &'static str, - identity: DirectoryIdentity, + identity: EntryIdentity, directory: Dir, } @@ -29,7 +30,7 @@ impl PinnedMigrationPoolDirectory { source, } })?; - let identity = DirectoryIdentity::read(&directory).map_err(|source| { + let identity = EntryIdentity::of_directory(&directory).map_err(|source| { FilesystemMigrationInventoryError::Io { namespace: MigrationInventoryNamespace::from(pool), operation: FilesystemMigrationInventoryOperation::OpenPool, @@ -45,7 +46,7 @@ impl PinnedMigrationPoolDirectory { } pub(super) fn verify(&self, root: &Dir) -> Result<(), FilesystemMigrationInventoryError> { - let handle = DirectoryIdentity::read(&self.directory).map_err(|source| { + let handle = EntryIdentity::of_directory(&self.directory).map_err(|source| { FilesystemMigrationInventoryError::Io { namespace: MigrationInventoryNamespace::from(self.pool), operation: FilesystemMigrationInventoryOperation::VerifyPool, @@ -59,7 +60,7 @@ impl PinnedMigrationPoolDirectory { source, } })?; - let current = DirectoryIdentity::from(&metadata); + let current = EntryIdentity::from(&metadata); if metadata.is_dir() && handle == self.identity && current == self.identity { Ok(()) } else { @@ -71,26 +72,3 @@ impl PinnedMigrationPoolDirectory { &self.directory } } - -#[derive(Clone, Copy, Debug, Eq, PartialEq)] -struct DirectoryIdentity { - device: u64, - inode: u64, -} - -impl DirectoryIdentity { - fn read(directory: &Dir) -> std::io::Result { - directory - .dir_metadata() - .map(|metadata| Self::from(&metadata)) - } -} - -impl From<&Metadata> for DirectoryIdentity { - fn from(metadata: &Metadata) -> Self { - Self { - device: metadata.dev(), - inode: metadata.ino(), - } - } -} diff --git a/src/adapters/store_migration/filesystem_migration_namespace_directory.rs b/src/adapters/store_migration/filesystem_migration_namespace_directory.rs index 4368c85..297dd82 100644 --- a/src/adapters/store_migration/filesystem_migration_namespace_directory.rs +++ b/src/adapters/store_migration/filesystem_migration_namespace_directory.rs @@ -3,8 +3,9 @@ use std::ffi::OsStr; use std::io; -use cap_fs_ext::MetadataExt; -use cap_std::fs::{Dir, Metadata}; +use cap_std::fs::Dir; + +use crate::adapters::filesystem_exact_record::EntryIdentity; use crate::adapters::{ filesystem_catalog_artifact, filesystem_platform_profile, sync_capable_directory, @@ -12,7 +13,7 @@ use crate::adapters::{ pub(super) struct PinnedMigrationDirectory { name: &'static str, - identity: DirectoryIdentity, + identity: EntryIdentity, directory: Dir, } @@ -25,7 +26,7 @@ impl PinnedMigrationDirectory { }; let directory = sync_capable_directory::open(parent, name)?; require_same_filesystem(parent, &directory)?; - let identity = DirectoryIdentity::from(&directory.dir_metadata()?); + let identity = EntryIdentity::from(&directory.dir_metadata()?); let pinned = Self { name, identity, @@ -39,15 +40,15 @@ impl PinnedMigrationDirectory { } pub(super) fn verify(&self, parent: &Dir) -> io::Result<()> { - let handle = DirectoryIdentity::from(&self.directory.dir_metadata()?); + let handle = EntryIdentity::from(&self.directory.dir_metadata()?); let current = sync_capable_directory::open(parent, self.name)?; require_same_filesystem(parent, ¤t)?; - let current = DirectoryIdentity::from(¤t.dir_metadata()?); + let current = EntryIdentity::from(¤t.dir_metadata()?); let metadata = parent.symlink_metadata(self.name)?; if metadata.is_dir() && handle == self.identity && current == self.identity - && DirectoryIdentity::from(&metadata) == handle + && EntryIdentity::from(&metadata) == handle { Ok(()) } else { @@ -60,21 +61,6 @@ impl PinnedMigrationDirectory { } } -#[derive(Clone, Copy, Debug, Eq, PartialEq)] -struct DirectoryIdentity { - device: u64, - inode: u64, -} - -impl From<&Metadata> for DirectoryIdentity { - fn from(metadata: &Metadata) -> Self { - Self { - device: metadata.dev(), - inode: metadata.ino(), - } - } -} - pub(super) fn optional_directory( parent: &Dir, name: &'static str, @@ -87,7 +73,7 @@ pub(super) fn optional_directory( require_same_filesystem(parent, &directory)?; let pinned = PinnedMigrationDirectory { name, - identity: DirectoryIdentity::from(&metadata), + identity: EntryIdentity::from(&metadata), directory, }; pinned.verify(parent)?; From aaf7ccc3dd4db9b9ccd5bb79060f885d5d69afb7 Mon Sep 17 00:00:00 2001 From: James Ross Date: Mon, 7 Sep 2026 16:27:44 -0700 Subject: [PATCH 098/111] Refactor: render pool-name digests through one shared DigestHex The version-1 physical pool names and the version-2 retention pool and namespace names each carried an identical private DigestHex renderer for the 64-lowercase-hex digest component that namespace and pool admission later parse. Two copies of the emitter for one on-disk grammar can drift. adapters::digest_hex now owns the renderer and both pool-name modules consume it. Filename output is unchanged; the golden worldline, conformance, and retention namespace laws pin the grammar. Self-review finding D2 (P3). Refs #78 --- CHANGELOG.md | 3 +++ src/adapters/digest_hex.rs | 19 +++++++++++++++++++ src/adapters/mod.rs | 1 + src/adapters/physical_pool_name.rs | 14 +------------- .../filesystem_retention_pool_name.rs | 14 +------------- 5 files changed, 25 insertions(+), 26 deletions(-) create mode 100644 src/adapters/digest_hex.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index 928909c..82ef06c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -242,6 +242,9 @@ after its public API and format compatibility policies are established. ### Changed +- Version-1 and version-2 pool and namespace filenames render their digest + component through one shared lowercase-hexadecimal renderer instead of two + identical private copies. - Stage publishers and fixed-record readers share one exact-record module for no-follow non-blocking opens, exact-length reads, trailing-byte refusal, device-and-inode reverification, absence checks, and no-replacement links; diff --git a/src/adapters/digest_hex.rs b/src/adapters/digest_hex.rs new file mode 100644 index 0000000..f9b21a1 --- /dev/null +++ b/src/adapters/digest_hex.rs @@ -0,0 +1,19 @@ +//! This module owns lowercase hexadecimal rendering of 32-byte digests for pool names. + +use std::fmt; + +/// Renders one 32-byte digest as 64 lowercase hexadecimal characters. +/// +/// Every immutable-pool and namespace filename in versions 1 and 2 derives its +/// digest component from this one renderer, so the on-disk grammar the +/// admission checks expect is emitted from a single place. +pub(super) struct DigestHex<'digest>(pub(super) &'digest [u8; 32]); + +impl fmt::Display for DigestHex<'_> { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + for byte in self.0 { + write!(formatter, "{byte:02x}")?; + } + Ok(()) + } +} diff --git a/src/adapters/mod.rs b/src/adapters/mod.rs index 840b9bf..3962760 100644 --- a/src/adapters/mod.rs +++ b/src/adapters/mod.rs @@ -65,6 +65,7 @@ mod checksummed_publication_head; mod checksummed_segment_record; mod closed_segment; mod decoded_catalog_entry; +mod digest_hex; mod exports; mod filesystem_catalog_artifact; mod filesystem_catalog_catalog; diff --git a/src/adapters/physical_pool_name.rs b/src/adapters/physical_pool_name.rs index 8828c2f..72bab7c 100644 --- a/src/adapters/physical_pool_name.rs +++ b/src/adapters/physical_pool_name.rs @@ -1,8 +1,7 @@ //! Exact immutable-pool filename emission. -use std::fmt; - use super::SegmentDigest; +use super::digest_hex::DigestHex; use crate::{CatalogDigest, CatalogGeneration}; pub(super) fn segment(digest: SegmentDigest) -> String { @@ -16,14 +15,3 @@ pub(super) fn catalog(generation: CatalogGeneration, digest: CatalogDigest) -> S DigestHex(digest.as_bytes()) ) } - -struct DigestHex<'a>(&'a [u8; 32]); - -impl fmt::Display for DigestHex<'_> { - fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { - for byte in self.0 { - write!(formatter, "{byte:02x}")?; - } - Ok(()) - } -} diff --git a/src/adapters/retention/filesystem_retention_pool_name.rs b/src/adapters/retention/filesystem_retention_pool_name.rs index f50805a..a058096 100644 --- a/src/adapters/retention/filesystem_retention_pool_name.rs +++ b/src/adapters/retention/filesystem_retention_pool_name.rs @@ -1,7 +1,6 @@ //! Exact retention immutable-pool and namespace filename emission. -use std::fmt; - +use crate::adapters::digest_hex::DigestHex; use crate::{ LivenessGeneration, RetentionManifestDigest, RetentionNamespaceDigest, RetentionRootDigest, RootGeneration, @@ -34,14 +33,3 @@ pub(super) fn manifest(generation: LivenessGeneration, digest: RetentionManifest DigestHex(digest.as_bytes()) ) } - -struct DigestHex<'a>(&'a [u8; 32]); - -impl fmt::Display for DigestHex<'_> { - fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { - for byte in self.0 { - write!(formatter, "{byte:02x}")?; - } - Ok(()) - } -} From fb538e8b32f51ac71bcbe56a36f93fd55f078c32 Mon Sep 17 00:00:00 2001 From: James Ross Date: Mon, 7 Sep 2026 16:46:55 -0700 Subject: [PATCH 099/111] Fix: bind the root-link and namespace-sync phases to the admitted namespace link_root and synchronize_root_namespace receive the candidate root from the orchestrator but used whatever namespace directory the attempt held, so a root from another namespace handed to either phase would have been linked into, or synchronized as, the admitted directory. The attempt now records the admitted namespace's name beside its capability, and both phases require the root they receive to name exactly that namespace, refusing with the new RetentionCurrentStateRefusal::AttemptNamespaceDisagreed otherwise. A law drives synchronize_root_namespace with a root from a second namespace after admission; it failed before this change and passes now. The Display impl is split into a short fmt over a const message table so it stays under clippy's line ceiling; every message is unchanged. Self-review finding R13 (P4). Refs #78 --- CHANGELOG.md | 4 + .../retention/filesystem_retention_attempt.rs | 22 ++- .../filesystem_retention_attempt_tests.rs | 40 +++++- .../retention/filesystem_retention_refusal.rs | 136 +++++++++--------- .../retention/filesystem_retention_storage.rs | 18 ++- 5 files changed, 136 insertions(+), 84 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 82ef06c..bd3cce6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -590,6 +590,10 @@ after its public API and format compatibility policies are established. Review corrections to the unreleased retention and migration work above; none of these shipped in a release. +- The root-link and namespace-synchronization phases require the root handed + to them to name the namespace the attempt admitted, refusing with + `RetentionCurrentStateRefusal::AttemptNamespaceDisagreed` instead of + synchronizing whatever directory the attempt holds. - `FilesystemRecoveryStageError::LengthChanged` reports the stage's actual on-disk length in `observed` when trailing bytes are found, instead of the expected length plus the one byte that detected them. diff --git a/src/adapters/retention/filesystem_retention_attempt.rs b/src/adapters/retention/filesystem_retention_attempt.rs index 1c3449e..1edd7f6 100644 --- a/src/adapters/retention/filesystem_retention_attempt.rs +++ b/src/adapters/retention/filesystem_retention_attempt.rs @@ -4,9 +4,9 @@ use std::io; use cap_std::fs::Dir; -use super::CanonicalRetentionManifest; use super::filesystem_retention_pool_name as pool_name; use super::filesystem_retention_stage::{FilesystemRetentionStage, invalid_data}; +use super::{CanonicalRetentionManifest, RetentionCurrentStateRefusal}; use crate::{LivenessGeneration, RetentionGenerationExpectation}; /// Everything one publication attempt retains between storage-port phases. @@ -19,7 +19,7 @@ use crate::{LivenessGeneration, RetentionGenerationExpectation}; pub(super) struct PublicationAttempt { expected: RetentionGenerationExpectation, liveness_generation: LivenessGeneration, - namespace: Option, + namespace: Option<(String, Dir)>, retained_root: Option, retained_manifest: Option, root_stage: Option, @@ -69,16 +69,30 @@ impl PublicationAttempt { pool_name::manifest(self.liveness_generation, manifest.digest()) } - pub(super) fn retain_namespace(&mut self, namespace: Dir) { - self.namespace = Some(namespace); + pub(super) fn retain_namespace(&mut self, name: String, namespace: Dir) { + self.namespace = Some((name, namespace)); } pub(super) fn namespace(&self) -> io::Result<&Dir> { self.namespace .as_ref() + .map(|(_name, namespace)| namespace) .ok_or_else(|| invalid_data("retention root namespace was not admitted")) } + /// Returns the admitted namespace only if `name` is the namespace it admitted. + pub(super) fn require_namespace(&self, name: &str) -> io::Result<&Dir> { + let (admitted, namespace) = self + .namespace + .as_ref() + .ok_or_else(|| invalid_data("retention root namespace was not admitted"))?; + if admitted == name { + Ok(namespace) + } else { + Err(RetentionCurrentStateRefusal::AttemptNamespaceDisagreed.into_io()) + } + } + pub(super) fn retain_root_name(&mut self, name: String) { self.retained_root = Some(name); } diff --git a/src/adapters/retention/filesystem_retention_attempt_tests.rs b/src/adapters/retention/filesystem_retention_attempt_tests.rs index b69ef55..ff19694 100644 --- a/src/adapters/retention/filesystem_retention_attempt_tests.rs +++ b/src/adapters/retention/filesystem_retention_attempt_tests.rs @@ -5,11 +5,12 @@ use std::fs; use std::io; use super::filesystem_retention_test_fixture::{ - ROOT_HEX, fixture, initial_preparation, open_authority, refusal, retention_witness, - root_pool_path, + ROOT_HEX, fixture, initial_preparation, initial_root, open_authority, refusal, + retention_witness, root_pool_path, }; use super::{ - RetentionCurrentStateRefusal, RetentionPublicationStorage, RetentionTransitionDisposition, + AdmittedRetentionRoot, RetentionCurrentStateRefusal, RetentionNamespaceAdmission, + RetentionPublicationStorage, RetentionTransitionDisposition, }; #[test] @@ -107,3 +108,36 @@ fn namespace_admission_refuses_a_directory_the_expectation_excludes() -> Result< sandbox.remove()?; Ok(()) } + +#[test] +fn namespace_phases_refuse_a_root_outside_the_admitted_namespace() -> Result<(), Box> { + let (sandbox, mut authority) = open_authority("filesystem-retention-attempt-other-root")?; + let root_bytes = fixture(ROOT_HEX)?; + let preparation = initial_preparation(&root_bytes)?; + assert_eq!( + authority.verify_current(&preparation)?, + RetentionTransitionDisposition::Publish + ); + authority.write_root_stage(preparation.candidate())?; + authority.synchronize_root_stage()?; + assert_eq!( + authority.admit_root_namespace(preparation.candidate())?, + RetentionNamespaceAdmission::Created + ); + let template = AdmittedRetentionRoot::decode(&root_bytes)?; + let other = initial_root(b"a-namespace-the-attempt-did-not-admit", &template)?; + let other = AdmittedRetentionRoot::decode(other.encoded())?; + + let error = authority + .synchronize_root_namespace(&other) + .err() + .ok_or("a root outside the admitted namespace was synchronized")?; + + assert!(matches!( + refusal(&error), + Some(RetentionCurrentStateRefusal::AttemptNamespaceDisagreed) + )); + drop(authority); + sandbox.remove()?; + Ok(()) +} diff --git a/src/adapters/retention/filesystem_retention_refusal.rs b/src/adapters/retention/filesystem_retention_refusal.rs index 21d59f8..ee288f8 100644 --- a/src/adapters/retention/filesystem_retention_refusal.rs +++ b/src/adapters/retention/filesystem_retention_refusal.rs @@ -103,6 +103,9 @@ pub enum RetentionCurrentStateRefusal { /// The candidate's namespace directory disagreed with the claimed /// expectation, at verification or when it was admitted between phases. NamespaceExpectationViolated, + /// A later phase received a root whose namespace is not the one the + /// attempt admitted. + AttemptNamespaceDisagreed, /// A record's kind or length disagreed with its declaration. RecordKindOrLength, /// A record carried bytes beyond its declared length. @@ -121,35 +124,6 @@ impl RetentionCurrentStateRefusal { impl fmt::Display for RetentionCurrentStateRefusal { fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { match self { - Self::RetainedStage => { - formatter.write_str("retained retention stage requires recovery before publication") - } - Self::HeadAbsentWithArtifacts => formatter.write_str( - "retention head is absent while retention pools hold artifacts; recovery is \ - required", - ), - Self::ExpectedCurrentOverAbsentHead => formatter - .write_str("expected a current retention generation but no head is published"), - Self::NonInitialOverAbsentHead => formatter.write_str( - "absent retention head admits only an initial publication with no predecessor", - ), - Self::HeadRefused { .. } => { - formatter.write_str("current retention head refused admission") - } - Self::PreparedHeadRefused { .. } => { - formatter.write_str("prepared retention head refused admission") - } - Self::ManifestAbsent => { - formatter.write_str("current retention head names an absent manifest") - } - Self::ManifestRefused { .. } => { - formatter.write_str("current retention manifest refused admission") - } - Self::ManifestDisagreed => { - formatter.write_str("current retention manifest disagreed with its head") - } - Self::HeadPredecessorDisagreed => formatter - .write_str("current retention head and its manifest name different predecessors"), Self::CatalogDisagreed { expected_generation, .. @@ -159,14 +133,6 @@ impl fmt::Display for RetentionCurrentStateRefusal { current catalog", expected_generation.get() ), - Self::CatalogHeadRefused { .. } => { - formatter.write_str("this store's catalog head refused admission") - } - Self::LivenessExhausted => { - formatter.write_str("current liveness generation cannot advance") - } - Self::StaleCommittedRetry => formatter - .write_str("already-committed retry is stale: another successor is current"), Self::Superseded { current_generation, .. } => write!( @@ -174,51 +140,79 @@ impl fmt::Display for RetentionCurrentStateRefusal { "candidate is superseded: the current head is liveness generation {}", current_generation.get() ), - Self::CommittedSelectionMissing => { - formatter.write_str("committed manifest does not select the candidate namespace") + Self::NoncanonicalPoolEntry { pool } => { + write!(formatter, "retention {pool} carries a noncanonical entry") } - Self::CommittedSelectionMismatch => formatter.write_str( - "committed manifest selects a different root for the candidate namespace", - ), - Self::CommittedNamespaceUnavailable => { - formatter.write_str("committed root namespace directory is unavailable") + _ => formatter.write_str(self.message()), + } + } +} + +impl RetentionCurrentStateRefusal { + /// The fixed description of every variant that renders no field. + const fn message(&self) -> &'static str { + match self { + Self::RetainedStage => "retained retention stage requires recovery before publication", + Self::HeadAbsentWithArtifacts => { + "retention head is absent while retention pools hold artifacts; recovery is \ + required" } - Self::CommittedRootAbsent => formatter.write_str("committed root pool entry is absent"), - Self::CommittedRootChanged => { - formatter.write_str("committed root pool entry bytes disagreed") + Self::ExpectedCurrentOverAbsentHead => { + "expected a current retention generation but no head is published" } - Self::PredecessorMismatch => { - formatter.write_str("candidate does not name the current root as its predecessor") + Self::NonInitialOverAbsentHead => { + "absent retention head admits only an initial publication with no predecessor" } - Self::PredecessorRootAbsent => formatter - .write_str("predecessor root pool entry is absent or exceeds the format bound"), - Self::PredecessorRootChanged => formatter.write_str( - "predecessor root pool entry does not decode to the manifest's selection", - ), - Self::UnknownRetentionEntry => { - formatter.write_str("retention namespace carries an unknown entry") + Self::HeadRefused { .. } => "current retention head refused admission", + Self::PreparedHeadRefused { .. } => "prepared retention head refused admission", + Self::ManifestAbsent => "current retention head names an absent manifest", + Self::ManifestRefused { .. } => "current retention manifest refused admission", + Self::ManifestDisagreed => "current retention manifest disagreed with its head", + Self::HeadPredecessorDisagreed => { + "current retention head and its manifest name different predecessors" } - Self::NonNamespaceEntry => { - formatter.write_str("retention roots carries a non-namespace entry") + Self::CatalogHeadRefused { .. } => "this store's catalog head refused admission", + Self::LivenessExhausted => "current liveness generation cannot advance", + Self::StaleCommittedRetry => { + "already-committed retry is stale: another successor is current" } - Self::NoncanonicalPoolEntry { pool } => { - write!(formatter, "retention {pool} carries a noncanonical entry") + Self::CommittedSelectionMissing => { + "committed manifest does not select the candidate namespace" } - Self::NamespaceCapacity => { - formatter.write_str("retention namespace or pool count would exceed its ceiling") + Self::CommittedSelectionMismatch => { + "committed manifest selects a different root for the candidate namespace" } - Self::NamespaceExpectationViolated => formatter.write_str( - "namespace directory state disagreed with the claimed generation expectation", - ), - Self::RecordKindOrLength => { - formatter.write_str("retention record kind or length disagreed") + Self::CommittedNamespaceUnavailable => { + "committed root namespace directory is unavailable" + } + Self::CommittedRootAbsent => "committed root pool entry is absent", + Self::CommittedRootChanged => "committed root pool entry bytes disagreed", + Self::PredecessorMismatch => { + "candidate does not name the current root as its predecessor" + } + Self::PredecessorRootAbsent => { + "predecessor root pool entry is absent or exceeds the format bound" + } + Self::PredecessorRootChanged => { + "predecessor root pool entry does not decode to the manifest's selection" + } + Self::UnknownRetentionEntry => "retention namespace carries an unknown entry", + Self::NonNamespaceEntry => "retention roots carries a non-namespace entry", + Self::NamespaceCapacity => "retention namespace or pool count would exceed its ceiling", + Self::NamespaceExpectationViolated => { + "namespace directory state disagreed with the claimed generation expectation" } - Self::RecordTrailingBytes => { - formatter.write_str("retention record carried trailing bytes") + Self::RecordKindOrLength => "retention record kind or length disagreed", + Self::RecordTrailingBytes => "retention record carried trailing bytes", + Self::RecordLengthOverflow => "retention record length exceeded the addressable range", + Self::AttemptNamespaceDisagreed => { + "root handed to a publication phase names a different namespace than admitted" } - Self::RecordLengthOverflow => { - formatter.write_str("retention record length exceeded the addressable range") + Self::CatalogDisagreed { .. } => { + "closure was verified against a catalog that is not this store's current catalog" } + Self::Superseded { .. } => "candidate is superseded by the current head", + Self::NoncanonicalPoolEntry { .. } => "retention pool carries a noncanonical entry", } } } diff --git a/src/adapters/retention/filesystem_retention_storage.rs b/src/adapters/retention/filesystem_retention_storage.rs index 1af1d17..303c2f2 100644 --- a/src/adapters/retention/filesystem_retention_storage.rs +++ b/src/adapters/retention/filesystem_retention_storage.rs @@ -115,7 +115,7 @@ impl RetentionPublicationStorage for FilesystemRetentionPublicationAuthority { } Err(source) => return Err(source), }; - attempt.retain_namespace(namespace); + attempt.retain_namespace(name, namespace); Ok(admission) } @@ -125,16 +125,22 @@ impl RetentionPublicationStorage for FilesystemRetentionPublicationAuthority { fn link_root(&mut self, root: &AdmittedRetentionRoot<'_>) -> io::Result<()> { let attempt = attempt::require_mut(&mut self.attempt)?; + let namespace = pool_name::namespace(root.root().namespace().digest()); let name = pool_name::root(root.root().generation(), root.digest()); - attempt - .root_stage()? - .link(&self.retention, attempt.namespace()?, &name)?; + attempt.root_stage()?.link( + &self.retention, + attempt.require_namespace(&namespace)?, + &name, + )?; attempt.retain_root_name(name); Ok(()) } - fn synchronize_root_namespace(&mut self, _root: &AdmittedRetentionRoot<'_>) -> io::Result<()> { - synchronize_directory(attempt::require(self.attempt.as_ref())?.namespace()?) + fn synchronize_root_namespace(&mut self, root: &AdmittedRetentionRoot<'_>) -> io::Result<()> { + let namespace = pool_name::namespace(root.root().namespace().digest()); + synchronize_directory( + attempt::require(self.attempt.as_ref())?.require_namespace(&namespace)?, + ) } fn write_manifest_stage(&mut self, manifest: &CanonicalRetentionManifest) -> io::Result<()> { From 6ce8393ab8b2e779001c9effa843aa610176c12c Mon Sep 17 00:00:00 2001 From: James Ross Date: Mon, 7 Sep 2026 17:15:43 -0700 Subject: [PATCH 100/111] Fix: name the already-committed retry over an absent head for what it is When no retention head is published and the preparation carries no forward publication (the byte-identical already-committed shape), the initial-path check refused as StaleCommittedRetry, a name that means "another successor is current". Nothing is current in that state; the retry claims a commit that cannot have happened. The refusal is now CommittedRetryOverAbsentHead, with its own description. The successor-over-absent-head law now downcasts to the refusal it actually receives, ExpectedCurrentOverAbsentHead, instead of asserting only the error kind. Of the grouped findings, R14 (the authority doc listing the pinned root), R16 (the LivenessGeneration import), and R22 (the stale identity-probe comment) were resolved by the PublicationAttempt and identity-policy commits earlier in this pass; this commit closes the group. Self-review findings R14, R16, R22, R23 (P4). Refs #78 --- CHANGELOG.md | 4 ++++ src/adapters/retention/filesystem_retention_current.rs | 2 +- src/adapters/retention/filesystem_retention_refusal.rs | 6 ++++++ .../retention/filesystem_retention_successor_tests.rs | 4 ++++ 4 files changed, 15 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index bd3cce6..75d82ed 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -590,6 +590,10 @@ after its public API and format compatibility policies are established. Review corrections to the unreleased retention and migration work above; none of these shipped in a release. +- An already-committed retry presented over an absent retention head refuses + as `CommittedRetryOverAbsentHead` instead of the misnamed + `StaleCommittedRetry`; the successor-over-absent-head law now downcasts to + `ExpectedCurrentOverAbsentHead`. - The root-link and namespace-synchronization phases require the root handed to them to name the namespace the attempt admitted, refusing with `RetentionCurrentStateRefusal::AttemptNamespaceDisagreed` instead of diff --git a/src/adapters/retention/filesystem_retention_current.rs b/src/adapters/retention/filesystem_retention_current.rs index 6c4ecd9..fd22ae2 100644 --- a/src/adapters/retention/filesystem_retention_current.rs +++ b/src/adapters/retention/filesystem_retention_current.rs @@ -256,7 +256,7 @@ fn require_initial_publication( ) -> io::Result<()> { let publication = preparation .publication() - .ok_or_else(|| RetentionCurrentStateRefusal::StaleCommittedRetry.into_io())?; + .ok_or_else(|| RetentionCurrentStateRefusal::CommittedRetryOverAbsentHead.into_io())?; let prepared = ChecksummedRetentionHead::decode(publication.head().encoded()) .map_err(|source| RetentionCurrentStateRefusal::PreparedHeadRefused { source }.into_io())?; if prepared.head().generation() == crate::LivenessGeneration::INITIAL diff --git a/src/adapters/retention/filesystem_retention_refusal.rs b/src/adapters/retention/filesystem_retention_refusal.rs index ee288f8..6f83875 100644 --- a/src/adapters/retention/filesystem_retention_refusal.rs +++ b/src/adapters/retention/filesystem_retention_refusal.rs @@ -106,6 +106,9 @@ pub enum RetentionCurrentStateRefusal { /// A later phase received a root whose namespace is not the one the /// attempt admitted. AttemptNamespaceDisagreed, + /// A byte-identical already-committed retry was presented while no + /// retention head is published, so nothing can have committed it. + CommittedRetryOverAbsentHead, /// A record's kind or length disagreed with its declaration. RecordKindOrLength, /// A record carried bytes beyond its declared length. @@ -205,6 +208,9 @@ impl RetentionCurrentStateRefusal { Self::RecordKindOrLength => "retention record kind or length disagreed", Self::RecordTrailingBytes => "retention record carried trailing bytes", Self::RecordLengthOverflow => "retention record length exceeded the addressable range", + Self::CommittedRetryOverAbsentHead => { + "already-committed retry presented while no retention head is published" + } Self::AttemptNamespaceDisagreed => { "root handed to a publication phase names a different namespace than admitted" } diff --git a/src/adapters/retention/filesystem_retention_successor_tests.rs b/src/adapters/retention/filesystem_retention_successor_tests.rs index a716fc2..2d283d1 100644 --- a/src/adapters/retention/filesystem_retention_successor_tests.rs +++ b/src/adapters/retention/filesystem_retention_successor_tests.rs @@ -117,6 +117,10 @@ fn expected_current_generation_refuses_when_no_head_is_published() -> Result<(), .ok_or("successor over an absent head was unexpectedly admitted")?; assert_eq!(error.kind(), io::ErrorKind::InvalidData); + assert!(matches!( + refusal(&error), + Some(RetentionCurrentStateRefusal::ExpectedCurrentOverAbsentHead) + )); assert!(authority.observe_current()?.is_none()); assert!(!head_path(sandbox.path()).exists()); drop(authority); From a6b612040e37b4a84998956a80de8f1ae5f79f6f Mon Sep 17 00:00:00 2001 From: James Ross Date: Mon, 7 Sep 2026 17:42:14 -0700 Subject: [PATCH 101/111] Refactor: remove the test sandbox on drop and fold the private copies onto it Every filesystem law ended with a manual drop(authority); sandbox.remove()? pair, and a law that returned early on a failed assertion left its sandbox behind in the scratch root. Two test modules also carried their own private TestDirectory with a different scratch location. The shared sandbox now removes itself on drop; remove(self) stays for laws that want to observe the removal error. A law pins the behaviour (a dropped sandbox no longer exists); it failed before this change and passes now. The recovery-stage materialization and writer-lock laws use the shared sandbox, and the 43 manual teardown lines in the retention laws are gone; locals drop in reverse declaration order, so the authority releases its handles before the sandbox removes the directory. Self-review finding R17 (P4). Refs #78 --- src/adapters/filesystem_exact_record_tests.rs | 13 +++++++++ ...lesystem_recovery_stage_materialization.rs | 28 ++----------------- src/adapters/filesystem_writer_lock_tests.rs | 27 +----------------- .../filesystem_recovery_admission_tests.rs | 2 -- .../filesystem_retention_attempt_tests.rs | 10 +------ .../filesystem_retention_capacity_tests.rs | 4 --- .../filesystem_retention_catalog_tests.rs | 6 ---- .../filesystem_retention_current_tests.rs | 8 ------ .../filesystem_retention_expectation_tests.rs | 10 ------- .../filesystem_retention_fifo_tests.rs | 3 -- .../filesystem_retention_namespace_tests.rs | 8 ------ .../filesystem_retention_storage_tests.rs | 12 -------- .../filesystem_retention_successor_tests.rs | 6 ---- .../filesystem_version_two_admission_tests.rs | 7 ----- tests/segment_filesystem_stage/sandbox.rs | 13 ++++++++- 15 files changed, 29 insertions(+), 128 deletions(-) diff --git a/src/adapters/filesystem_exact_record_tests.rs b/src/adapters/filesystem_exact_record_tests.rs index 0de8ace..28c4e87 100644 --- a/src/adapters/filesystem_exact_record_tests.rs +++ b/src/adapters/filesystem_exact_record_tests.rs @@ -131,3 +131,16 @@ fn require_absent_refuses_a_visible_entry_and_links_never_replace() -> Result<() sandbox.remove()?; Ok(()) } + +#[test] +fn a_dropped_sandbox_no_longer_exists() -> Result<(), Box> { + let sandbox = TestDirectory::create("exact-record-dropped-sandbox")?; + let path = sandbox.path().to_path_buf(); + fs::write(path.join("evidence"), b"left behind by an early return")?; + assert!(path.is_dir()); + + drop(sandbox); + + assert!(!path.exists(), "a dropped sandbox must remove itself"); + Ok(()) +} diff --git a/src/adapters/filesystem_recovery_stage_materialization.rs b/src/adapters/filesystem_recovery_stage_materialization.rs index 9705158..3c25e7b 100644 --- a/src/adapters/filesystem_recovery_stage_materialization.rs +++ b/src/adapters/filesystem_recovery_stage_materialization.rs @@ -150,16 +150,14 @@ mod tests { use std::error::Error; use std::fs; use std::io; - use std::path::{Path, PathBuf}; - use std::sync::atomic::{AtomicU64, Ordering}; + use std::path::Path; use cap_fs_ext::{FollowSymlinks, OpenOptionsFollowExt, OpenOptionsSyncExt}; use cap_std::fs::OpenOptions; use cap_std::{ambient_authority, fs::Dir}; use super::*; - - static NEXT_DIRECTORY: AtomicU64 = AtomicU64::new(0); + use crate::adapters::filesystem_test_sandbox::TestDirectory; #[test] fn read_exact_reads_expected_bytes_without_trailing() -> Result<(), Box> { @@ -260,26 +258,4 @@ mod tests { let file = directory.open_with(file_name, &options)?; Ok(file) } - - struct TestDirectory { - path: PathBuf, - } - - impl TestDirectory { - fn create(name: &str) -> std::io::Result { - let sequence = NEXT_DIRECTORY.fetch_add(1, Ordering::Relaxed); - let path = - std::env::temp_dir().join(format!("keep-{name}-{}-{sequence}", std::process::id())); - fs::create_dir(&path)?; - Ok(Self { path }) - } - - fn path(&self) -> &std::path::Path { - &self.path - } - - fn remove(self) -> std::io::Result<()> { - fs::remove_dir_all(self.path) - } - } } diff --git a/src/adapters/filesystem_writer_lock_tests.rs b/src/adapters/filesystem_writer_lock_tests.rs index 55e903b..0343ca8 100644 --- a/src/adapters/filesystem_writer_lock_tests.rs +++ b/src/adapters/filesystem_writer_lock_tests.rs @@ -2,17 +2,14 @@ use std::error::Error; use std::fs; -use std::path::{Path, PathBuf}; -use std::sync::atomic::{AtomicU64, Ordering}; use cap_std::ambient_authority; use cap_std::fs::Dir; use super::{FileIdentity, LOCK_FILE_NAME, open_existing, verify_current_identity}; +use crate::adapters::filesystem_test_sandbox::TestDirectory; use crate::adapters::{WriterLockAcquireError, WriterLockAcquirePhase}; -static NEXT_DIRECTORY: AtomicU64 = AtomicU64::new(0); - #[test] fn replaced_lock_entry_cannot_authorize_the_opened_handle() -> Result<(), Box> { let sandbox = TestDirectory::create("writer-lock-identity")?; @@ -41,25 +38,3 @@ fn replaced_lock_entry_cannot_authorize_the_opened_handle() -> Result<(), Box std::io::Result { - let sequence = NEXT_DIRECTORY.fetch_add(1, Ordering::Relaxed); - let path = - std::env::temp_dir().join(format!("keep-{name}-{}-{sequence}", std::process::id())); - fs::create_dir(&path)?; - Ok(Self { path }) - } - - fn path(&self) -> &Path { - &self.path - } - - fn remove(self) -> std::io::Result<()> { - fs::remove_dir_all(self.path) - } -} diff --git a/src/adapters/retention/filesystem_recovery_admission_tests.rs b/src/adapters/retention/filesystem_recovery_admission_tests.rs index e433a55..d366608 100644 --- a/src/adapters/retention/filesystem_recovery_admission_tests.rs +++ b/src/adapters/retention/filesystem_recovery_admission_tests.rs @@ -25,7 +25,6 @@ fn recovery_inventory_reader_refuses_a_migrated_root() -> Result<(), Box Result<(), Box Result<(), Box> { assert_eq!(error.kind(), io::ErrorKind::InvalidData); assert_eq!(retention_witness(sandbox.path())?, before); - drop(authority); - sandbox.remove()?; Ok(()) } @@ -73,8 +71,6 @@ fn stale_stage_handle_does_not_survive_a_refused_verification() -> Result<(), Bo sandbox.path().join("retention").join("root.next").is_file(), "retained stage evidence must remain for recovery" ); - drop(authority); - sandbox.remove()?; Ok(()) } @@ -104,14 +100,12 @@ fn namespace_admission_refuses_a_directory_the_expectation_excludes() -> Result< refusal(&error), Some(RetentionCurrentStateRefusal::NamespaceExpectationViolated) )); - drop(authority); - sandbox.remove()?; Ok(()) } #[test] fn namespace_phases_refuse_a_root_outside_the_admitted_namespace() -> Result<(), Box> { - let (sandbox, mut authority) = open_authority("filesystem-retention-attempt-other-root")?; + let (_sandbox, mut authority) = open_authority("filesystem-retention-attempt-other-root")?; let root_bytes = fixture(ROOT_HEX)?; let preparation = initial_preparation(&root_bytes)?; assert_eq!( @@ -137,7 +131,5 @@ fn namespace_phases_refuse_a_root_outside_the_admitted_namespace() -> Result<(), refusal(&error), Some(RetentionCurrentStateRefusal::AttemptNamespaceDisagreed) )); - drop(authority); - sandbox.remove()?; Ok(()) } diff --git a/src/adapters/retention/filesystem_retention_capacity_tests.rs b/src/adapters/retention/filesystem_retention_capacity_tests.rs index 4fb0d68..7441948 100644 --- a/src/adapters/retention/filesystem_retention_capacity_tests.rs +++ b/src/adapters/retention/filesystem_retention_capacity_tests.rs @@ -46,8 +46,6 @@ fn a_full_namespace_pool_refuses_a_new_namespace_before_staging() -> Result<(), Some(RetentionCurrentStateRefusal::NamespaceCapacity) )); assert_eq!(retention_witness(sandbox.path())?, before); - drop(authority); - sandbox.remove()?; Ok(()) } @@ -74,8 +72,6 @@ fn a_full_namespace_pool_admits_a_successor_in_an_existing_namespace() -> Result let disposition = RetentionPublicationStorage::verify_current(&mut authority, &preparation)?; assert_eq!(disposition, RetentionTransitionDisposition::Publish); - drop(authority); - sandbox.remove()?; Ok(()) } diff --git a/src/adapters/retention/filesystem_retention_catalog_tests.rs b/src/adapters/retention/filesystem_retention_catalog_tests.rs index e55bd68..e030f1c 100644 --- a/src/adapters/retention/filesystem_retention_catalog_tests.rs +++ b/src/adapters/retention/filesystem_retention_catalog_tests.rs @@ -35,8 +35,6 @@ fn closure_verified_against_another_catalog_refuses_before_staging() -> Result<( Some(RetentionCurrentStateRefusal::CatalogDisagreed { .. }) )); assert_eq!(retention_witness(sandbox.path())?, before); - drop(authority); - sandbox.remove()?; Ok(()) } @@ -64,8 +62,6 @@ fn committed_retry_over_a_foreign_catalog_refuses_before_reporting_committed() Some(RetentionCurrentStateRefusal::CatalogDisagreed { .. }) )); assert_eq!(retention_witness(sandbox.path())?, before); - drop(authority); - sandbox.remove()?; Ok(()) } @@ -99,7 +95,5 @@ fn a_corrupt_catalog_head_refuses_with_its_decode_error() -> Result<(), Box Result<(), Box< Some(RetentionCurrentStateRefusal::CommittedRootAbsent) )); assert_eq!(retention_witness(sandbox.path())?, before); - drop(authority); - sandbox.remove()?; Ok(()) } @@ -66,8 +64,6 @@ fn committed_retry_refuses_when_the_selected_root_bytes_changed() -> Result<(), refusal(&source), Some(RetentionCurrentStateRefusal::CommittedRootChanged) )); - drop(authority); - sandbox.remove()?; Ok(()) } @@ -92,8 +88,6 @@ fn committed_retry_refuses_when_the_selected_manifest_is_corrupt() -> Result<(), error, RetentionPublicationError::CurrentVerification { .. } )); - drop(authority); - sandbox.remove()?; Ok(()) } @@ -137,7 +131,5 @@ fn head_predecessor_disagreeing_with_its_manifest_refuses() -> Result<(), Box Result<(), Box< Some(RetentionCurrentStateRefusal::HeadAbsentWithArtifacts) )); assert_eq!(retention_witness(sandbox.path())?, before); - drop(authority); - sandbox.remove()?; Ok(()) } @@ -71,8 +69,6 @@ fn absent_expectation_refuses_an_orphan_directory_for_a_new_namespace() -> Resul Some(super::RetentionCurrentStateRefusal::NamespaceExpectationViolated) )); assert_eq!(retention_witness(sandbox.path())?, before); - drop(authority); - sandbox.remove()?; Ok(()) } @@ -105,8 +101,6 @@ fn current_expectation_refuses_when_the_namespace_directory_is_absent() -> Resul super::filesystem_retention_test_fixture::refusal(&error), Some(super::RetentionCurrentStateRefusal::NamespaceExpectationViolated) )); - drop(authority); - sandbox.remove()?; Ok(()) } @@ -135,8 +129,6 @@ fn successor_refuses_when_the_predecessor_root_file_is_absent() -> Result<(), Bo Some(RetentionCurrentStateRefusal::PredecessorRootAbsent) )); assert_eq!(retention_witness(sandbox.path())?, before); - drop(authority); - sandbox.remove()?; Ok(()) } @@ -166,8 +158,6 @@ fn successor_refuses_when_the_predecessor_root_bytes_changed() -> Result<(), Box refusal(&error), Some(RetentionCurrentStateRefusal::PredecessorRootChanged) )); - drop(authority); - sandbox.remove()?; Ok(()) } diff --git a/src/adapters/retention/filesystem_retention_fifo_tests.rs b/src/adapters/retention/filesystem_retention_fifo_tests.rs index e91a160..8bbbc2a 100644 --- a/src/adapters/retention/filesystem_retention_fifo_tests.rs +++ b/src/adapters/retention/filesystem_retention_fifo_tests.rs @@ -24,7 +24,6 @@ fn a_fifo_at_the_retention_head_refuses_instead_of_blocking() -> Result<(), Box< let outcome = completes_within(move || authority.observe_current().map(|_| ()))?; assert!(outcome.is_err(), "FIFO head was unexpectedly admitted"); - sandbox.remove()?; Ok(()) } @@ -43,7 +42,6 @@ fn a_fifo_at_the_format_marker_refuses_instead_of_blocking() -> Result<(), Box Result<(), Bo outcome.is_err(), "FIFO pool target was unexpectedly admitted" ); - sandbox.remove()?; Ok(()) } diff --git a/src/adapters/retention/filesystem_retention_namespace_tests.rs b/src/adapters/retention/filesystem_retention_namespace_tests.rs index f1c2966..a800265 100644 --- a/src/adapters/retention/filesystem_retention_namespace_tests.rs +++ b/src/adapters/retention/filesystem_retention_namespace_tests.rs @@ -30,8 +30,6 @@ fn unknown_retention_entry_refuses_before_any_stage_is_written() -> Result<(), B Some(super::RetentionCurrentStateRefusal::UnknownRetentionEntry) )); assert_eq!(retention_witness(sandbox.path())?, before); - drop(authority); - sandbox.remove()?; Ok(()) } @@ -57,8 +55,6 @@ fn non_digest_root_namespace_directory_refuses() -> Result<(), Box> { super::filesystem_retention_test_fixture::refusal(&error), Some(super::RetentionCurrentStateRefusal::NonNamespaceEntry) )); - drop(authority); - sandbox.remove()?; Ok(()) } @@ -85,8 +81,6 @@ fn malformed_manifest_pool_name_refuses() -> Result<(), Box> { super::filesystem_retention_test_fixture::refusal(&error), Some(super::RetentionCurrentStateRefusal::NoncanonicalPoolEntry { .. }) )); - drop(authority); - sandbox.remove()?; Ok(()) } @@ -115,7 +109,5 @@ fn uppercase_root_pool_name_refuses() -> Result<(), Box> { super::filesystem_retention_test_fixture::refusal(&error), Some(super::RetentionCurrentStateRefusal::NoncanonicalPoolEntry { .. }) )); - drop(authority); - sandbox.remove()?; Ok(()) } diff --git a/src/adapters/retention/filesystem_retention_storage_tests.rs b/src/adapters/retention/filesystem_retention_storage_tests.rs index 5d02378..3c13257 100644 --- a/src/adapters/retention/filesystem_retention_storage_tests.rs +++ b/src/adapters/retention/filesystem_retention_storage_tests.rs @@ -34,8 +34,6 @@ fn complete_publication_preserves_migrated_bytes_and_publishes_exact_retention_p fixture(MANIFEST_HEX)? ); assert_stages_absent(sandbox.path())?; - drop(authority); - sandbox.remove()?; Ok(()) } @@ -58,8 +56,6 @@ fn existing_root_stage_is_never_truncated() -> Result<(), Box> { assert_eq!(error.kind(), io::ErrorKind::AlreadyExists); assert_eq!(fs::read(&stage)?, b"retained partial evidence"); - drop(authority); - sandbox.remove()?; Ok(()) } @@ -82,8 +78,6 @@ fn retained_stage_refuses_publication_before_recovery() -> Result<(), Box Result<(), Box Result<(), Box> { RetentionPublicationOutcome::AlreadyCommitted ); assert_eq!(retention_witness(sandbox.path())?, after_publication); - drop(authority); - sandbox.remove()?; Ok(()) } @@ -178,7 +168,5 @@ fn retained_manifest_stage_refuses_publication_before_recovery() -> Result<(), B }; assert_eq!(source.kind(), io::ErrorKind::InvalidData); assert_eq!(retention_witness(sandbox.path())?, before); - drop(authority); - sandbox.remove()?; Ok(()) } diff --git a/src/adapters/retention/filesystem_retention_successor_tests.rs b/src/adapters/retention/filesystem_retention_successor_tests.rs index 2d283d1..6fe5ffb 100644 --- a/src/adapters/retention/filesystem_retention_successor_tests.rs +++ b/src/adapters/retention/filesystem_retention_successor_tests.rs @@ -61,8 +61,6 @@ fn successor_publication_over_existing_head_publishes_exact_successor() -> Resul ); } } - drop(authority); - sandbox.remove()?; Ok(()) } @@ -96,8 +94,6 @@ fn superseded_candidate_refuses_once_a_successor_is_current() -> Result<(), Box< Some(RetentionCurrentStateRefusal::Superseded { .. }) )); assert_eq!(retention_witness(sandbox.path())?, after_successor); - drop(authority); - sandbox.remove()?; Ok(()) } @@ -123,7 +119,5 @@ fn expected_current_generation_refuses_when_no_head_is_published() -> Result<(), )); assert!(authority.observe_current()?.is_none()); assert!(!head_path(sandbox.path()).exists()); - drop(authority); - sandbox.remove()?; Ok(()) } diff --git a/src/adapters/retention/filesystem_version_two_admission_tests.rs b/src/adapters/retention/filesystem_version_two_admission_tests.rs index 91c7602..a26171c 100644 --- a/src/adapters/retention/filesystem_version_two_admission_tests.rs +++ b/src/adapters/retention/filesystem_version_two_admission_tests.rs @@ -34,7 +34,6 @@ fn version_two_reopen_refuses_a_corrupt_format_marker() -> Result<(), Box()), Some(VersionTwoRecordRefusal::Marker { .. }) )); - sandbox.remove()?; Ok(()) } @@ -54,7 +53,6 @@ fn version_two_reopen_refuses_an_oversized_format_marker() -> Result<(), Box Result<(), Box Result<(), let admission = FilesystemVersionTwoAdmission::reopen(sandbox.path())?; drop(admission); - sandbox.remove()?; Ok(()) } @@ -166,7 +160,6 @@ fn refuses_namespace( matches!(error, FilesystemPlatformAdmissionError::Namespace { .. }), "{name}: expected a Namespace refusal, got {error:?}" ); - sandbox.remove()?; Ok(()) } diff --git a/tests/segment_filesystem_stage/sandbox.rs b/tests/segment_filesystem_stage/sandbox.rs index a95cd3e..126b533 100644 --- a/tests/segment_filesystem_stage/sandbox.rs +++ b/tests/segment_filesystem_stage/sandbox.rs @@ -39,10 +39,21 @@ impl TestDirectory { /// Removes the complete sandbox after all test handles are closed. /// + /// Dropping the sandbox also removes it, so a law that returns early on a + /// failed assertion leaves no evidence behind; call this only to observe + /// the removal error itself. + /// /// # Errors /// /// Returns the exact recursive-removal filesystem failure. pub(super) fn remove(self) -> io::Result<()> { - fs::remove_dir_all(self.path) + fs::remove_dir_all(&self.path) + } +} + +impl Drop for TestDirectory { + /// Best-effort removal; an already removed sandbox is not an error. + fn drop(&mut self) { + let _ = fs::remove_dir_all(&self.path); } } From d647261a7923e8ce0d165db96c753065ec83b6a4 Mon Sep 17 00:00:00 2001 From: James Ross Date: Mon, 7 Sep 2026 18:07:50 -0700 Subject: [PATCH 102/111] Test: prove the admission type boundary with a compile_fail doctest tests/version_two_admission_contract.rs pinned the exact text of two function signatures to show that retention publication accepts only version-two authority and the version-one publishers only version-one authority. A signature contract breaks on any harmless reformatting and proves nothing the compiler does not already enforce. The doc of FilesystemRetentionPublicationAuthority::open now carries a compile_fail doctest that hands it a FilesystemPlatformAdmission; the doctest passes only while that call fails to type-check, which is the actual boundary. The source contract keeps per-file markers (each publisher names its own admission type and never the other) without pinning a signature. Self-review finding R19 (P4). Refs #78 --- CHANGELOG.md | 3 +++ .../retention/filesystem_retention_authority.rs | 9 ++++++++- tests/version_two_admission_contract.rs | 10 +++++++--- 3 files changed, 18 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 75d82ed..d668dfa 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -242,6 +242,9 @@ after its public API and format compatibility policies are established. ### Changed +- The version-two admission type boundary is proven by a `compile_fail` + doctest on `FilesystemRetentionPublicationAuthority::open`; the source + contract keeps only per-file markers instead of exact signatures. - Version-1 and version-2 pool and namespace filenames render their digest component through one shared lowercase-hexadecimal renderer instead of two identical private copies. diff --git a/src/adapters/retention/filesystem_retention_authority.rs b/src/adapters/retention/filesystem_retention_authority.rs index 26a0044..88df5d8 100644 --- a/src/adapters/retention/filesystem_retention_authority.rs +++ b/src/adapters/retention/filesystem_retention_authority.rs @@ -41,7 +41,14 @@ impl FilesystemRetentionPublicationAuthority { /// Pins one admitted version-two root for retention publication. /// /// Only [`FilesystemVersionTwoAdmission`] is accepted, so version-one - /// writer authority can never reach retention publication. + /// writer authority can never reach retention publication; the type + /// system refuses it: + /// + /// ```compile_fail + /// fn publish(admission: keep::FilesystemPlatformAdmission) { + /// let _ = keep::FilesystemRetentionPublicationAuthority::open(admission); + /// } + /// ``` /// /// This synchronous constructor opens pinned directory capabilities but /// materializes no record bodies and performs no protocol mutation. diff --git a/tests/version_two_admission_contract.rs b/tests/version_two_admission_contract.rs index 5ae4a9d..b6ad2df 100644 --- a/tests/version_two_admission_contract.rs +++ b/tests/version_two_admission_contract.rs @@ -11,17 +11,21 @@ const MIGRATION_AUTHORITY: &str = const ADMISSION_ERROR: &str = include_str!("../src/adapters/filesystem_platform_admission_error.rs"); +/// The type-level proof is the `compile_fail` doctest on +/// `FilesystemRetentionPublicationAuthority::open`, which refuses a +/// `FilesystemPlatformAdmission` argument at compile time. These markers only +/// keep each file on its side of the boundary without pinning a signature. #[test] fn retention_publication_consumes_only_version_two_authority() { - assert!(RETENTION_AUTHORITY.contains("pub fn open(admission: FilesystemVersionTwoAdmission)")); + assert!(RETENTION_AUTHORITY.contains("FilesystemVersionTwoAdmission")); assert!(!RETENTION_AUTHORITY.contains("admission: FilesystemPlatformAdmission")); } #[test] fn version_one_publishers_consume_only_version_one_authority() { - assert!(CATALOG_PUBLISHER.contains("admission: FilesystemPlatformAdmission,")); + assert!(CATALOG_PUBLISHER.contains("FilesystemPlatformAdmission")); assert!(!CATALOG_PUBLISHER.contains("FilesystemVersionTwoAdmission")); - assert!(MIGRATION_AUTHORITY.contains("admission: FilesystemPlatformAdmission,")); + assert!(MIGRATION_AUTHORITY.contains("FilesystemPlatformAdmission")); assert!(!MIGRATION_AUTHORITY.contains("FilesystemVersionTwoAdmission")); } From 400ba69f1792ec68b02f6aa5a0c3f5dd9610d018 Mon Sep 17 00:00:00 2001 From: James Ross Date: Mon, 7 Sep 2026 18:31:28 -0700 Subject: [PATCH 103/111] Refactor: keep pool-name predicates beside their emitters and bound the census The namespace census carried its own copies of the pool-name grammar (is_pool_name, is_lower_hex, the suffixes and hex widths) apart from the module that emits those names, and classified every entry through a full metadata call when the directory listing already reports the file type. The predicates and constants now live in filesystem_retention_pool_name next to root(), manifest(), and namespace(), so emission and admission cannot drift. The census reads DirEntry::file_type, and its doc states the bound: one visit per roots entry, root pool, and manifest pool, two counters, no entry bytes, no link following, work proportional to the entry count under the 4,096-namespace ceiling. Behaviour and refusals are unchanged; the namespace, capacity, and expectation laws cover the census. Self-review findings R20 and X3 (P4). Refs #78 --- CHANGELOG.md | 4 ++ .../filesystem_retention_namespace.rs | 42 +++++-------------- .../filesystem_retention_pool_name.rs | 40 +++++++++++++++++- 3 files changed, 53 insertions(+), 33 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d668dfa..8567ff1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -242,6 +242,10 @@ after its public API and format compatibility policies are established. ### Changed +- Retention pool and namespace name predicates live beside their emitters in + one module, and the namespace census classifies entries from the directory + listing's file type instead of a metadata call per entry; its doc states the + bound on the work it performs. - The version-two admission type boundary is proven by a `compile_fail` doctest on `FilesystemRetentionPublicationAuthority::open`; the source contract keeps only per-file markers instead of exact signatures. diff --git a/src/adapters/retention/filesystem_retention_namespace.rs b/src/adapters/retention/filesystem_retention_namespace.rs index 546bbee..c4ba062 100644 --- a/src/adapters/retention/filesystem_retention_namespace.rs +++ b/src/adapters/retention/filesystem_retention_namespace.rs @@ -1,6 +1,5 @@ //! This module owns exact admission of the `retention` protocol namespace. -use std::ffi::OsStr; use std::io; use cap_fs_ext::DirExt; @@ -12,10 +11,6 @@ use super::filesystem_retention_pool_name as pool_name; use crate::{RetentionGenerationExpectation, RetentionManifest}; const CANONICAL_ENTRIES: [&str; 3] = [pool_name::HEAD, pool_name::ROOTS, pool_name::MANIFESTS]; -const DIGEST_HEX: usize = 64; -const GENERATION_HEX: usize = 16; -const ROOT_SUFFIX: &str = ".root"; -const MANIFEST_SUFFIX: &str = ".manifest"; /// Bounded observation of the admitted retention namespace. #[must_use] @@ -41,6 +36,12 @@ impl RetentionNamespaceCensus { /// regular `-.root` files; every `manifests` entry must be /// a regular `-.manifest` file. Kinds are observed without /// following links. Any other entry is unrecoverable ambiguity and refuses. +/// +/// The census is bounded: it visits every `roots` entry, every root pool, and +/// the manifest pool exactly once, retains only two counters, reads no entry +/// bytes, and classifies each entry from the directory listing's own file type. +/// Its work is therefore proportional to the entry count, which the 4,096 +/// namespace ceiling and the pools' generation histories bound. pub(super) fn admit( retention: &Dir, roots: &Dir, @@ -56,16 +57,16 @@ pub(super) fn admit( for entry in roots.entries()? { let entry = entry?; let name = entry.file_name(); - if !is_lower_hex(&name, DIGEST_HEX) || !entry.metadata()?.is_dir() { + if !pool_name::is_namespace_name(&name) || !entry.file_type()?.is_dir() { return Err(Refusal::NonNamespaceEntry.into_io()); } namespace_count = namespace_count .checked_add(1) .ok_or_else(|| Refusal::NamespaceCapacity.into_io())?; let namespace = roots.open_dir_nofollow(&name)?; - let _roots = admit_pool(&namespace, ROOT_SUFFIX, "root pool")?; + let _roots = admit_pool(&namespace, pool_name::ROOT_SUFFIX, "root pool")?; } - let manifest_count = admit_pool(manifests, MANIFEST_SUFFIX, "manifest pool")?; + let manifest_count = admit_pool(manifests, pool_name::MANIFEST_SUFFIX, "manifest pool")?; Ok(RetentionNamespaceCensus { namespace_count, manifest_count, @@ -129,7 +130,7 @@ fn admit_pool(directory: &Dir, suffix: &str, pool: &'static str) -> io::Result io::Result bool { - let Some(name) = name.to_str() else { - return false; - }; - let Some(stem) = name.strip_suffix(suffix) else { - return false; - }; - let Some((generation, digest)) = stem.split_once('-') else { - return false; - }; - is_lower_hex(OsStr::new(generation), GENERATION_HEX) - && is_lower_hex(OsStr::new(digest), DIGEST_HEX) -} - -fn is_lower_hex(name: &OsStr, length: usize) -> bool { - name.to_str().is_some_and(|text| { - text.len() == length - && text - .bytes() - .all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte)) - }) -} diff --git a/src/adapters/retention/filesystem_retention_pool_name.rs b/src/adapters/retention/filesystem_retention_pool_name.rs index a058096..15b6ac4 100644 --- a/src/adapters/retention/filesystem_retention_pool_name.rs +++ b/src/adapters/retention/filesystem_retention_pool_name.rs @@ -1,4 +1,9 @@ -//! Exact retention immutable-pool and namespace filename emission. +//! Exact retention immutable-pool and namespace filename emission and admission. +//! +//! The emitters and the predicates that admit their output live together so +//! the on-disk grammar cannot drift between writing and census. + +use std::ffi::OsStr; use crate::adapters::digest_hex::DigestHex; use crate::{ @@ -13,6 +18,10 @@ pub(super) const HEAD: &str = "HEAD"; pub(super) const ROOT_STAGE: &str = "root.next"; pub(super) const MANIFEST_STAGE: &str = "manifest.next"; pub(super) const HEAD_STAGE: &str = "head.next"; +pub(super) const ROOT_SUFFIX: &str = ".root"; +pub(super) const MANIFEST_SUFFIX: &str = ".manifest"; +const DIGEST_HEX: usize = 64; +const GENERATION_HEX: usize = 16; pub(super) fn namespace(digest: RetentionNamespaceDigest) -> String { DigestHex(digest.as_bytes()).to_string() @@ -33,3 +42,32 @@ pub(super) fn manifest(generation: LivenessGeneration, digest: RetentionManifest DigestHex(digest.as_bytes()) ) } + +/// Whether `name` is a 64-lowercase-hex namespace directory name. +pub(super) fn is_namespace_name(name: &OsStr) -> bool { + is_lower_hex(name, DIGEST_HEX) +} + +/// Whether `name` is a canonical `-` pool entry name. +pub(super) fn is_pool_name(name: &OsStr, suffix: &str) -> bool { + let Some(name) = name.to_str() else { + return false; + }; + let Some(stem) = name.strip_suffix(suffix) else { + return false; + }; + let Some((generation, digest)) = stem.split_once('-') else { + return false; + }; + is_lower_hex(OsStr::new(generation), GENERATION_HEX) + && is_lower_hex(OsStr::new(digest), DIGEST_HEX) +} + +fn is_lower_hex(name: &OsStr, length: usize) -> bool { + name.to_str().is_some_and(|text| { + text.len() == length + && text + .bytes() + .all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte)) + }) +} From ffba9a79c3b1b7006bb65425ee6472d8c07d6d27 Mon Sep 17 00:00:00 2001 From: James Ross Date: Mon, 7 Sep 2026 18:37:29 -0700 Subject: [PATCH 104/111] Docs: use reference-style links for the two overlong README lines Two README lines ran past 80 columns because a Markdown URL cannot wrap. Both now use reference-style links with the targets collected at the end of the file; the rendered text and link targets are unchanged, and the phrases the README contract tests pin are untouched. Self-review finding R21 (P5). Refs #78 --- CHANGELOG.md | 2 ++ README.md | 11 ++++++----- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8567ff1..7311350 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -242,6 +242,8 @@ after its public API and format compatibility policies are established. ### Changed +- The README's two overlong link lines use reference-style links so every + line fits the 80-column width without breaking a URL. - Retention pool and namespace name predicates live beside their emitters in one module, and the namespace census classifies entries from the directory listing's file type instead of a metadata call per entry; its doc states the diff --git a/README.md b/README.md index 94a0550..3a20ed0 100644 --- a/README.md +++ b/README.md @@ -7,8 +7,7 @@ Everything else in this repository exists to make that sentence true under power loss, process death, corrupted disks, and byte-identical files swapped -in underneath it. The -[authenticated reconstruction contract](docs/invariants/authenticated-reconstruction/README.md) +in underneath it. The [authenticated reconstruction contract][reconstruction] states the promise precisely, including its limits. Keep is a standalone Rust library. It is the storage layer beneath @@ -81,9 +80,8 @@ publishes a successor generation; it does not assert that bytes were destroyed. The authoritative status of every requirement, with the test that proves it, -is the ledger in -[`docs/formats/segment-store-v2/requirements.md`](docs/formats/segment-store-v2/requirements.md). -Its first rule: *a planned case is not evidence.* +is the [requirements ledger][ledger]. Its first rule: *a planned case is not +evidence.* ## How it works @@ -225,3 +223,6 @@ Report vulnerabilities through [SECURITY.md](SECURITY.md). Do not include plaintext content, keys, or sensitive paths in a public issue. Licensed under the [Apache License 2.0](LICENSE). + +[reconstruction]: docs/invariants/authenticated-reconstruction/README.md +[ledger]: docs/formats/segment-store-v2/requirements.md From abb5cf6591cbc1cbc54ca870ff13fc16184de180 Mon Sep 17 00:00:00 2001 From: James Ross Date: Mon, 7 Sep 2026 18:43:46 -0700 Subject: [PATCH 105/111] Docs: split the retention and recovery pages at their largest sections retention.md and recovery.md both sat at exactly the 300-line review threshold, so every later sentence had to be paid for by rewrapping an earlier one. Closure admission and the generation transition now live in retention-publication.md; the ordered one-way migration protocol and partial migration recovery live in migration-recovery.md. Each old page keeps its record grammars and routes to its new companion, and the version-two overview routes to all four. No sentence changed meaning; no section-anchor link pointed into the moved sections. The protocol contract pins the moved phrases on their new pages, reads the recovery pair as the one contract they state, and adds both new pages to the review-threshold list. The pages now stand at 242, 65, 242, and 67 lines. Self-review finding R24 (P5). Refs #78 --- CHANGELOG.md | 6 ++ docs/formats/segment-store-v2/README.md | 10 ++- .../segment-store-v2/migration-recovery.md | 67 +++++++++++++++++++ docs/formats/segment-store-v2/recovery.md | 62 +---------------- .../segment-store-v2/retention-publication.md | 65 ++++++++++++++++++ docs/formats/segment-store-v2/retention.md | 64 +----------------- .../retention_store_v2_protocol_contract.rs | 18 ++++- .../migration_contract_laws.rs | 11 ++- 8 files changed, 174 insertions(+), 129 deletions(-) create mode 100644 docs/formats/segment-store-v2/migration-recovery.md create mode 100644 docs/formats/segment-store-v2/retention-publication.md diff --git a/CHANGELOG.md b/CHANGELOG.md index 7311350..8ff2252 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -242,6 +242,12 @@ after its public API and format compatibility policies are established. ### Changed +- The version-two retention and recovery pages each split their largest + sections into `retention-publication.md` (closure admission and the + generation transition) and `migration-recovery.md` (the one-way migration + protocol and partial-migration recovery), so every page sits well under the + 300-line review threshold; the overview routes to both and the protocol + contract pins their phrases. - The README's two overlong link lines use reference-style links so every line fits the 80-column width without breaking a URL. - Retention pool and namespace name predicates live beside their emitters in diff --git a/docs/formats/segment-store-v2/README.md b/docs/formats/segment-store-v2/README.md index 04cb7b3..7f65830 100644 --- a/docs/formats/segment-store-v2/README.md +++ b/docs/formats/segment-store-v2/README.md @@ -40,8 +40,10 @@ Version 2 retains every version-1 physical law and adds these: The following pages form one protocol: -- [Retention records and publication](retention.md) owns canonical namespace, - root-generation, manifest, retention-head, and transition rules. +- [Retention records](retention.md) owns canonical namespace, root-generation, + manifest, and retention-head rules. +- [Retention publication](retention-publication.md) owns closure admission and + the generation transition. - [Closure verification](closure.md) owns deterministic traversal, exact resource accounting, authenticated reconstruction, and closure evidence. - [Closure corruption boundary](closure-corruption.md) owns the admitted-record @@ -49,8 +51,10 @@ The following pages form one protocol: - [GC and disposition records](gc.md) owns the canonical planned intent, completion, and recovery-disposition byte grammars. - [Migration and recovery](recovery.md) owns the exact root namespace, - version marker, reader fence, one-way migration, crash states, GC reservation, + version marker, reader fence, migration records, GC reservation, recovery-disposition reservation, and restart behavior. +- [Migration protocol and recovery](migration-recovery.md) owns the ordered + one-way migration protocol and partial-migration recovery. - [Migration crash points](migration-crash.md) owns fixed-stage publication and the exact process-death boundaries for migration. - [Migration inventory](migration-inventory.md) owns the bounded canonical diff --git a/docs/formats/segment-store-v2/migration-recovery.md b/docs/formats/segment-store-v2/migration-recovery.md new file mode 100644 index 0000000..63f1c19 --- /dev/null +++ b/docs/formats/segment-store-v2/migration-recovery.md @@ -0,0 +1,67 @@ +# Migration Protocol and Recovery + +This page owns the ordered one-way migration protocol and partial-migration +recovery for `keep.segment-store/v2`. The root namespace, format marker, reader +fence, and migration record grammars are owned by +[Migration and recovery](recovery.md); the exact process-death boundaries are +owned by [Migration crash points](migration-crash.md). + +## One-way migration protocol + +Migration performs these ordered steps: + +1. Admit and completely recover the exact version-1 store. +2. Revalidate its head, catalog, pools, root identity, and writer authority. +3. Publish `migration.intent` from `migration.intent.next` through the + no-replacement fixed-stage protocol. +4. Create and verify persistent `reader.lock`. +5. Create the exact `retention`, `retention/roots`, + `retention/manifests`, `gc`, `recovery`, and + `recovery/dispositions` directories. +6. Synchronize every created parent and the store root. +7. Publish `FORMAT` from `FORMAT.next` through the fixed-stage protocol. +8. Reopen and verify the complete version-2 view. +9. Publish `migration.receipt` from `migration.receipt.next` through the + fixed-stage protocol. + +The [migration crash-point specification](migration-crash.md) owns +`KEEP-CRASH-053` through `KEEP-CRASH-073`. +Migration never rewrites or deletes admitted version-1 immutable bytes and +provides no automatic downgrade. + +`FilesystemStoreMigrationAuthority` retains the writer lock and pinned root +and pools. It admits the version-1 namespace, Linux root identity, `HEAD`, +complete immutable-pool inventory, and selected catalog. Before mutation, it +requires the same canonical intent. Its port retains fixed-record handles, +verifies bytes and inode identity at each publication boundary, and admits only +ordered prefixes. It reopens the complete view before receipt staging and +leaves all version-1 immutable bytes untouched. +Version-1 admission refuses after a migration stage, `migration.intent`, +`reader.lock`, `FORMAT`, or version-2 directory exists. After durable intent, +only version-2 migration recovery may continue. + +## Partial migration recovery + +The migration recovery boundary admits only these ordered prefixes: + + + +| State | Required response | +| --- | --- | +| no migration artifact | admit exact version 1 | +| intent stage only | finalize an exact stage or explicitly discard an incomplete pre-effect stage | +| durable intent only | verify intent and continue | +| intent plus a canonical prefix of v2 names | verify each name and continue | +| complete v2 shape without marker | verify directories and write marker | +| marker without receipt | reopen full v2 view and publish receipt | +| exact receipt with optional exact receipt stage | clean the stage and admit complete migration | + + + +A partial migration retry revalidates intent and existing bytes, resumes at the +first absent canonical step, and never replaces an entry. A missing predecessor, +changed version-1 coordinate, out-of-order name, wrong kind or bytes, conflicting +receipt, unknown entry, or changed root identity is unrecoverable ambiguity. + +Death before durable intent leaves v1 plus at most its non-authoritative stage. +Death after durable intent leaves recovery-required v2 migration state. diff --git a/docs/formats/segment-store-v2/recovery.md b/docs/formats/segment-store-v2/recovery.md index 958dd58..c389940 100644 --- a/docs/formats/segment-store-v2/recovery.md +++ b/docs/formats/segment-store-v2/recovery.md @@ -1,6 +1,8 @@ # Migration and Recovery This page owns version-2 filesystem migration and recovery. +[Migration protocol and recovery](migration-recovery.md) owns the ordered +one-way migration protocol and partial-migration recovery. ## Exact filesystem namespace @@ -175,66 +177,6 @@ writer emits only those canonical records; success is not restart evidence. Version 2 remains unavailable as production until partial-prefix recovery and `KEEP-MIGRATION-007` process-death evidence exist. -## One-way migration protocol - -Migration performs these ordered steps: - -1. Admit and completely recover the exact version-1 store. -2. Revalidate its head, catalog, pools, root identity, and writer authority. -3. Publish `migration.intent` from `migration.intent.next` through the - no-replacement fixed-stage protocol. -4. Create and verify persistent `reader.lock`. -5. Create the exact `retention`, `retention/roots`, - `retention/manifests`, `gc`, `recovery`, and - `recovery/dispositions` directories. -6. Synchronize every created parent and the store root. -7. Publish `FORMAT` from `FORMAT.next` through the fixed-stage protocol. -8. Reopen and verify the complete version-2 view. -9. Publish `migration.receipt` from `migration.receipt.next` through the - fixed-stage protocol. - -The [migration crash-point specification](migration-crash.md) owns -`KEEP-CRASH-053` through `KEEP-CRASH-073`. -Migration never rewrites or deletes admitted version-1 immutable bytes and -provides no automatic downgrade. - -`FilesystemStoreMigrationAuthority` retains the writer lock and pinned root -and pools. It admits the version-1 namespace, Linux root identity, `HEAD`, -complete immutable-pool inventory, and selected catalog. Before mutation, it -requires the same canonical intent. Its port retains fixed-record handles, -verifies bytes and inode identity at each publication boundary, and admits only -ordered prefixes. It reopens the complete view before receipt staging and -leaves all version-1 immutable bytes untouched. -Version-1 admission refuses after a migration stage, `migration.intent`, -`reader.lock`, `FORMAT`, or version-2 directory exists. After durable intent, -only version-2 migration recovery may continue. - -## Partial migration recovery - -The migration recovery boundary admits only these ordered prefixes: - - - -| State | Required response | -| --- | --- | -| no migration artifact | admit exact version 1 | -| intent stage only | finalize an exact stage or explicitly discard an incomplete pre-effect stage | -| durable intent only | verify intent and continue | -| intent plus a canonical prefix of v2 names | verify each name and continue | -| complete v2 shape without marker | verify directories and write marker | -| marker without receipt | reopen full v2 view and publish receipt | -| exact receipt with optional exact receipt stage | clean the stage and admit complete migration | - - - -A partial migration retry revalidates intent and existing bytes, resumes at the -first absent canonical step, and never replaces an entry. A missing predecessor, -changed version-1 coordinate, out-of-order name, wrong kind or bytes, conflicting -receipt, unknown entry, or changed root identity is unrecoverable ambiguity. - -Death before durable intent leaves v1 plus at most its non-authoritative stage. -Death after durable intent leaves recovery-required v2 migration state. - ## Retention publication recovery At restart, a fixed retention stage is classified from its exact framing and diff --git a/docs/formats/segment-store-v2/retention-publication.md b/docs/formats/segment-store-v2/retention-publication.md new file mode 100644 index 0000000..d432c0f --- /dev/null +++ b/docs/formats/segment-store-v2/retention-publication.md @@ -0,0 +1,65 @@ +# Retention Publication + +This page owns closure admission and the generation transition for +`keep.segment-store/v2`. The record grammars it publishes are owned by +[Retention records](retention.md); restart behaviour for a retained stage is +owned by [Migration and recovery](recovery.md). + +## Closure admission + +Before publication, Keep pins one completely verified catalog generation and +applies the exact deterministic traversal, counter units, failure order, +authenticated reconstruction, and canonical digest defined by +[Closure verification](closure.md). Any closure failure refuses the entire +transition. Keep never omits one failed member and continues with a smaller +live set. + +Preflight verifies steps 3 and 4 without I/O; preparation binds that proof to +the current manifest and derives exact canonical successors. +`execute_retention_publication` revalidates current authority, executes all 17 +ordered durability phases, and returns the complete receipt only after cleanup; +exact already-committed retry revalidates authority and performs no mutation. + +Version-2 catalog publication holds the same writer authority and proves every +current retained closure against its candidate catalog before replacing the +catalog `HEAD`. + +## Generation transition + +A transition supplies a namespace, an expected state of absent or one exact +`RootGeneration`, a complete canonical anchor set, the exact realization +profile coordinate, and admitted limits. + +Under exclusive writer authority, publication: + +1. completes recovery of every fixed retention stage; +2. admits the current retention head, manifest, and selected namespace root; +3. compares expected and observed generations; +4. verifies the candidate closure against one pinned catalog; +5. writes and synchronizes `retention/root.next`; +6. for a new namespace, exclusively creates and verifies its exact digest-named + directory, then synchronizes `retention/roots`; +7. links and verifies the root pool entry, then synchronizes its directory; +8. writes and synchronizes `retention/manifest.next`; +9. links and verifies the manifest pool entry and synchronizes its directory; +10. writes and synchronizes `retention/head.next`; +11. atomically replaces `retention/HEAD` and synchronizes `retention`; + `root.next` and `manifest.next` remain durable until the retention head + commits, then are removed and `retention` is synchronized again; and +12. returns a consequential `#[must_use]` receipt. + +The receipt binds the namespace, expected and observed generations, committed +root generation and digest, global manifest generation and digest, profile +coordinate, anchor-set and closure digests, catalog generation and digest, and +every durable publication outcome. + +A stale transition preserves expected and observed generations. A +byte-identical retry returns **already committed** only while that exact root +successor remains current; otherwise it returns the precise stale state. + +A reader holds one shared `ReaderFence` and double-collects the catalog and +retention heads around complete transitive admission. It accepts only the same +coordinates before and after for both heads. Any generation, length, digest, or +checksum change discards the view and retries within a bounded attempt limit; +exhaustion refuses. The accepted view observes one complete root generation for +its snapshot lifetime. diff --git a/docs/formats/segment-store-v2/retention.md b/docs/formats/segment-store-v2/retention.md index 238fc82..6f7402f 100644 --- a/docs/formats/segment-store-v2/retention.md +++ b/docs/formats/segment-store-v2/retention.md @@ -1,7 +1,8 @@ # Retention Records and Publication -This page owns retention values, root-generation records, manifests, heads, -closure admission, and publication for `keep.segment-store/v2`. +This page owns retention values, root-generation records, manifests, and heads +for `keep.segment-store/v2`. [Retention publication](retention-publication.md) +owns closure admission and the generation transition. ## Scalar and identity rules @@ -239,62 +240,3 @@ retention/manifests/ | 112 | 32 | checksum | BLAKE3-256 over bytes `0..112` | The checksum domain is `keep.retention-head-checksum/v2\0`. - -## Closure admission - -Before publication, Keep pins one completely verified catalog generation and -applies the exact deterministic traversal, counter units, failure order, -authenticated reconstruction, and canonical digest defined by -[Closure verification](closure.md). Any closure failure refuses the entire -transition. Keep never omits one failed member and continues with a smaller -live set. - -Preflight verifies steps 3 and 4 without I/O; preparation binds that proof to -the current manifest and derives exact canonical successors. -`execute_retention_publication` revalidates current authority, executes all 17 -ordered durability phases, and returns the complete receipt only after cleanup; -exact already-committed retry revalidates authority and performs no mutation. - -Version-2 catalog publication holds the same writer authority and proves every -current retained closure against its candidate catalog before replacing the -catalog `HEAD`. - -## Generation transition - -A transition supplies a namespace, an expected state of absent or one exact -`RootGeneration`, a complete canonical anchor set, the exact realization -profile coordinate, and admitted limits. - -Under exclusive writer authority, publication: - -1. completes recovery of every fixed retention stage; -2. admits the current retention head, manifest, and selected namespace root; -3. compares expected and observed generations; -4. verifies the candidate closure against one pinned catalog; -5. writes and synchronizes `retention/root.next`; -6. for a new namespace, exclusively creates and verifies its exact digest-named - directory, then synchronizes `retention/roots`; -7. links and verifies the root pool entry, then synchronizes its directory; -8. writes and synchronizes `retention/manifest.next`; -9. links and verifies the manifest pool entry and synchronizes its directory; -10. writes and synchronizes `retention/head.next`; -11. atomically replaces `retention/HEAD` and synchronizes `retention`; - `root.next` and `manifest.next` remain durable until the retention head - commits, then are removed and `retention` is synchronized again; and -12. returns a consequential `#[must_use]` receipt. - -The receipt binds the namespace, expected and observed generations, committed -root generation and digest, global manifest generation and digest, profile -coordinate, anchor-set and closure digests, catalog generation and digest, and -every durable publication outcome. - -A stale transition preserves expected and observed generations. A -byte-identical retry returns **already committed** only while that exact root -successor remains current; otherwise it returns the precise stale state. - -A reader holds one shared `ReaderFence` and double-collects the catalog and -retention heads around complete transitive admission. It accepts only the same -coordinates before and after for both heads. Any generation, length, digest, or -checksum change discards the view and retries within a bounded attempt limit; -exhaustion refuses. The accepted view observes one complete root generation for -its snapshot lifetime. diff --git a/xtask/tests/retention_store_v2_protocol_contract.rs b/xtask/tests/retention_store_v2_protocol_contract.rs index 09f2316..0847610 100644 --- a/xtask/tests/retention_store_v2_protocol_contract.rs +++ b/xtask/tests/retention_store_v2_protocol_contract.rs @@ -48,11 +48,13 @@ fn version_two_is_one_routed_protocol() -> Result<(), Box for required in [ "`keep.segment-store/v2`", "successor to `keep.segment-store/v1`", - "[Retention records and publication](retention.md)", + "[Retention records](retention.md)", + "[Retention publication](retention-publication.md)", "[Closure verification](closure.md)", "[Closure corruption boundary](closure-corruption.md)", "[GC and disposition records](gc.md)", "[Migration and recovery](recovery.md)", + "[Migration protocol and recovery](migration-recovery.md)", "[Migration crash points](migration-crash.md)", "[Migration inventory](migration-inventory.md)", "[Requirements and evidence](requirements.md)", @@ -87,6 +89,14 @@ fn retention_records_have_exact_canonical_grammars() -> Result<(), Box Result<(), Box Result<(), Box Result<(), Box> { - let recovery = normalized(&read(&format!("{FORMAT_ROOT}/recovery.md"))?); + // recovery.md owns the namespace, marker, fence, and record grammars; + // migration-recovery.md owns the ordered protocol and partial recovery. The + // two pages state one contract, so the laws read them together. + let recovery = format!( + "{}\n{}", + normalized(&read(&format!("{FORMAT_ROOT}/recovery.md"))?), + normalized(&read(&format!("{FORMAT_ROOT}/migration-recovery.md"))?) + ); for required in [ "one-way explicit migration", @@ -44,7 +51,7 @@ fn migration_and_recovery_define_every_authority_boundary() -> Result<(), Box Date: Mon, 7 Sep 2026 18:59:09 -0700 Subject: [PATCH 106/111] Test: build the byte-equal substitute so its inode always differs The exact-record law for byte-equal substitution deleted the record and wrote a new file at the same path. On APFS the new file always receives a fresh inode number, so the law passed locally; on ext4 the kernel may hand the new file the inode number just freed, the identity matched, and CI refused the law with "byte-equal substitute admitted" on abb5cf65. The law now creates the substitute beside the original and renames it over the entry. Two files that exist at the same time cannot share an inode, so the substitute's identity differs on every filesystem, which is the situation the primitive exists to refuse. The retention and migration substitution laws already used rename and passed on Linux; only this one was platform-bound. Refs #78 --- src/adapters/filesystem_exact_record_tests.rs | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/adapters/filesystem_exact_record_tests.rs b/src/adapters/filesystem_exact_record_tests.rs index 28c4e87..fa1dc3d 100644 --- a/src/adapters/filesystem_exact_record_tests.rs +++ b/src/adapters/filesystem_exact_record_tests.rs @@ -94,8 +94,12 @@ fn verify_named_refuses_different_bytes_and_a_byte_equal_substitute() -> Result< let bytes = super::verify_named(&directory, "record", b"other", identity) .err() .ok_or("different bytes admitted")?; - fs::remove_file(&path)?; - fs::write(&path, b"exact")?; + // Create the substitute beside the original and rename it over the entry: + // two files that exist at once cannot share an inode, whereas a file + // recreated after deletion may be handed the freed inode number on ext4. + let substitute_path = sandbox.path().join("substitute"); + fs::write(&substitute_path, b"exact")?; + fs::rename(&substitute_path, &path)?; let substitute = super::verify_named(&directory, "record", b"exact", identity) .err() .ok_or("byte-equal substitute admitted")?; From 04ac69cab527d4fe79219ac3aa5f6d4e8ca1853b Mon Sep 17 00:00:00 2001 From: James Ross Date: Mon, 7 Sep 2026 20:39:48 -0700 Subject: [PATCH 107/111] Fix: refuse zero-generation retention pool names as noncanonical is_pool_name accepted any 16 lowercase hex digits as the generation component, so an entry such as 0000000000000000-.root passed the census although root and liveness generations are positive by definition and the emitters can never produce that name. The census then treated an impossible coordinate as canonical and publication continued over it. The predicate now parses the generation component and requires a positive value, refusing zero as NoncanonicalPoolEntry like any other malformed name. A law plants a zero-generation root pool entry and requires the refusal; it failed before this change and passes now. Codex review, third pass (filesystem_retention_pool_name.rs). Refs #78 --- CHANGELOG.md | 3 ++ .../filesystem_retention_namespace_tests.rs | 28 +++++++++++++++++++ .../filesystem_retention_pool_name.rs | 12 ++++++-- 3 files changed, 41 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8ff2252..aad1116 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -605,6 +605,9 @@ after its public API and format compatibility policies are established. Review corrections to the unreleased retention and migration work above; none of these shipped in a release. +- Retention pool names whose generation component encodes zero refuse as + noncanonical; root and liveness generations are positive, so the census no + longer treats an impossible coordinate as a canonical entry. - An already-committed retry presented over an absent retention head refuses as `CommittedRetryOverAbsentHead` instead of the misnamed `StaleCommittedRetry`; the successor-over-absent-head law now downcasts to diff --git a/src/adapters/retention/filesystem_retention_namespace_tests.rs b/src/adapters/retention/filesystem_retention_namespace_tests.rs index a800265..2cb02e9 100644 --- a/src/adapters/retention/filesystem_retention_namespace_tests.rs +++ b/src/adapters/retention/filesystem_retention_namespace_tests.rs @@ -111,3 +111,31 @@ fn uppercase_root_pool_name_refuses() -> Result<(), Box> { )); Ok(()) } + +#[test] +fn zero_generation_root_pool_name_refuses() -> Result<(), Box> { + let (sandbox, mut authority) = open_authority("filesystem-retention-zero-generation")?; + let namespace = sandbox + .path() + .join("retention") + .join("roots") + .join("b".repeat(64)); + fs::create_dir(&namespace)?; + fs::write( + namespace.join(format!("{}-{}.root", "0".repeat(16), "c".repeat(64))), + b"impossible generation zero", + )?; + let root_bytes = fixture(ROOT_HEX)?; + let preparation = initial_preparation(&root_bytes)?; + + let error = RetentionPublicationStorage::verify_current(&mut authority, &preparation) + .err() + .ok_or("a zero-generation pool name was unexpectedly admitted")?; + + assert_eq!(error.kind(), io::ErrorKind::InvalidData); + assert!(matches!( + super::filesystem_retention_test_fixture::refusal(&error), + Some(super::RetentionCurrentStateRefusal::NoncanonicalPoolEntry { .. }) + )); + Ok(()) +} diff --git a/src/adapters/retention/filesystem_retention_pool_name.rs b/src/adapters/retention/filesystem_retention_pool_name.rs index 15b6ac4..ec5add0 100644 --- a/src/adapters/retention/filesystem_retention_pool_name.rs +++ b/src/adapters/retention/filesystem_retention_pool_name.rs @@ -49,6 +49,10 @@ pub(super) fn is_namespace_name(name: &OsStr) -> bool { } /// Whether `name` is a canonical `-` pool entry name. +/// +/// The generation component must spell a positive `u64` in exactly 16 lowercase +/// hex digits: root and liveness generations are positive, so a zero encoding +/// names no protocol coordinate and refuses as noncanonical. pub(super) fn is_pool_name(name: &OsStr, suffix: &str) -> bool { let Some(name) = name.to_str() else { return false; @@ -59,8 +63,12 @@ pub(super) fn is_pool_name(name: &OsStr, suffix: &str) -> bool { let Some((generation, digest)) = stem.split_once('-') else { return false; }; - is_lower_hex(OsStr::new(generation), GENERATION_HEX) - && is_lower_hex(OsStr::new(digest), DIGEST_HEX) + is_positive_generation(generation) && is_lower_hex(OsStr::new(digest), DIGEST_HEX) +} + +fn is_positive_generation(text: &str) -> bool { + is_lower_hex(OsStr::new(text), GENERATION_HEX) + && u64::from_str_radix(text, 16).is_ok_and(|generation| generation != 0) } fn is_lower_hex(name: &OsStr, length: usize) -> bool { From 3e63d07705fd7ef91ce6b8f7c51782f408cfe036 Mon Sep 17 00:00:00 2001 From: James Ross Date: Mon, 7 Sep 2026 22:03:33 -0700 Subject: [PATCH 108/111] Fix: refuse every publication from a store beyond the namespace ceiling admit_capacity bounded only the creation of a new namespace directory: a candidate whose namespace already existed returned Ok without comparing the census against RetentionManifest::MAXIMUM_ENTRY_COUNT. A store that had somehow accumulated more than 4,096 namespace directories therefore kept publishing successors although no manifest can describe it. The census is now compared against the ceiling before the existing-namespace exception applies; an over-full store refuses as NamespaceCapacity for every candidate until recovery reduces it. A law fills the pool to 4,097 namespaces after publishing generation one and requires a successor in the existing namespace to refuse with an unchanged witness; it failed before this change and passes now. Exactly 4,096 still admits a successor (existing law). Codex review, third pass (filesystem_retention_namespace.rs). Refs #78 --- CHANGELOG.md | 3 ++ .../filesystem_retention_capacity_tests.rs | 33 +++++++++++++++++++ .../filesystem_retention_namespace.rs | 10 ++++-- 3 files changed, 44 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index aad1116..ecad578 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -605,6 +605,9 @@ after its public API and format compatibility policies are established. Review corrections to the unreleased retention and migration work above; none of these shipped in a release. +- A retention store already holding more than 4,096 namespace directories + refuses every publication as `NamespaceCapacity`, including a successor in + an existing namespace, instead of only refusing the 4,097th directory. - Retention pool names whose generation component encodes zero refuse as noncanonical; root and liveness generations are positive, so the census no longer treats an impossible coordinate as a canonical entry. diff --git a/src/adapters/retention/filesystem_retention_capacity_tests.rs b/src/adapters/retention/filesystem_retention_capacity_tests.rs index 7441948..22ed982 100644 --- a/src/adapters/retention/filesystem_retention_capacity_tests.rs +++ b/src/adapters/retention/filesystem_retention_capacity_tests.rs @@ -107,3 +107,36 @@ fn namespace_hex(candidate: &AdmittedRetentionRoot<'_>) -> String { rendered }) } + +#[test] +fn an_overfull_namespace_pool_refuses_even_a_successor_in_an_existing_namespace() +-> Result<(), Box> { + let (sandbox, mut authority) = open_authority("filesystem-retention-capacity-overfull")?; + let root_bytes = fixture(ROOT_HEX)?; + let _published = + execute_retention_publication(&mut authority, &initial_preparation(&root_bytes)?)?; + let current = authority + .observe_current()? + .ok_or("published retention head was not observed")?; + let current_root = AdmittedRetentionRoot::decode(&root_bytes)?; + let current_manifest = AdmittedRetentionManifest::decode(current.manifest_bytes())?; + create_orphans( + sandbox.path(), + RetentionManifest::MAXIMUM_ENTRY_COUNT, + &namespace_hex(¤t_root), + )?; + let candidate = successor_root(¤t_root)?; + let preparation = successor_preparation(¤t_root, ¤t_manifest, candidate.encoded())?; + let before = retention_witness(sandbox.path())?; + + let error = RetentionPublicationStorage::verify_current(&mut authority, &preparation) + .err() + .ok_or("a store beyond the namespace ceiling admitted a successor")?; + + assert!(matches!( + refusal(&error), + Some(RetentionCurrentStateRefusal::NamespaceCapacity) + )); + assert_eq!(retention_witness(sandbox.path())?, before); + Ok(()) +} diff --git a/src/adapters/retention/filesystem_retention_namespace.rs b/src/adapters/retention/filesystem_retention_namespace.rs index c4ba062..b6a03f0 100644 --- a/src/adapters/retention/filesystem_retention_namespace.rs +++ b/src/adapters/retention/filesystem_retention_namespace.rs @@ -78,13 +78,19 @@ pub(super) fn admit( /// Orphan namespace directories protected by recovery count exactly like /// manifest entries: a candidate whose namespace directory is absent may be /// admitted only while the observed count is below -/// [`RetentionManifest::MAXIMUM_ENTRY_COUNT`]. A candidate whose namespace -/// already exists creates nothing and is not bounded here. +/// [`RetentionManifest::MAXIMUM_ENTRY_COUNT`]. A store already holding more +/// namespaces than the ceiling is beyond the format and refuses every +/// publication, including a successor in an existing namespace, until +/// recovery reduces it. Otherwise a candidate whose namespace already exists +/// creates nothing and is not bounded here. pub(super) fn admit_capacity( census: RetentionNamespaceCensus, roots: &Dir, candidate: &AdmittedRetentionRoot<'_>, ) -> io::Result<()> { + if census.namespace_count > RetentionManifest::MAXIMUM_ENTRY_COUNT { + return Err(Refusal::NamespaceCapacity.into_io()); + } let name = pool_name::namespace(candidate.root().namespace().digest()); match roots.symlink_metadata(&name) { Ok(_) => Ok(()), From bf862426fe7ff5d7e0d10a8e33a51a10a6feed05 Mon Sep 17 00:00:00 2001 From: James Ross Date: Mon, 7 Sep 2026 22:27:27 -0700 Subject: [PATCH 109/111] Fix: refuse migration while version-one staging holds a retained stage FilesystemStoreMigrationAuthority admitted the published version-one root shape but never looked inside staging. A store carrying current.seg or current.cat from an interrupted version-one publication could therefore publish its migration intent, marker, and receipt with that recovery evidence still in place; afterwards the version-one recovery constructors refuse the version-two markers and version-two stage recovery does not yet exist, so a recoverable crash state became stranded. verify_namespace now opens staging without following links and requires it to be empty before the intent is observed, refusing at the Namespace boundary with the reason. A law plants a retained stage and requires observe_intent to refuse with the version-one witness unchanged and no intent or intent stage written; it failed before this change and passes now. The retention fixtures, which migrate a clean store, are unaffected. Codex review, third pass (filesystem_migration_authority.rs). Refs #78 --- CHANGELOG.md | 4 +++ .../filesystem_migration_authority.rs | 32 +++++++++++++++++-- .../filesystem_migration_storage_tests.rs | 25 +++++++++++++++ 3 files changed, 59 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ecad578..db07d97 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -605,6 +605,10 @@ after its public API and format compatibility policies are established. Review corrections to the unreleased retention and migration work above; none of these shipped in a release. +- Migration refuses a version-one store whose `staging` directory holds a + retained stage, so an interrupted version-one publication is recovered + before the intent exists instead of being stranded behind the version-two + markers. - A retention store already holding more than 4,096 namespace directories refuses every publication as `NamespaceCapacity`, including a successor in an existing namespace, instead of only refusing the 4,097th directory. diff --git a/src/adapters/store_migration/filesystem_migration_authority.rs b/src/adapters/store_migration/filesystem_migration_authority.rs index f6ca8d3..e894bcb 100644 --- a/src/adapters/store_migration/filesystem_migration_authority.rs +++ b/src/adapters/store_migration/filesystem_migration_authority.rs @@ -1,5 +1,7 @@ //! This module owns exact writer-locked filesystem migration authority. +use cap_fs_ext::DirExt; + use super::filesystem_inventory_file::{self, FilesystemInventoryFilePolicy}; use super::filesystem_migration_authority_error::{ FilesystemMigrationAuthorityArtifact as Artifact, FilesystemMigrationAuthorityError as Error, @@ -19,6 +21,7 @@ use crate::adapters::{ }; const HEAD_NAME: &str = "HEAD"; +const STAGING_NAME: &str = "staging"; const HEAD_LENGTH: u64 = 128; /// Exclusive authority to observe and migrate one pinned version-1 filesystem root. @@ -123,9 +126,34 @@ impl FilesystemStoreMigrationAuthority { } } + /// Admits the exact published version-one root and requires `staging` to + /// hold nothing. + /// + /// A retained `current.seg` or `current.cat` is version-one recovery + /// evidence. Migration never inventories `staging`, and once the version-two + /// markers exist the version-one recovery constructors refuse the root, so + /// migrating over a retained stage would strand a recoverable crash state. + /// Recovery must complete before the intent is observed. fn verify_namespace(&self) -> Result<(), Error> { - filesystem_initialization_namespace::admit_published(self.inventory.root()) - .map_err(|source| Error::Namespace { source }) + let root = self.inventory.root(); + filesystem_initialization_namespace::admit_published(root) + .map_err(|source| Error::Namespace { source })?; + let staging = root + .open_dir_nofollow(STAGING_NAME) + .map_err(|source| Error::Namespace { source })?; + let mut entries = staging + .entries() + .map_err(|source| Error::Namespace { source })?; + match entries.next().transpose() { + Ok(None) => Ok(()), + Ok(Some(_entry)) => Err(Error::Namespace { + source: std::io::Error::new( + std::io::ErrorKind::InvalidData, + "version-one staging holds a retained stage; recover it before migration", + ), + }), + Err(source) => Err(Error::Namespace { source }), + } } fn verify_root_identity(&self) -> Result { diff --git a/src/adapters/store_migration/filesystem_migration_storage_tests.rs b/src/adapters/store_migration/filesystem_migration_storage_tests.rs index cd21454..b0cd656 100644 --- a/src/adapters/store_migration/filesystem_migration_storage_tests.rs +++ b/src/adapters/store_migration/filesystem_migration_storage_tests.rs @@ -201,3 +201,28 @@ fn directory_names(path: &Path) -> io::Result> { .map(|entry| entry.map(|entry| entry.file_name())) .collect() } + +#[test] +fn a_retained_version_one_stage_refuses_migration_before_any_intent() -> Result<(), Box> +{ + let (sandbox, authority) = open_authority("filesystem-migration-retained-stage")?; + fs::write( + sandbox.path().join("staging").join("current.seg"), + b"interrupted version-one publication", + )?; + let before = version_one_witness(sandbox.path())?; + + let error = authority + .observe_intent() + .err() + .ok_or("a store with a retained version-one stage was admitted for migration")?; + + assert!(matches!( + error, + crate::adapters::FilesystemMigrationAuthorityError::Namespace { .. } + )); + assert_eq!(version_one_witness(sandbox.path())?, before); + assert!(!sandbox.path().join("migration.intent").exists()); + assert!(!sandbox.path().join("migration.intent.next").exists()); + Ok(()) +} From ff926a3af4c4dec459de01772c6fa7540fff9acc Mon Sep 17 00:00:00 2001 From: James Ross Date: Mon, 7 Sep 2026 22:52:43 -0700 Subject: [PATCH 110/111] Fix: keep the admitted retention directories pinned through authority construction FilesystemVersionTwoAdmission::reopen admitted every version-two directory and record, then returned a value holding only the writer lock. The publication authority reopened retention, roots, and manifests by name, so a directory renamed and replaced between reopen and open was opened without repeating admission; the advisory lock does not prevent such a replacement. The admission now retains the three capabilities it admitted and hands them to the authority through into_parts, so the authority never resolves those names itself. Current-state verification additionally requires each name to still resolve to the pinned directory, refusing with the new ProtocolDirectoryReplaced when the namespace no longer describes the admitted state, so a swapped-in directory is neither opened nor published into. A law publishes generation one, reopens, swaps retention out from under the admission, and requires open to succeed against the pinned directory and verification to refuse; it failed before this change and passes now. Codex review, second pass (filesystem_version_two_admission.rs), P1. Refs #78 --- CHANGELOG.md | 5 +++ .../filesystem_version_two_admission.rs | 31 +++++++++++-- .../filesystem_retention_authority.rs | 17 ++----- .../retention/filesystem_retention_refusal.rs | 6 +++ .../retention/filesystem_retention_storage.rs | 33 ++++++++++++++ .../filesystem_version_two_admission_tests.rs | 45 ++++++++++++++++++- 6 files changed, 118 insertions(+), 19 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index db07d97..18160ba 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -605,6 +605,11 @@ after its public API and format compatibility policies are established. Review corrections to the unreleased retention and migration work above; none of these shipped in a release. +- `FilesystemVersionTwoAdmission` retains the `retention`, `roots`, and + `manifests` capabilities it admitted and hands them to the publication + authority, and current-state verification requires those names to still + resolve to the pinned directories (`ProtocolDirectoryReplaced`), so a + directory swapped in after reopen is neither opened nor published into. - Migration refuses a version-one store whose `staging` directory holds a retained stage, so an interrupted version-one publication is recovered before the intent exists instead of being stranded behind the version-two diff --git a/src/adapters/filesystem_version_two_admission.rs b/src/adapters/filesystem_version_two_admission.rs index 61f52fc..2303c5a 100644 --- a/src/adapters/filesystem_version_two_admission.rs +++ b/src/adapters/filesystem_version_two_admission.rs @@ -2,6 +2,7 @@ use std::path::Path; +use cap_fs_ext::DirExt; #[cfg(test)] use cap_std::ambient_authority; use cap_std::fs::Dir; @@ -20,10 +21,16 @@ use super::{ /// [`FilesystemPlatformAdmission`](super::FilesystemPlatformAdmission): a /// version-one publisher cannot consume it, so version-one catalog publication /// can never run against a migrated root and leave residue no adapter admits. -/// Fields are private so only version-two admission can create values. +/// Fields are private so only version-two admission can create values. The +/// admitted `retention`, `roots`, and `manifests` capabilities are retained, +/// so the authority built from this value operates on the directories that +/// passed admission and not on whatever those names resolve to later. #[must_use] pub struct FilesystemVersionTwoAdmission { lock: FilesystemWriterLock, + retention: Dir, + roots: Dir, + manifests: Dir, } impl FilesystemVersionTwoAdmission { @@ -56,8 +63,9 @@ impl FilesystemVersionTwoAdmission { Self::admit(root) } - pub(super) fn into_lock(self) -> FilesystemWriterLock { - self.lock + /// Releases the writer lock and the three pinned retention capabilities. + pub(super) fn into_parts(self) -> (FilesystemWriterLock, Dir, Dir, Dir) { + (self.lock, self.retention, self.roots, self.manifests) } fn admit(root: Dir) -> Result { @@ -73,10 +81,25 @@ impl FilesystemVersionTwoAdmission { let bound = filesystem_version_two_records::admit(&directory) .map_err(|source| FilesystemPlatformAdmissionError::MigrationRecord { source })?; require_root_identity(bound, observed)?; - Ok(Self { lock }) + let retention = pin(&directory, "retention")?; + let roots = pin(&retention, "roots")?; + let manifests = pin(&retention, "manifests")?; + Ok(Self { + lock, + retention, + roots, + manifests, + }) } } +/// Pins one admitted protocol directory without following links. +fn pin(parent: &Dir, name: &str) -> Result { + parent + .open_dir_nofollow(name) + .map_err(|source| FilesystemPlatformAdmissionError::Namespace { source }) +} + /// Requires the reopened root to be the physical root the migration intent bound. /// /// Device, mount, and file coordinates are compared exactly, as the migration diff --git a/src/adapters/retention/filesystem_retention_authority.rs b/src/adapters/retention/filesystem_retention_authority.rs index 88df5d8..ada4ed5 100644 --- a/src/adapters/retention/filesystem_retention_authority.rs +++ b/src/adapters/retention/filesystem_retention_authority.rs @@ -2,7 +2,6 @@ use std::io; -use cap_fs_ext::DirExt; use cap_std::fs::Dir; use super::filesystem_retention_attempt::PublicationAttempt; @@ -10,7 +9,6 @@ use super::filesystem_retention_authority_error::{ FilesystemRetentionAuthorityError as Error, RetentionAuthorityDirectory as Directory, }; use super::filesystem_retention_current::{self, ObservedRetentionState}; -use super::filesystem_retention_pool_name as pool_name; use crate::adapters::{FilesystemVersionTwoAdmission, FilesystemWriterLock}; /// Exclusive authority to publish retention transitions on one pinned root. @@ -56,17 +54,14 @@ impl FilesystemRetentionPublicationAuthority { /// # Errors /// /// Returns [`FilesystemRetentionAuthorityError`](super::FilesystemRetentionAuthorityError) - /// when the root capability cannot be cloned or the retention namespace and - /// either immutable pool cannot be pinned without following links. + /// when the root capability cannot be cloned. The retention namespace and + /// both immutable pools arrive already pinned by admission. pub fn open(admission: FilesystemVersionTwoAdmission) -> Result { - let lock = admission.into_lock(); + let (lock, retention, roots, manifests) = admission.into_parts(); let root = lock.clone_directory().map_err(|source| Error::Directory { directory: Directory::Root, source, })?; - let retention = open_directory(&root, pool_name::RETENTION, Directory::Retention)?; - let roots = open_directory(&retention, pool_name::ROOTS, Directory::Roots)?; - let manifests = open_directory(&retention, pool_name::MANIFESTS, Directory::Manifests)?; Ok(Self { root, retention, @@ -91,9 +86,3 @@ impl FilesystemRetentionPublicationAuthority { filesystem_retention_current::observe(&self.retention, &self.manifests) } } - -fn open_directory(parent: &Dir, name: &str, directory: Directory) -> Result { - parent - .open_dir_nofollow(name) - .map_err(|source| Error::Directory { directory, source }) -} diff --git a/src/adapters/retention/filesystem_retention_refusal.rs b/src/adapters/retention/filesystem_retention_refusal.rs index 6f83875..e682b2c 100644 --- a/src/adapters/retention/filesystem_retention_refusal.rs +++ b/src/adapters/retention/filesystem_retention_refusal.rs @@ -109,6 +109,9 @@ pub enum RetentionCurrentStateRefusal { /// A byte-identical already-committed retry was presented while no /// retention head is published, so nothing can have committed it. CommittedRetryOverAbsentHead, + /// A protocol directory named at admission (`retention`, `roots`, or + /// `manifests`) no longer names the pinned directory that was admitted. + ProtocolDirectoryReplaced, /// A record's kind or length disagreed with its declaration. RecordKindOrLength, /// A record carried bytes beyond its declared length. @@ -208,6 +211,9 @@ impl RetentionCurrentStateRefusal { Self::RecordKindOrLength => "retention record kind or length disagreed", Self::RecordTrailingBytes => "retention record carried trailing bytes", Self::RecordLengthOverflow => "retention record length exceeded the addressable range", + Self::ProtocolDirectoryReplaced => { + "a retention protocol directory was replaced after admission" + } Self::CommittedRetryOverAbsentHead => { "already-committed retry presented while no retention head is published" } diff --git a/src/adapters/retention/filesystem_retention_storage.rs b/src/adapters/retention/filesystem_retention_storage.rs index 303c2f2..e5c897c 100644 --- a/src/adapters/retention/filesystem_retention_storage.rs +++ b/src/adapters/retention/filesystem_retention_storage.rs @@ -19,6 +19,7 @@ use super::{ }; use crate::RetentionGenerationExpectation; use crate::adapters::filesystem_catalog_artifact::synchronize_directory; +use crate::adapters::filesystem_exact_record::EntryIdentity; impl RetentionPublicationStorage for FilesystemRetentionPublicationAuthority { fn verify_current( @@ -26,6 +27,7 @@ impl RetentionPublicationStorage for FilesystemRetentionPublicationAuthority { preparation: &RetentionPublicationPreparation<'_>, ) -> io::Result { self.attempt = None; + require_pinned_directories(&self.root, &self.retention, &self.roots, &self.manifests)?; require_no_retained_stage(&self.retention)?; let census = filesystem_retention_namespace::admit(&self.retention, &self.roots, &self.manifests)?; @@ -230,6 +232,37 @@ impl RetentionPublicationStorage for FilesystemRetentionPublicationAuthority { } } +/// Requires the protocol names to still resolve to the directories admission pinned. +/// +/// The authority operates only on the pinned capabilities, but a `retention`, +/// `roots`, or `manifests` entry renamed and replaced after admission means the +/// store's namespace no longer describes the admitted state; publication +/// refuses instead of writing into a directory no reader would find. +fn require_pinned_directories( + root: &Dir, + retention: &Dir, + roots: &Dir, + manifests: &Dir, +) -> io::Result<()> { + for (parent, name, pinned) in [ + (root, pool_name::RETENTION, retention), + (retention, pool_name::ROOTS, roots), + (retention, pool_name::MANIFESTS, manifests), + ] { + let current = match parent.open_dir_nofollow(name) { + Ok(current) => current, + Err(source) if source.kind() == io::ErrorKind::NotFound => { + return Err(RetentionCurrentStateRefusal::ProtocolDirectoryReplaced.into_io()); + } + Err(source) => return Err(source), + }; + if EntryIdentity::of_directory(¤t)? != EntryIdentity::of_directory(pinned)? { + return Err(RetentionCurrentStateRefusal::ProtocolDirectoryReplaced.into_io()); + } + } + Ok(()) +} + fn require_no_retained_stage(retention: &Dir) -> io::Result<()> { for stage in [ pool_name::ROOT_STAGE, diff --git a/src/adapters/retention/filesystem_version_two_admission_tests.rs b/src/adapters/retention/filesystem_version_two_admission_tests.rs index a26171c..a5e86c6 100644 --- a/src/adapters/retention/filesystem_version_two_admission_tests.rs +++ b/src/adapters/retention/filesystem_version_two_admission_tests.rs @@ -3,13 +3,20 @@ use std::error::Error; use std::fs; -use super::filesystem_retention_test_fixture::migrated_store; +use super::filesystem_retention_test_fixture::{ + ROOT_HEX, fixture, initial_preparation, migrated_store, open_authority, refusal, +}; +use super::{ + FilesystemRetentionPublicationAuthority, RetentionCurrentStateRefusal, + RetentionPublicationStorage, +}; use crate::adapters::filesystem_root_identity::FilesystemRootIdentity; use crate::adapters::filesystem_version_two_admission::{BoundRootIdentity, require_root_identity}; use crate::adapters::{ FilesystemPlatformAdmissionError, FilesystemVersionTwoAdmission, StoreRootIdentityCoordinate, VersionTwoRecordRefusal, }; +use crate::execute_retention_publication; #[test] fn version_two_reopen_refuses_a_corrupt_format_marker() -> Result<(), Box> { @@ -194,3 +201,39 @@ fn version_two_reopen_refuses_a_missing_retention_pool() -> Result<(), Box Result<(), Box> { + let (sandbox, mut first) = open_authority("version-two-admission-replaced-retention")?; + let root_bytes = fixture(ROOT_HEX)?; + let preparation = initial_preparation(&root_bytes)?; + let _published = execute_retention_publication(&mut first, &preparation)?; + drop(first); + let admission = FilesystemVersionTwoAdmission::reopen_unchecked_for_tests(sandbox.path())?; + fs::rename( + sandbox.path().join("retention"), + sandbox.path().join("retention.moved"), + )?; + fs::create_dir_all(sandbox.path().join("retention").join("roots"))?; + fs::create_dir(sandbox.path().join("retention").join("manifests"))?; + + let mut authority = FilesystemRetentionPublicationAuthority::open(admission)?; + let observed = authority + .observe_current()? + .ok_or("the admitted retention directory lost its published head")?; + let error = authority + .verify_current(&preparation) + .err() + .ok_or("publication proceeded although retention was replaced after admission")?; + + assert_eq!( + observed.head().generation(), + preparation.liveness_generation() + ); + assert!(matches!( + refusal(&error), + Some(RetentionCurrentStateRefusal::ProtocolDirectoryReplaced) + )); + Ok(()) +} From d8d71ed41bae33fcc27650e6f0f0a0eaa09f875f Mon Sep 17 00:00:00 2001 From: James Ross Date: Mon, 7 Sep 2026 23:17:58 -0700 Subject: [PATCH 111/111] Fix: reopen the head-selected catalog under retention authority require_current_catalog bound this store's catalog HEAD to the generation and digest the closure was verified against, but it reopened only HEAD. A preparation verified from a byte-identical store, or before this store lost its catalog pool entry, still published: the head agreed while the catalog it named was absent or corrupt, and the receipt cited evidence these pools could not reproduce. Publication now also reopens the catalog pool entry HEAD selects, bounded by the head's declared length through the shared exact-record reader, decodes it, and requires its generation and digest to equal the head's, refusing with CatalogAbsent, CatalogRefused { source }, or CatalogChanged. A law deletes the selected catalog after the snapshot and requires CatalogAbsent; it failed before this change and passes now. Closure-member segments are deliberately not re-read at publication: every read authenticates them, and their re-verification under authority belongs to retention recovery. requirements.md records the nonclaim. Codex review, third pass (filesystem_retention_catalog.rs), P1. Refs #78 --- CHANGELOG.md | 5 ++ docs/formats/segment-store-v2/requirements.md | 4 ++ .../retention/filesystem_retention_catalog.rs | 46 +++++++++++++++---- .../filesystem_retention_catalog_tests.rs | 27 +++++++++++ .../retention/filesystem_retention_refusal.rs | 20 +++++++- .../filesystem_retention_test_fixture.rs | 2 +- 6 files changed, 92 insertions(+), 12 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 18160ba..fd6d2f4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -605,6 +605,11 @@ after its public API and format compatibility policies are established. Review corrections to the unreleased retention and migration work above; none of these shipped in a release. +- Retention publication reopens the catalog pool entry this store's `HEAD` + selects, bounded by the head's declared length, and requires it to decode + to that generation and digest (`CatalogAbsent`, `CatalogRefused`, + `CatalogChanged`), so a preparation verified before the catalog was lost no + longer publishes; closure-member segments are documented as not re-read. - `FilesystemVersionTwoAdmission` retains the `retention`, `roots`, and `manifests` capabilities it admitted and hands them to the publication authority, and current-state verification requires those names to still diff --git a/docs/formats/segment-store-v2/requirements.md b/docs/formats/segment-store-v2/requirements.md index a5cda67..ea516fd 100644 --- a/docs/formats/segment-store-v2/requirements.md +++ b/docs/formats/segment-store-v2/requirements.md @@ -64,5 +64,9 @@ case is not evidence. instead of continuing it. A stage left behind by a failed write is recovery evidence like any crash residue; it is never unlinked, and the next publication refuses until recovery classifies it. +- Publication binds this store's catalog `HEAD` to the verified closure and + reopens the head-selected catalog pool entry under authority, but it does + not re-read closure-member segments: every read authenticates them, and + their re-verification under authority belongs to retention recovery. - Benchmarks are required before performance-sensitive retention or migration optimization. diff --git a/src/adapters/retention/filesystem_retention_catalog.rs b/src/adapters/retention/filesystem_retention_catalog.rs index c08ec67..446f853 100644 --- a/src/adapters/retention/filesystem_retention_catalog.rs +++ b/src/adapters/retention/filesystem_retention_catalog.rs @@ -4,20 +4,26 @@ use std::io; use cap_std::fs::Dir; +use cap_fs_ext::DirExt; + use super::filesystem_retention_current::read_exact_optional; use super::{RetentionCurrentStateRefusal, RetentionPublicationPreparation}; -use crate::adapters::ChecksummedPublicationHead; +use crate::adapters::{ChecksummedCatalog, ChecksummedPublicationHead, physical_pool_name}; const HEAD_NAME: &str = "HEAD"; +const CATALOGS_NAME: &str = "catalogs"; const HEAD_LENGTH: usize = crate::adapters::publication_head_decoder::ENCODED_LENGTH; /// Requires the store's catalog head to name the catalog the closure was verified against. /// /// A preparation carries a closure verified against one pinned `CatalogSnapshot`, -/// which the caller may have taken from another store. Publication must not -/// proceed unless this store's own `HEAD` names exactly that catalog generation -/// and digest; otherwise the receipt would cite foreign evidence for anchors -/// whose records may be absent from these pools. +/// which the caller may have taken from another store or before this store +/// lost a pool entry. Publication must not proceed unless this store's own +/// `HEAD` names exactly that catalog generation and digest and the selected +/// catalog pool entry reopens under this authority, bounded by the head's +/// declared length, and decodes to that generation and digest. Closure-member +/// segments are not re-read here: every read authenticates them, and their +/// re-verification under authority belongs to retention recovery. pub(super) fn require_current_catalog( root: &Dir, preparation: &RetentionPublicationPreparation<'_>, @@ -33,14 +39,34 @@ pub(super) fn require_current_catalog( })?; let head = ChecksummedPublicationHead::decode(&bytes) .map_err(|source| RetentionCurrentStateRefusal::CatalogHeadRefused { source }.into_io())?; - if head.generation() == expected_generation && head.catalog_digest() == closure.catalog_digest() + if head.generation() != expected_generation || head.catalog_digest() != closure.catalog_digest() { - Ok(()) - } else { - Err(RetentionCurrentStateRefusal::CatalogDisagreed { + return Err(RetentionCurrentStateRefusal::CatalogDisagreed { expected_generation, observed_generation: Some(head.generation()), } - .into_io()) + .into_io()); + } + require_selected_catalog(root, head) +} + +/// Reopens the catalog pool entry `head` selects and requires it to be that catalog. +fn require_selected_catalog(root: &Dir, head: ChecksummedPublicationHead<'_>) -> io::Result<()> { + let catalogs = root.open_dir_nofollow(CATALOGS_NAME)?; + let name = physical_pool_name::catalog(head.generation(), head.catalog_digest()); + let length = usize::try_from(head.catalog_length().get()) + .map_err(|_source| RetentionCurrentStateRefusal::RecordLengthOverflow.into_io())?; + let bytes = read_exact_optional(&catalogs, &name, length)? + .ok_or_else(|| RetentionCurrentStateRefusal::CatalogAbsent.into_io())?; + let catalog = ChecksummedCatalog::decode(&bytes).map_err(|source| { + RetentionCurrentStateRefusal::CatalogRefused { + source: Box::new(source), + } + .into_io() + })?; + if (catalog.generation(), catalog.digest()) == (head.generation(), head.catalog_digest()) { + Ok(()) + } else { + Err(RetentionCurrentStateRefusal::CatalogChanged.into_io()) } } diff --git a/src/adapters/retention/filesystem_retention_catalog_tests.rs b/src/adapters/retention/filesystem_retention_catalog_tests.rs index e030f1c..87f745f 100644 --- a/src/adapters/retention/filesystem_retention_catalog_tests.rs +++ b/src/adapters/retention/filesystem_retention_catalog_tests.rs @@ -97,3 +97,30 @@ fn a_corrupt_catalog_head_refuses_with_its_decode_error() -> Result<(), Box Result<(), Box> { + let (sandbox, mut authority) = super::filesystem_retention_test_fixture::open_authority( + "filesystem-retention-catalog-absent", + )?; + let root_bytes = super::filesystem_retention_test_fixture::fixture( + super::filesystem_retention_test_fixture::ROOT_HEX, + )?; + let preparation = super::filesystem_retention_test_fixture::initial_preparation(&root_bytes)?; + std::fs::remove_file( + sandbox + .path() + .join("catalogs") + .join(super::filesystem_retention_test_fixture::CATALOG_NAME), + )?; + + let error = RetentionPublicationStorage::verify_current(&mut authority, &preparation) + .err() + .ok_or("publication proceeded although the head-selected catalog is absent")?; + + assert!(matches!( + super::filesystem_retention_test_fixture::refusal(&error), + Some(RetentionCurrentStateRefusal::CatalogAbsent) + )); + Ok(()) +} diff --git a/src/adapters/retention/filesystem_retention_refusal.rs b/src/adapters/retention/filesystem_retention_refusal.rs index e682b2c..be4d20d 100644 --- a/src/adapters/retention/filesystem_retention_refusal.rs +++ b/src/adapters/retention/filesystem_retention_refusal.rs @@ -5,7 +5,7 @@ use std::fmt; use std::io; use super::{RetentionHeadDecodeError, RetentionManifestDecodeError}; -use crate::adapters::PublicationHeadDecodeError; +use crate::adapters::{CatalogDecodeError, PublicationHeadDecodeError}; use crate::{CatalogGeneration, LivenessGeneration, RetentionManifestDigest}; /// Exact reason filesystem current-state verification refused a transition. @@ -60,6 +60,16 @@ pub enum RetentionCurrentStateRefusal { /// The exact decode refusal. source: PublicationHeadDecodeError, }, + /// The catalog pool entry this store's `HEAD` selects is absent. + CatalogAbsent, + /// The catalog pool entry this store's `HEAD` selects did not decode. + CatalogRefused { + /// The exact decode refusal. + source: Box, + }, + /// The catalog pool entry decodes to a generation or digest other than the + /// one `HEAD` names. + CatalogChanged, /// The current liveness generation has no successor. LivenessExhausted, /// A byte-identical retry found that another successor is current. @@ -177,6 +187,13 @@ impl RetentionCurrentStateRefusal { Self::HeadPredecessorDisagreed => { "current retention head and its manifest name different predecessors" } + Self::CatalogAbsent => "this store's catalog head selects an absent catalog pool entry", + Self::CatalogRefused { .. } => { + "this store's selected catalog pool entry refused admission" + } + Self::CatalogChanged => { + "this store's selected catalog pool entry names another generation or digest" + } Self::CatalogHeadRefused { .. } => "this store's catalog head refused admission", Self::LivenessExhausted => "current liveness generation cannot advance", Self::StaleCommittedRetry => { @@ -235,6 +252,7 @@ impl Error for RetentionCurrentStateRefusal { Self::HeadRefused { source } | Self::PreparedHeadRefused { source } => Some(source), Self::ManifestRefused { source } => Some(source), Self::CatalogHeadRefused { source } => Some(source), + Self::CatalogRefused { source } => Some(source.as_ref()), _ => None, } } diff --git a/src/adapters/retention/filesystem_retention_test_fixture.rs b/src/adapters/retention/filesystem_retention_test_fixture.rs index a72aca1..9672c10 100644 --- a/src/adapters/retention/filesystem_retention_test_fixture.rs +++ b/src/adapters/retention/filesystem_retention_test_fixture.rs @@ -45,7 +45,7 @@ const CATALOG_HEAD_HEX: &str = include_str!("../../../conformance/segment-store/v1/one-zero-bundle-head.hex"); const SEGMENT_NAME: &str = "221f6745cd8a5221c9a87c3707593608479282b54a4a74d0e753fd76f70e8db2.seg"; -const CATALOG_NAME: &str = +pub(super) const CATALOG_NAME: &str = "0000000000000001-0b7cad1b6de663d34beacbc214db7497f2e36ab6b08dfbd5febbc8d06a418811.cat"; /// Builds one migrated version-2 store and pins its retention authority.