diff --git a/Cargo.lock b/Cargo.lock index 5f96ed94..244cf9fd 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -863,8 +863,7 @@ dependencies = [ [[package]] name = "ant-protocol" version = "2.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "83f2b55e89a468584cfc91dc2f596583e089c17107de6f2f386f8339b665d1f0" +source = "git+https://github.com/grumbach/ant-protocol?branch=settlement-version-quote-gate#e97901694cba5cad84da0b99ee8af02a791df2cf" dependencies = [ "blake3", "bytes", @@ -3106,7 +3105,7 @@ dependencies = [ "libc", "percent-encoding", "pin-project-lite", - "socket2 0.5.10", + "socket2 0.6.4", "tokio", "tower-service", "tracing", @@ -4293,7 +4292,7 @@ dependencies = [ "quinn-udp 0.5.15", "rustc-hash", "rustls", - "socket2 0.5.10", + "socket2 0.6.4", "thiserror 2.0.18", "tokio", "tracing", @@ -4332,7 +4331,7 @@ dependencies = [ "cfg_aliases", "libc", "once_cell", - "socket2 0.5.10", + "socket2 0.6.4", "tracing", "windows-sys 0.61.2", ] @@ -4345,7 +4344,7 @@ checksum = "76150b617afc75e6e21ac5f39bc196e80b65415ae48d62dbef8e2519d040ce42" dependencies = [ "cfg_aliases", "libc", - "socket2 0.5.10", + "socket2 0.6.4", "tracing", "windows-sys 0.61.2", ] @@ -6418,7 +6417,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.48.0", + "windows-sys 0.61.2", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 2b99f0f1..74feee3d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -39,7 +39,10 @@ mimalloc = "0.1" # Until then, the git pin tracks the matching saorsa-core lineage # (the rc-2026.4.2 branch) so Cargo can unify the wire types here # with ant-protocol's re-exports. -ant-protocol = "2.3.2" +# Branch pin while the settlement-version wire types are in review +# (WithAutonomi/ant-protocol#23). Swap back to a published `ant-protocol` +# version pin once that PR merges and the release train publishes it. +ant-protocol = { git = "https://github.com/grumbach/ant-protocol", branch = "settlement-version-quote-gate" } # Core (provides EVERYTHING: networking, DHT, security, trust, storage) saorsa-core = "0.27.0" diff --git a/docs/adr/ADR-0010-settlement-version-and-pre-payment-compatibility.md b/docs/adr/ADR-0010-settlement-version-and-pre-payment-compatibility.md new file mode 100644 index 00000000..39af0594 --- /dev/null +++ b/docs/adr/ADR-0010-settlement-version-and-pre-payment-compatibility.md @@ -0,0 +1,188 @@ +# ADR-0010: Settlement version and pre-payment compatibility + +- **Status:** Proposed +- **Date:** 2026-08-13 +- **Decision owners:** Anselme Grumbach +- **Reviewers:** David Irvine +- **Supersedes:** none +- **Superseded by:** none +- **Related:** ADR-0008 (storage economics and payment protocol); ant-protocol#23; ant-node#204; ant-client#171 + +## Context + +ADR-0008 raised the merkle settlement multiplier to 3x. It changed client code and node code and **changed no wire type**, so nothing tied the two together. A client built before the change could still collect quotes, still pay, and only then discover that every storer refused the upload. + +That ordering is what makes it expensive. A merkle batch settles on-chain **before any storer sees a PUT**, and merkle receipts are not refundable. So the check that mattered ran after the money was gone. Production has been rejecting a slow trickle of such uploads at an exact 3x shortfall, each one a user's payment destroyed, reported to them as two integers to compare. + +ADR-0008 anticipated this and listed it as a re-open trigger: *"a rise in refused batch uploads after the boundary, indicating clients that never upgraded."* This ADR is that trigger being answered. + +The general problem is that **`PROTOCOL_VERSION` describes what a peer can parse, and nothing described how a peer settles**. Those two move independently. Every future change to settlement arithmetic reproduces this failure unless something carries the second fact. + +## Decision Drivers + +- A refusal must land **before** payment. Merkle settlement is irreversible, so after-the-fact verification cannot be the only check. +- A client-first rollout must stay **usable**, though this decision does narrow it. ADR-0008 chose client-first because a client paying more was accepted by an old node for free. Bounding the upper end ends that: an old node now refuses a newer client rather than quoting a payment it cannot promise to honour. The refusal is deliberately not a client fault, so a newer client routes to peers that can serve it and its user sees nothing, but **node rollout becomes a prerequisite for the next settlement bump** rather than merely desirable. That is the price of not paying before compatibility is established, and it is paid knowingly. +- Old peers must not be misread. `ChunkMessage` is postcard-encoded and non-self-describing, so a new field silently changes how existing peers parse every message. +- The user must be told what to do. The failure is only expensive because it is silent. + +## Considered Options + +1. **Bump `CHUNK_PROTOCOL_ID` to v3.** The established mechanism for wire changes here (ADR-0004, ADR-0005). Clean end state, but a hard cutover: old peers cannot negotiate a stream at all, so they get a handshake failure rather than a diagnosis, and the whole fleet plus all clients must move together. +2. **Append versioned request variants.** No cutover. Existing discriminants are untouched, unknown variants are rejected cleanly, and nodes and clients can move independently. +3. **Move the multiplier into the contract.** Client submits the 1x price, the contract applies the multiplier. This removes the payer's ability to get it wrong at all, and is the only option that protects clients which never upgrade. Requires a contract upgrade. +4. **Do nothing; drive client adoption.** Cheapest, and closes the *current* burn faster than any of the above. Does nothing for the next settlement change. + +## Decision + +We will adopt **option 2**, and treat **option 3 as the eventual structural fix** rather than a competing one. + +A `settlement_version` travels in the quote request. It is a separate constant from `PROTOCOL_VERSION` and is bumped whenever a change makes an older client pay an amount storers will refuse. + +### The compatibility rule + +A storer quotes only when the client's version falls inside its own inclusive range: + +``` +MIN_SUPPORTED_SETTLEMENT_VERSION <= client_version <= CURRENT_SETTLEMENT_VERSION +``` + +**Both ends are bounded, and the upper bound is the part that is easy to get wrong.** An earlier revision of this work accepted everything at or above the minimum, reasoning that a storer verifies whatever payment actually arrives, so letting a newer client through weakens nothing. That is true only when a settlement change *raises* what is paid. ADR-0008's 3x cleared an old node's 1x minimum, so old nodes accepted new clients for free, and that special case was mistaken for the general rule. A change that redefines the median rule, or which field the contract pays from, produces a payment an older verifier rejects, after the client has already settled. + +### Two refusals, not one + +The two out-of-range directions need opposite handling and are therefore separate wire variants. + +| Condition | Wire error | Whose problem | Client behaviour | +|---|---|---|---| +| `client_version < MIN` | `ClientUpdateRequired` | the client | **Terminal.** Abort before payment, show the upgrade instruction. | +| `client_version > CURRENT` | `StorerUpdateRequired` | this node | **Skip the peer.** Say nothing to the user; use other storers. | + +Collapsing them would either tell up-to-date users to upgrade, or strand new clients whenever the node fleet lags. A lagging fleet is the *normal* state during the client-first rollout ADR-0008 prescribes, so this distinction is what keeps the two ADRs compatible. + +If too few peers remain after skipping, the operation fails for lack of quotes. That is the correct outcome: it fails before payment rather than after. + +### The legacy fallback, and the rule that retires it + +A storer built before the versioned requests cannot decode them and simply never answers. Clients therefore retry each silent peer once in the legacy shape, or nothing would work until the whole fleet had upgraded. + +**This is a downgrade path.** Silence is not proof a peer cannot parse the request: a dropped response, packet loss, an overloaded peer, and one deliberately discarding versioned requests are indistinguishable from the client side. So the retry can be provoked, and a provoked retry bypasses the gate. + +It is safe **only while no client can be refused on version grounds**, which holds exactly while `MIN_SUPPORTED_SETTLEMENT_VERSION` is the first declarable version. The rule is therefore: + +> The unversioned retry must be deleted **before either `MIN_SUPPORTED_SETTLEMENT_VERSION` or `CURRENT_SETTLEMENT_VERSION` is raised.** + +Both, not just the minimum. Raising `MIN` creates the too-old refusal; raising `CURRENT` creates the node-behind refusal as soon as any node lags, and the retry routes around that one just as readily. + +This is enforced by a compile-time assertion in the client, not by review discipline: raising the minimum while a fallback exists fails the build. + +There are **two** independent fallbacks, single-node and merkle. The guard is therefore a single shared constant that each site references, so deleting one path cannot orphan the check for the other. An earlier revision put the assertion beside the merkle path only, which left the single-node retry unguarded while this document claimed otherwise. + +### A refusal is a verdict about the client, and needs corroboration + +Nothing authenticates a refusal. Acting on one peer's word would let a single hostile or misconfigured storer answer `ClientUpdateRequired` to everything and deny every upload the client attempts, converting an over-query design that tolerates many bad peers into one that tolerates none. So a refusal is believed only once `SETTLEMENT_REFUSAL_QUORUM` distinct peers agree, and a refusal that does not describe this client (wrong echoed version, or a stated minimum this client already meets) is discarded as a bad peer rather than counted. + +A genuine incompatibility reaches the threshold immediately, because every peer enforcing the newer rule refuses and a client queries far more than two. + +The corroborated verdict is then held **client-wide and sticky**, not in one collector's local state. It concerns this build rather than this upload, so an upload that begins after another has already established it must not proceed to pay, and every payment entry point checks it first. + +### A refusal must not depend on who answered first + +A refusal is a verdict about the client, not about one peer, so it cannot be treated as one failed response among many. Two things follow, and both were wrong in the first implementation: + +- The collector stops **launching** new peers once it has enough quotes, but keeps **draining** those already launched. Otherwise a refusal still in flight is discarded because faster peers filled the quota, and the upload pays. +- The verdict is recorded outside the collection timeout. The elapsed arm deliberately falls through so quotes from fast peers stay usable, and would otherwise throw away a refusal observed just before the clock ran out. + +`StorerUpdateRequired` deliberately does **not** get this treatment. It is not a verdict about the client, so one lagging peer must not abort an upload the rest of the close group can serve. + +### Unversioned requests are still served + +A storer cannot distinguish a client that settles correctly but predates the version field (ant-core 0.5.1 through 0.6.0) from one that does not. Refusing both would break clients that are behaving, so unversioned requests are served and counted. Nodes log a running total per path under `ant_node::quote::settlement`. + +Flipping that to a refusal is a **follow-up**, gated on that count decaying, and should use a dated self-retiring boundary in the style of `MERKLE_PARITY_ENFORCED_FROM_UNIX`. + +## Consequences + +### Positive + +- A settlement change no longer burns an outdated client's money **on the path the gate covers**: the refusal lands at quote time, where it costs nothing. It is a large reduction rather than an elimination, and the two holes are named under trade-offs below. Read this bullet with those. +- The user is told what happened and what to do, in both the quote refusal and the PUT-time underpayment message. +- No protocol cutover. Nodes and clients roll independently. +- The wire discriminants of every pre-existing message and error are pinned by a regression test, so the append-only property this rests on is enforced rather than assumed. + +### Negative / Trade-offs + +- **It does not fix the current burn.** A client too old to settle correctly is also too old to declare a version, so the gate cannot see the population causing today's rejections. Only the reworded error reaches them. Driving client adoption remains the cheaper and faster remedy for the incident that prompted this. +- **Merkle storers are not exactly the quoted peers.** The gate covers the 16 candidates a client quotes, but the chunk is stored by each chunk's close group, which may include a peer that never quoted, and routing can move that group between quoting and storing. A storer outside the candidate set can still refuse at PUT time, after payment. This narrows the exposure substantially without closing it, and only option 3 closes it fully. +- **A client binary already in the field cannot be reached.** The cutover guard bounds what future builds may do; it does nothing about a released client that still carries the fallback when nodes later raise their minimum. That is inherent to shipping software, and it is the same reason the storer verifies every payment it is actually offered rather than trusting the declared version. +- **A refusal in a later merkle sub-batch arrives after earlier sub-batches have paid.** Batches above `MAX_LEAVES` settle sequentially, so the gate cannot be consulted for sub-batch two before sub-batch one's money is spent. That call still returns its earlier proofs rather than failing: the caller writes the receipt cache only on the success path, so failing would strand a spend that has already settled on-chain, which is the destruction this work exists to prevent. Nothing is lost by returning them, because the verdict is latched client-wide and stops the *next* payment instead. +- **A single upload can still be denied by two colluding peers.** The corroboration threshold trades a lone-peer denial-of-service for a two-peer one. Two is chosen because a genuine incompatibility clears it instantly while a lone attacker cannot; raising it would blunt the real signal, and the failure mode is availability rather than lost funds. +- The fallback costs one probe per silent peer, bounded by `VERSIONED_QUOTE_PROBE_CEILING` and paid once per peer rather than once per request. Before those two bounds it was one full quote timeout on **every** request, which took the merkle E2E suite from ~24 minutes past the 60-minute CI cap. See the validation section. +- Clients declaring a version are refused by nodes that have not upgraded. Harmless today because no node has a lower `CURRENT`, but it makes node rollout a prerequisite for the next settlement bump. + +### Neutral / Operational + +- Deploy order is **nodes before clients**, since a node on `ant-protocol` 2.3.x cannot decode versioned requests. The fallback makes this a preference rather than a hard gate. +- The unversioned-quote counter is the input to the follow-up decision above. +- The `ant-protocol` change is additive, so it carries a minor semver impact. The version bump itself is left to the release train, not taken in the PR. + +## Validation + +- **Wire safety.** `appending_v2_variants_leaves_existing_discriminants_untouched` and `client_update_required_is_appended_to_protocol_error` pin the discriminant of every pre-existing message body **and** every pre-existing `ProtocolError`, responses included. An earlier revision pinned only the request half and the endpoints of the error enum, so a reordered response variant would have passed while breaking old peers. +- **Range policy.** `a_newer_settlement_version_is_refused_rather_than_assumed_compatible` pins the upper bound, which is the specific error this ADR corrects. +- **Refusal direction.** `a_newer_settlement_version_is_refused_as_this_nodes_fault` and `the_two_refusals_do_not_blame_the_same_party` pin that a lagging node never reports a client fault. +- **Terminality.** `a_refusal_aborts_quote_collection_instead_of_counting_as_one_bad_peer` pins that a refusal stops collection rather than joining the failure list, which is what would otherwise let the remaining peers form a quorum and pay anyway. +- **Order independence.** `a_refusal_is_recorded_where_the_collection_timeout_cannot_discard_it` and `meeting_the_target_stops_launching_without_stopping_collection`, plus `a_lagging_storer_does_not_populate_the_refusal_slot`. Stated precisely, because it is easy to overclaim: these pin the *mechanism* — that the verdict is stored where the elapsed arm cannot reach it, and that the launch budget stops recruiting at the target. **No test drives a real collection to timeout**, so the end-to-end ordering behaviour is argued from those two pieces rather than observed. +- **Downgrade bound.** The shared compile-time constant referenced from both fallback sites, plus `only_silence_triggers_the_legacy_retry` and `only_silence_is_evidence_worth_caching`. The constant now bounds `CURRENT` as well as `MIN`: raising either opens a refusal the unversioned retry could route around, and an earlier revision guarded only `MIN`. +### Mixed-version validation, and what it found + +The new-client-against-old-fleet case has been exercised for real, not simulated. `ant-client`'s merkle E2E suite spawns a 35-node testnet from the **published** `ant-node`, which predates the versioned requests, so the suite is a live mixed-version run: every node logs `Failed to decode message: deserialization failed` for each versioned probe and answers only the unversioned retry. + +It passed functionally and **failed on cost**, which is exactly what a unit test could not have shown: + +| | Merkle E2E, ubuntu | +|---|---| +| Baseline on `main` | 24–38 min | +| First implementation | exceeded the 60-min CI cap with 4 of 7 tests done | + +The cause was that a peer which cannot decode the versioned request never answers, so the client waited the **full** quote timeout before falling back, and paid that on every request rather than once per peer. The suite runs `quote_timeout_secs = 120`, and a merkle pool asks sixteen candidates. + +Two changes came out of it, both in the client: + +- **Remember the answer.** A peer that fails to answer a versioned request is asked in the legacy shape from then on, so the probe is paid once per peer instead of once per request. +- **Cap the probe**, with `VERSIONED_QUOTE_PROBE_CEILING`, but only far enough to bound a pathological timeout. Production's 10s sits below it, so **the ceiling never binds on a production client**. + +That last point is a safety property, not a tuning choice, and it was nearly got wrong. A shorter ceiling was tried to bring the slower CI runner under its job cap. It would have been a defect: the probe wait is the only window in which a peer can refuse, and the fallback re-asks under a *new* request id, so a refusal arriving after the ceiling answers a request nobody is listening to. It would never count toward corroboration, never set the latch, and the racing unversioned request could return a quote the client then pays against. Neither the never-demote rule nor the compile-time guard covers this: the first only stops a peer being *cached* as legacy, and the second binds future builds while the clients at risk are the ones already released. + +Cutting a probe short therefore does more than misjudge a slow peer. The rule is: **keep the ceiling at or above the largest production quote timeout.** + +The cost of that is roughly two minutes per merkle E2E test for as long as the suite's devnet speaks the pre-versioned dialect. That is an artifact of the temporary fork-branch protocol pin rather than of the design; when the fleet under test can answer a versioned request there are no probes to pay for and the suite returns to baseline. + +**Still outstanding:** the reverse direction, old client against an upgraded node, and a fleet with both. Those need a testnet built from this branch's `ant-node` rather than the published one, which is not available until the coordinated set lands. Also unproven end to end: structured refusal, lost refusal, and send failure against real peers, which are pinned at unit level only. + +### Release gates + +Merging puts this in the next release, so the bar is fleet-ready, not code-complete. Green CI covers the code gate and nothing else. **The set is not production ready while any of these is open:** + +| Gate | Status | +|---|---| +| Code / CI | Proven across all three repos | +| Dependency | Open: both downstream crates pin a mutable protocol branch | +| Mixed-version | Partial: new-client against an old fleet is proven by the client suite; the gate itself has never run over a real connection, because the devnet speaks the pre-versioned dialect | +| Deployment ordering | Open, and **not inert**: against a fleet that cannot answer a versioned request every first contact costs a probe wait, so releasing the client ahead of the nodes adds real latency to cold uploads | +| Observability | Open: the unversioned-quote counter is the signal that retires the legacy path and has never been read in production | +| NAT / canary | Open: no canary; relayed and NAT'd peers are the paths this adds work to | +| Rollback | Argued, not rehearsed | +| Fleet safety | Open: the corroboration quorum and the client-wide latch have never met a real fleet | + +The refusal machinery is inert on arrival, since `MIN` and `CURRENT` are both the first declarable version, so nothing can be refused yet. That lowers the risk. It does not close a gate, and the client-side cost above is live regardless. + +### Re-open triggers + +- The unversioned-quote count failing to decay, which would mean a long tail of clients the gate can never protect and would raise the priority of option 3. +- Any settlement change that is **not** a monotonic increase, which makes the upper bound load-bearing for the first time. +- Evidence of peers selectively dropping versioned requests, which would mean the downgrade path is being probed and the fallback should be retired early. +- A decision to move the multiplier on-chain, which would supersede most of this. + +## Notes for AI-assisted work + +AI tools may help draft this ADR, but **must not mark it Accepted without human review**. Accepted ADRs are immutable: create a new superseding ADR rather than editing an Accepted ADR. diff --git a/src/ant_protocol/mod.rs b/src/ant_protocol/mod.rs index ed01cf54..74921117 100644 --- a/src/ant_protocol/mod.rs +++ b/src/ant_protocol/mod.rs @@ -17,9 +17,11 @@ pub use ::ant_protocol::chunk; pub use ::ant_protocol::chunk::{ - ChunkGetRequest, ChunkGetResponse, ChunkMessage, ChunkMessageBody, ChunkPutRequest, - ChunkPutResponse, ChunkQuoteRequest, ChunkQuoteResponse, MerkleCandidateQuoteRequest, - MerkleCandidateQuoteResponse, ProtocolError, XorName, CHUNK_PROTOCOL_ID, CLOSE_GROUP_MAJORITY, - CLOSE_GROUP_SIZE, DATA_TYPE_CHUNK, MAX_CHUNK_SIZE, MAX_WIRE_MESSAGE_SIZE, PROOF_TAG_MERKLE, + settlement_compatibility, ChunkGetRequest, ChunkGetResponse, ChunkMessage, ChunkMessageBody, + ChunkPutRequest, ChunkPutResponse, ChunkQuoteRequest, ChunkQuoteRequestV2, ChunkQuoteResponse, + MerkleCandidateQuoteRequest, MerkleCandidateQuoteRequestV2, MerkleCandidateQuoteResponse, + ProtocolError, SettlementCompatibility, XorName, CHUNK_PROTOCOL_ID, CLOSE_GROUP_MAJORITY, + CLOSE_GROUP_SIZE, CURRENT_SETTLEMENT_VERSION, DATA_TYPE_CHUNK, MAX_CHUNK_SIZE, + MAX_WIRE_MESSAGE_SIZE, MIN_SUPPORTED_SETTLEMENT_VERSION, PROOF_TAG_MERKLE, PROOF_TAG_SINGLE_NODE, PROTOCOL_VERSION, XORNAME_LEN, }; diff --git a/src/payment/verifier.rs b/src/payment/verifier.rs index fd550c77..90a21942 100644 --- a/src/payment/verifier.rs +++ b/src/payment/verifier.rs @@ -354,6 +354,24 @@ fn merkle_required_multiplier(receipt_timestamp: u64, enforced_from: u64) -> u64 /// scaling a 1x result is wrong. The order here — multiply the total, then /// divide once — is deliberate, and matches the contract, which computes /// `totalAmount` before splitting it. +/// Does this underpayment look like a client that applied no multiplier at +/// all, as opposed to one that merely paid too little? +/// +/// Compares against the **bare** expectation rather than scaling the paid +/// amount up by the required multiplier. [`merkle_expected_per_node`] floors +/// after multiplying, so `expected(m)` is not generally `m * expected(1)`: at +/// median 901 and depth 7 they differ by one wei. A scaled comparison +/// therefore misses genuinely unmultiplied settlements at every depth that +/// does not divide its leaf count evenly, and those are exactly the clients +/// the upgrade advice exists for. +fn underpayment_looks_like_a_stale_client( + paid: Amount, + expected_bare: Amount, + required_multiplier: u64, +) -> bool { + required_multiplier > 1 && paid == expected_bare +} + fn merkle_expected_per_node( candidate_prices: &[Amount], depth: u8, @@ -3363,11 +3381,36 @@ impl PaymentVerifier { ))); } if *paid_amount < expected_per_node { + // An exact `required_multiplier`x shortfall is the signature of + // a client that settles under the superseded rule: it applied + // no multiplier at all. That is not a pricing dispute, it is an + // outdated client, and the person reading this has already paid + // and cannot get it back. Say so, and say what to do about it, + // rather than leaving them with two integers to compare. + // + // This message is the only thing that reaches such a client. + // The settlement-version gate refuses these uploads before they + // pay, but only for clients new enough to declare a version, + // which by construction excludes every client this branch + // catches. + let looks_like_stale_client = underpayment_looks_like_a_stale_client( + *paid_amount, + expected_per_node_bare, + required_multiplier, + ); + let advice = if looks_like_stale_client { + " This is exactly the pre-parity settlement amount, so the paying client is \ + too old to settle correctly. Run `ant update` to upgrade, or reinstall from \ + https://github.com/WithAutonomi/ant-client/releases/latest. Note the payment \ + that was already made cannot be recovered." + } else { + "" + }; return Err(Error::Payment(format!( "Underpayment for node at index {idx}: paid {paid_amount}, \ expected at least {expected_per_node} \ (median16 formula, depth={}, {required_multiplier}x required for a \ - receipt stamped {} vs parity boundary {parity_from})", + receipt stamped {} vs parity boundary {parity_from}).{advice}", payment_info.depth, payment_info.merkle_payment_timestamp ))); } @@ -3632,6 +3675,39 @@ mod tests { ); } + /// The same non-linearity that test pins also breaks a naive check for + /// "did this client apply no multiplier". + /// + /// Scaling the paid amount up by the multiplier and comparing against the + /// parity expectation asks `49_426 == 49_425` at median 901 depth 7, which + /// is false, so a genuinely unmultiplied settlement is not recognised and + /// the payer never learns their client is too old. Comparing against the + /// bare expectation asks the same arithmetic the expectation was built + /// with, and holds at every depth. + #[test] + fn a_stale_client_is_recognised_where_depth_does_not_divide_its_leaves() { + let prices = prices_with_median(901); + let bare = merkle_expected_per_node(&prices, 7, 1).expect("bare expectation is payable"); + let parity = merkle_expected_per_node(&prices, 7, PAID_QUOTE_PAYMENT_MULTIPLIER) + .expect("parity expectation is payable"); + + // The scaled comparison this replaced would have missed it. + assert_ne!(parity, bare * Amount::from(PAID_QUOTE_PAYMENT_MULTIPLIER)); + + assert!( + underpayment_looks_like_a_stale_client(bare, bare, PAID_QUOTE_PAYMENT_MULTIPLIER), + "an exactly-1x settlement must be recognised at depth 7" + ); + // A merely-cheap payment still is not blamed on the client's version. + assert!(!underpayment_looks_like_a_stale_client( + bare.saturating_sub(Amount::from(1u64)), + bare, + PAID_QUOTE_PAYMENT_MULTIPLIER + )); + // And under the legacy regime there is nothing to upgrade away from. + assert!(!underpayment_looks_like_a_stale_client(bare, bare, 1)); + } + /// The economic invariant ADR-0008 restores: a merkle **leaf** settles /// for what the single-node path settles — 3x the median quote — up to /// the contract's integer division. @@ -7040,6 +7116,59 @@ mod tests { err_msg.contains("Underpayment") && err_msg.contains("3x required"), "Error should name the required multiplier: {err_msg}" ); + // An exact 1x settlement is an outdated client, not a pricing dispute. + // The payer has already spent money they cannot recover, so the + // rejection has to tell them what to do rather than hand them two + // integers. This is the only message that reaches a client too old to + // declare a settlement version at quote time. + assert!( + err_msg.contains("too old to settle correctly") && err_msg.contains("ant update"), + "Error should tell the user to upgrade: {err_msg}" + ); + } + + /// The upgrade advice is keyed on an EXACT multiplier shortfall, which is + /// what an unmultiplied settlement looks like. A merely-cheap payment is a + /// different fault and must not be blamed on the client's version, or the + /// advice stops meaning anything. + #[tokio::test] + async fn a_partial_shortfall_is_not_blamed_on_an_outdated_client() { + let verifier = merkle_test_verifier(); + let (xorname, tagged_proof, pool_hash, ts) = make_valid_merkle_proof_bytes(); + + // One wei under parity: short, but not the 1x signature. + let almost = merkle_parity_per_node_depth2().saturating_sub(Amount::from(1u64)); + { + let info = evmlib::merkle_payments::OnChainPaymentInfo { + depth: 2, + merkle_payment_timestamp: ts, + paid_node_addresses: vec![ + (RewardsAddress::new([0u8; 20]), 0, almost), + (RewardsAddress::new([1u8; 20]), 1, almost), + ], + }; + verifier.pool_cache.lock().put(pool_hash, info); + } + + let err_msg = format!( + "{}", + verifier + .verify_payment( + &xorname, + Some(&tagged_proof), + VerificationContext::ClientPut + ) + .await + .expect_err("a short settlement must be refused") + ); + assert!( + err_msg.contains("Underpayment"), + "Error should still name the underpayment: {err_msg}" + ); + assert!( + !err_msg.contains("ant update"), + "A partial shortfall must not be reported as an outdated client: {err_msg}" + ); } /// A receipt stamped BEFORE the boundary keeps its 1x price. The money was diff --git a/src/storage/handler.rs b/src/storage/handler.rs index 08a40754..ba30d7fa 100644 --- a/src/storage/handler.rs +++ b/src/storage/handler.rs @@ -30,9 +30,11 @@ #[cfg(test)] use crate::ant_protocol::DATA_TYPE_CHUNK; use crate::ant_protocol::{ - ChunkGetRequest, ChunkGetResponse, ChunkMessage, ChunkMessageBody, ChunkPutRequest, - ChunkPutResponse, ChunkQuoteRequest, ChunkQuoteResponse, MerkleCandidateQuoteRequest, - MerkleCandidateQuoteResponse, ProtocolError, CHUNK_PROTOCOL_ID, MAX_CHUNK_SIZE, + settlement_compatibility, ChunkGetRequest, ChunkGetResponse, ChunkMessage, ChunkMessageBody, + ChunkPutRequest, ChunkPutResponse, ChunkQuoteRequest, ChunkQuoteRequestV2, ChunkQuoteResponse, + MerkleCandidateQuoteRequest, MerkleCandidateQuoteRequestV2, MerkleCandidateQuoteResponse, + ProtocolError, SettlementCompatibility, CHUNK_PROTOCOL_ID, CURRENT_SETTLEMENT_VERSION, + MAX_CHUNK_SIZE, MIN_SUPPORTED_SETTLEMENT_VERSION, }; use crate::client::compute_address; use crate::error::{Error, Result}; @@ -45,6 +47,7 @@ use crate::storage::lmdb::LmdbStorage; use bytes::Bytes; use parking_lot::RwLock; use saorsa_core::P2PNode; +use std::sync::atomic::{AtomicU64, Ordering}; use std::sync::Arc; use tokio::sync::mpsc; @@ -59,6 +62,70 @@ use tokio::sync::mpsc; /// turned away. const SELF_CLOSENESS_GATE_WIDTH: usize = K_BUCKET_SIZE; +/// How many unversioned quote requests to serve between adoption log lines. +/// +/// One line per request would drown the log at production quote rates, and one +/// line total would say nothing about the trend. A running count emitted every +/// `N` gives the shape of client adoption, which is the number that decides +/// when unversioned requests can start being refused outright. +const UNVERSIONED_QUOTE_LOG_INTERVAL: u64 = 1_000; + +/// Unversioned quote requests served since start, by path. +/// +/// Indices are `[single_node, merkle]`. A plain counter rather than a metric +/// because the only consumer is the rollout decision, and that reads logs. +static UNVERSIONED_QUOTES_SERVED: [AtomicU64; 2] = [AtomicU64::new(0), AtomicU64::new(0)]; + +/// Refuse a quote when the requesting client settles under rules this node no +/// longer accepts. `None` means the request may proceed. +/// +/// This is the whole point of the settlement version. A merkle batch pays +/// on-chain **before** any storer sees a PUT, so refusing the *payment* is +/// refusing something already spent and unrefundable. Refusing the *quote* +/// costs the client nothing: no quote means no pool commitment, which means no +/// payment. +/// +/// A version NEWER than this node understands is refused too, as +/// `StorerUpdateRequired`. Quoting it would promise to accept a payment whose +/// rules this build does not know, and that promise can only be broken at PUT +/// time, once the client has settled on-chain. The client is told to use a +/// different storer rather than to upgrade, because it is this node that is +/// behind. +fn settlement_gate(client_settlement_version: u32, path: &str) -> Option { + match settlement_compatibility(client_settlement_version) { + SettlementCompatibility::Compatible => None, + SettlementCompatibility::ClientTooOld => { + warn!( + target: "ant_node::quote::settlement", + "Refusing {path} quote: client settlement version {client_settlement_version} \ + is below the minimum {MIN_SUPPORTED_SETTLEMENT_VERSION}. No quote issued, \ + so the client has not been charged.", + ); + Some(ProtocolError::ClientUpdateRequired { + client_settlement_version, + min_settlement_version: MIN_SUPPORTED_SETTLEMENT_VERSION, + }) + } + // This node is the old one. Quoting would promise to accept a payment + // whose rules it does not know, and a promise broken at PUT time is + // broken after the client has settled on-chain. Refusing sends the + // client to a peer that can actually honour the quote, at no cost. + SettlementCompatibility::NodeTooOld => { + warn!( + target: "ant_node::quote::settlement", + "Refusing {path} quote: client settles under version \ + {client_settlement_version}, newer than this node's \ + {CURRENT_SETTLEMENT_VERSION}. This node needs upgrading; the client \ + has not been charged.", + ); + Some(ProtocolError::StorerUpdateRequired { + client_settlement_version, + node_settlement_version: CURRENT_SETTLEMENT_VERSION, + }) + } + } +} + /// ANT protocol handler. /// /// Handles chunk PUT/GET/Quote requests using LMDB storage for persistence @@ -238,13 +305,23 @@ impl AntProtocol { ChunkMessageBody::GetResponse(self.handle_get(req).await) } ChunkMessageBody::QuoteRequest(ref req) => { + Self::note_unversioned_quote("single_node"); ChunkMessageBody::QuoteResponse(self.handle_quote(req)) } ChunkMessageBody::MerkleCandidateQuoteRequest(ref req) => { + Self::note_unversioned_quote("merkle"); ChunkMessageBody::MerkleCandidateQuoteResponse( self.handle_merkle_candidate_quote(req), ) } + ChunkMessageBody::QuoteRequestV2(ref req) => { + ChunkMessageBody::QuoteResponse(self.handle_quote_v2(req)) + } + ChunkMessageBody::MerkleCandidateQuoteRequestV2(ref req) => { + ChunkMessageBody::MerkleCandidateQuoteResponse( + self.handle_merkle_candidate_quote_v2(req), + ) + } // Anything else — response messages are handled by client // subscribers (e.g. send_and_await_chunk_response), not by the // protocol handler. Returning None prevents the caller from @@ -609,6 +686,66 @@ impl AntProtocol { } } + /// Handle a version-declaring storage quote request. + /// + /// Refuse first, then fall through to the existing unversioned handler + /// with the same request. The gate is the only difference between the two + /// paths, so nothing else is duplicated and the two cannot drift. + fn handle_quote_v2(&self, request: &ChunkQuoteRequestV2) -> ChunkQuoteResponse { + if let Some(refusal) = settlement_gate(request.settlement_version, "single_node") { + return ChunkQuoteResponse::Error(refusal); + } + self.handle_quote(&ChunkQuoteRequest { + address: request.address, + data_size: request.data_size, + data_type: request.data_type, + }) + } + + /// Handle a version-declaring merkle candidate quote request. + /// + /// The gate matters most here: a merkle batch settles on-chain before any + /// storer sees a PUT, so this is the last point at which refusing costs + /// the client nothing. + fn handle_merkle_candidate_quote_v2( + &self, + request: &MerkleCandidateQuoteRequestV2, + ) -> MerkleCandidateQuoteResponse { + if let Some(refusal) = settlement_gate(request.settlement_version, "merkle") { + return MerkleCandidateQuoteResponse::Error(refusal); + } + self.handle_merkle_candidate_quote(&MerkleCandidateQuoteRequest { + address: request.address, + data_type: request.data_type, + data_size: request.data_size, + merkle_payment_timestamp: request.merkle_payment_timestamp, + }) + } + + /// Record that a client asked for a quote without declaring a settlement + /// version, and periodically report the running count. + /// + /// Unversioned requests are still served. A node cannot tell a client that + /// settles correctly but predates the version field from one that does + /// not, and refusing both would break clients that are behaving. What the + /// count buys is the evidence for flipping that policy later: once + /// unversioned traffic has decayed, refusing it stops turning away only + /// the clients that were already going to lose their money. + fn note_unversioned_quote(path: &str) { + let slot = usize::from(path == "merkle"); + let Some(counter) = UNVERSIONED_QUOTES_SERVED.get(slot) else { + return; + }; + let served = counter.fetch_add(1, Ordering::Relaxed).saturating_add(1); + if served % UNVERSIONED_QUOTE_LOG_INTERVAL == 0 { + info!( + target: "ant_node::quote::settlement", + "Served {served} {path} quotes to clients that declare no settlement version. \ + These clients cannot be told to upgrade before they pay.", + ); + } + } + /// Handle a merkle candidate quote request. fn handle_merkle_candidate_quote( &self, @@ -1232,6 +1369,193 @@ mod tests { } } + /// A current client gets a quote through the versioned request, exactly as + /// it would through the legacy one. The gate must be invisible to clients + /// that can pay. + #[tokio::test] + async fn v2_merkle_quote_is_served_at_the_current_settlement_version() { + let (protocol, _temp) = create_test_protocol().await; + let timestamp = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .expect("system time") + .as_secs(); + + let msg = ChunkMessage { + request_id: 700, + body: ChunkMessageBody::MerkleCandidateQuoteRequestV2( + MerkleCandidateQuoteRequestV2::new([0x88; 32], 4096, timestamp), + ), + }; + let response_bytes = protocol + .try_handle_request(&msg.encode().expect("encode request")) + .await + .expect("handle v2 merkle candidate quote") + .expect("expected response"); + let response = ChunkMessage::decode(&response_bytes).expect("decode response"); + + assert_eq!(response.request_id, 700); + match response.body { + ChunkMessageBody::MerkleCandidateQuoteResponse( + MerkleCandidateQuoteResponse::Success { .. }, + ) => {} + other => panic!("expected Success, got: {other:?}"), + } + } + + /// The point of the whole change: a client that cannot settle correctly is + /// turned away at quote time, so it never reaches the on-chain payment it + /// would not be able to spend. + #[tokio::test] + async fn v2_merkle_quote_is_refused_below_the_minimum_settlement_version() { + let (protocol, _temp) = create_test_protocol().await; + let timestamp = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .expect("system time") + .as_secs(); + + let stale = MIN_SUPPORTED_SETTLEMENT_VERSION.saturating_sub(1); + let mut request = MerkleCandidateQuoteRequestV2::new([0x99; 32], 4096, timestamp); + request.settlement_version = stale; + + let msg = ChunkMessage { + request_id: 701, + body: ChunkMessageBody::MerkleCandidateQuoteRequestV2(request), + }; + let response_bytes = protocol + .try_handle_request(&msg.encode().expect("encode request")) + .await + .expect("handle v2 merkle candidate quote") + .expect("expected response"); + let response = ChunkMessage::decode(&response_bytes).expect("decode response"); + + match response.body { + ChunkMessageBody::MerkleCandidateQuoteResponse( + MerkleCandidateQuoteResponse::Error(ProtocolError::ClientUpdateRequired { + client_settlement_version, + min_settlement_version, + }), + ) => { + assert_eq!(client_settlement_version, stale); + assert_eq!(min_settlement_version, MIN_SUPPORTED_SETTLEMENT_VERSION); + // The refusal has to be actionable, not just correct. + let rendered = ProtocolError::ClientUpdateRequired { + client_settlement_version, + min_settlement_version, + } + .to_string(); + assert!(rendered.contains("ant update"), "{rendered}"); + } + other => panic!("expected ClientUpdateRequired, got: {other:?}"), + } + } + + /// Same gate on the single-node path, so a refused merkle client cannot + /// simply fall back to per-chunk quotes and burn money that way instead. + #[tokio::test] + async fn v2_single_node_quote_is_refused_below_the_minimum_settlement_version() { + let (protocol, _temp) = create_test_protocol().await; + + let mut request = ChunkQuoteRequestV2::new([0xAA; 32], 4096); + request.settlement_version = MIN_SUPPORTED_SETTLEMENT_VERSION.saturating_sub(1); + + let msg = ChunkMessage { + request_id: 702, + body: ChunkMessageBody::QuoteRequestV2(request), + }; + let response_bytes = protocol + .try_handle_request(&msg.encode().expect("encode request")) + .await + .expect("handle v2 quote") + .expect("expected response"); + let response = ChunkMessage::decode(&response_bytes).expect("decode response"); + + match response.body { + ChunkMessageBody::QuoteResponse(ChunkQuoteResponse::Error( + ProtocolError::ClientUpdateRequired { .. }, + )) => {} + other => panic!("expected ClientUpdateRequired, got: {other:?}"), + } + } + + /// A settlement version this node has never heard of is refused, because + /// quoting it would promise to accept a payment whose rules the verifier + /// does not know. That promise would be broken at PUT time, which is after + /// the client has settled on-chain and can no longer be refunded. + /// + /// It must be refused as `StorerUpdateRequired`, not `ClientUpdateRequired`. + /// The client is fine; this node is behind. Telling an up-to-date user to + /// upgrade would be wrong, and during a client-first rollout it would be + /// wrong for most of the fleet at once. + #[tokio::test] + async fn a_newer_settlement_version_is_refused_as_this_nodes_fault() { + let (protocol, _temp) = create_test_protocol().await; + + let mut request = ChunkQuoteRequestV2::new([0xBB; 32], 4096); + request.settlement_version = CURRENT_SETTLEMENT_VERSION.saturating_add(1); + + let msg = ChunkMessage { + request_id: 703, + body: ChunkMessageBody::QuoteRequestV2(request), + }; + let response_bytes = protocol + .try_handle_request(&msg.encode().expect("encode request")) + .await + .expect("handle v2 quote") + .expect("expected response"); + let response = ChunkMessage::decode(&response_bytes).expect("decode response"); + + match response.body { + ChunkMessageBody::QuoteResponse(ChunkQuoteResponse::Error( + ProtocolError::StorerUpdateRequired { + client_settlement_version, + node_settlement_version, + }, + )) => { + assert_eq!( + client_settlement_version, + CURRENT_SETTLEMENT_VERSION.saturating_add(1) + ); + assert_eq!(node_settlement_version, CURRENT_SETTLEMENT_VERSION); + } + other => panic!("expected StorerUpdateRequired, got: {other:?}"), + } + } + + /// Legacy requests keep working. A node cannot tell a client that settles + /// correctly but predates the version field from one that does not, so + /// refusing both would break clients that are behaving. + #[tokio::test] + async fn unversioned_requests_are_still_served() { + let (protocol, _temp) = create_test_protocol().await; + let timestamp = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .expect("system time") + .as_secs(); + + let msg = ChunkMessage { + request_id: 704, + body: ChunkMessageBody::MerkleCandidateQuoteRequest(MerkleCandidateQuoteRequest { + address: [0xCC; 32], + data_type: DATA_TYPE_CHUNK, + data_size: 4096, + merkle_payment_timestamp: timestamp, + }), + }; + let response_bytes = protocol + .try_handle_request(&msg.encode().expect("encode request")) + .await + .expect("handle legacy merkle candidate quote") + .expect("expected response"); + let response = ChunkMessage::decode(&response_bytes).expect("decode response"); + + match response.body { + ChunkMessageBody::MerkleCandidateQuoteResponse( + MerkleCandidateQuoteResponse::Success { .. }, + ) => {} + other => panic!("expected Success for a legacy request, got: {other:?}"), + } + } + #[tokio::test] async fn test_handle_unexpected_response_message() { let (protocol, _temp) = create_test_protocol().await;