From 2f7a05ff6e7575e732c6b3ae09c9f3ada66beb97 Mon Sep 17 00:00:00 2001 From: grumbach Date: Thu, 13 Aug 2026 15:00:27 +0900 Subject: [PATCH 01/11] feat(quote): refuse quotes to clients that cannot settle correctly A merkle batch pays on-chain before any storer sees a PUT, so checking the settlement rule at PUT time checks it after the money is gone. Merkle receipts are not refundable, so every such rejection destroys a user's payment. Production is currently rejecting a steady trickle of uploads for an exact 3x underpayment, which is the signature of a client that predates the ADR-0008 multiplier and applies none at all. Handle the settlement-version quote requests and refuse any client below MIN_SUPPORTED_SETTLEMENT_VERSION with ClientUpdateRequired. No quote means no pool commitment, which means no payment, so a refused client has spent nothing. A version NEWER than this build understands is deliberately served: the storer still verifies whatever payment arrives, so nothing is weakened, while refusing would let a node that has not been upgraded veto a rule set the network has already moved to. Unversioned requests are still served. A node cannot distinguish a client that settles correctly but predates the version field from one that does not, so refusing both would break clients that are behaving. A running count of unversioned quotes is logged every 1000 per path, which is the evidence needed to decide when that policy can be flipped. Also extend the underpayment rejection itself. An exact multiplier shortfall is an outdated client rather than a pricing dispute, so the message now says so and tells the reader how to upgrade, noting the payment already made cannot be recovered. This is the only signal that reaches clients losing money today, because a client too old to settle correctly is also too old to declare a settlement version at quote time. The advice is keyed on an exact shortfall so a merely-cheap payment is not misreported as a stale client. Pins ant-protocol to the branch carrying the wire types while WithAutonomi/ant-protocol#23 is in review. --- Cargo.lock | 5 +- Cargo.toml | 5 +- src/ant_protocol/mod.rs | 8 +- src/payment/verifier.rs | 77 +++++++++- src/storage/handler.rs | 302 +++++++++++++++++++++++++++++++++++++++- 5 files changed, 385 insertions(+), 12 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 5f96ed94..6a644637 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -862,9 +862,8 @@ dependencies = [ [[package]] name = "ant-protocol" -version = "2.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "83f2b55e89a468584cfc91dc2f596583e089c17107de6f2f386f8339b665d1f0" +version = "2.4.0" +source = "git+https://github.com/grumbach/ant-protocol?branch=settlement-version-quote-gate#19845c8c20e1a3505cfbfc446b7e2c26bf5b0726" dependencies = [ "blake3", "bytes", diff --git a/Cargo.toml b/Cargo.toml index 2b99f0f1..c250cc70 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -39,7 +39,10 @@ mimalloc = "0.1" # Until then, the git pin tracks the matching saorsa-core lineage # (the rc-2026.4.2 branch) so Cargo can unify the wire types here # with ant-protocol's re-exports. -ant-protocol = "2.3.2" +# Branch pin while the settlement-version wire types are in review +# (WithAutonomi/ant-protocol#23). Swap back to `ant-protocol = "2.4.0"` once +# that PR merges and 2.4.0 is published. +ant-protocol = { git = "https://github.com/grumbach/ant-protocol", branch = "settlement-version-quote-gate" } # Core (provides EVERYTHING: networking, DHT, security, trust, storage) saorsa-core = "0.27.0" diff --git a/src/ant_protocol/mod.rs b/src/ant_protocol/mod.rs index ed01cf54..150e1eb2 100644 --- a/src/ant_protocol/mod.rs +++ b/src/ant_protocol/mod.rs @@ -17,9 +17,11 @@ pub use ::ant_protocol::chunk; pub use ::ant_protocol::chunk::{ - ChunkGetRequest, ChunkGetResponse, ChunkMessage, ChunkMessageBody, ChunkPutRequest, - ChunkPutResponse, ChunkQuoteRequest, ChunkQuoteResponse, MerkleCandidateQuoteRequest, + settlement_version_is_supported, ChunkGetRequest, ChunkGetResponse, ChunkMessage, + ChunkMessageBody, ChunkPutRequest, ChunkPutResponse, ChunkQuoteRequest, ChunkQuoteRequestV2, + ChunkQuoteResponse, MerkleCandidateQuoteRequest, MerkleCandidateQuoteRequestV2, MerkleCandidateQuoteResponse, ProtocolError, XorName, CHUNK_PROTOCOL_ID, CLOSE_GROUP_MAJORITY, - CLOSE_GROUP_SIZE, DATA_TYPE_CHUNK, MAX_CHUNK_SIZE, MAX_WIRE_MESSAGE_SIZE, PROOF_TAG_MERKLE, + CLOSE_GROUP_SIZE, CURRENT_SETTLEMENT_VERSION, DATA_TYPE_CHUNK, MAX_CHUNK_SIZE, + MAX_WIRE_MESSAGE_SIZE, MIN_SUPPORTED_SETTLEMENT_VERSION, PROOF_TAG_MERKLE, PROOF_TAG_SINGLE_NODE, PROTOCOL_VERSION, XORNAME_LEN, }; diff --git a/src/payment/verifier.rs b/src/payment/verifier.rs index fd550c77..da6cb75b 100644 --- a/src/payment/verifier.rs +++ b/src/payment/verifier.rs @@ -3363,11 +3363,33 @@ impl PaymentVerifier { ))); } if *paid_amount < expected_per_node { + // An exact `required_multiplier`x shortfall is the signature of + // a client that settles under the superseded rule: it applied + // no multiplier at all. That is not a pricing dispute, it is an + // outdated client, and the person reading this has already paid + // and cannot get it back. Say so, and say what to do about it, + // rather than leaving them with two integers to compare. + // + // This message is the only thing that reaches such a client. + // The settlement-version gate refuses these uploads before they + // pay, but only for clients new enough to declare a version, + // which by construction excludes every client this branch + // catches. + let looks_like_stale_client = expected_per_node + == paid_amount.saturating_mul(Amount::from(required_multiplier)); + let advice = if looks_like_stale_client { + " This is exactly the pre-parity settlement amount, so the paying client is \ + too old to settle correctly. Run `ant update` to upgrade, or reinstall from \ + https://github.com/WithAutonomi/ant-client/releases/latest. Note the payment \ + that was already made cannot be recovered." + } else { + "" + }; return Err(Error::Payment(format!( "Underpayment for node at index {idx}: paid {paid_amount}, \ expected at least {expected_per_node} \ (median16 formula, depth={}, {required_multiplier}x required for a \ - receipt stamped {} vs parity boundary {parity_from})", + receipt stamped {} vs parity boundary {parity_from}).{advice}", payment_info.depth, payment_info.merkle_payment_timestamp ))); } @@ -7040,6 +7062,59 @@ mod tests { err_msg.contains("Underpayment") && err_msg.contains("3x required"), "Error should name the required multiplier: {err_msg}" ); + // An exact 1x settlement is an outdated client, not a pricing dispute. + // The payer has already spent money they cannot recover, so the + // rejection has to tell them what to do rather than hand them two + // integers. This is the only message that reaches a client too old to + // declare a settlement version at quote time. + assert!( + err_msg.contains("too old to settle correctly") && err_msg.contains("ant update"), + "Error should tell the user to upgrade: {err_msg}" + ); + } + + /// The upgrade advice is keyed on an EXACT multiplier shortfall, which is + /// what an unmultiplied settlement looks like. A merely-cheap payment is a + /// different fault and must not be blamed on the client's version, or the + /// advice stops meaning anything. + #[tokio::test] + async fn a_partial_shortfall_is_not_blamed_on_an_outdated_client() { + let verifier = merkle_test_verifier(); + let (xorname, tagged_proof, pool_hash, ts) = make_valid_merkle_proof_bytes(); + + // One wei under parity: short, but not the 1x signature. + let almost = merkle_parity_per_node_depth2().saturating_sub(Amount::from(1u64)); + { + let info = evmlib::merkle_payments::OnChainPaymentInfo { + depth: 2, + merkle_payment_timestamp: ts, + paid_node_addresses: vec![ + (RewardsAddress::new([0u8; 20]), 0, almost), + (RewardsAddress::new([1u8; 20]), 1, almost), + ], + }; + verifier.pool_cache.lock().put(pool_hash, info); + } + + let err_msg = format!( + "{}", + verifier + .verify_payment( + &xorname, + Some(&tagged_proof), + VerificationContext::ClientPut + ) + .await + .expect_err("a short settlement must be refused") + ); + assert!( + err_msg.contains("Underpayment"), + "Error should still name the underpayment: {err_msg}" + ); + assert!( + !err_msg.contains("ant update"), + "A partial shortfall must not be reported as an outdated client: {err_msg}" + ); } /// A receipt stamped BEFORE the boundary keeps its 1x price. The money was diff --git a/src/storage/handler.rs b/src/storage/handler.rs index 08a40754..741f94d4 100644 --- a/src/storage/handler.rs +++ b/src/storage/handler.rs @@ -27,13 +27,15 @@ //! └─────────────────────────────────────────────────────────┘ //! ``` -#[cfg(test)] -use crate::ant_protocol::DATA_TYPE_CHUNK; use crate::ant_protocol::{ - ChunkGetRequest, ChunkGetResponse, ChunkMessage, ChunkMessageBody, ChunkPutRequest, - ChunkPutResponse, ChunkQuoteRequest, ChunkQuoteResponse, MerkleCandidateQuoteRequest, + settlement_version_is_supported, ChunkGetRequest, ChunkGetResponse, ChunkMessage, + ChunkMessageBody, ChunkPutRequest, ChunkPutResponse, ChunkQuoteRequest, ChunkQuoteRequestV2, + ChunkQuoteResponse, MerkleCandidateQuoteRequest, MerkleCandidateQuoteRequestV2, MerkleCandidateQuoteResponse, ProtocolError, CHUNK_PROTOCOL_ID, MAX_CHUNK_SIZE, + MIN_SUPPORTED_SETTLEMENT_VERSION, }; +#[cfg(test)] +use crate::ant_protocol::{CURRENT_SETTLEMENT_VERSION, DATA_TYPE_CHUNK}; use crate::client::compute_address; use crate::error::{Error, Result}; use crate::logging::{debug, info, warn}; @@ -45,6 +47,7 @@ use crate::storage::lmdb::LmdbStorage; use bytes::Bytes; use parking_lot::RwLock; use saorsa_core::P2PNode; +use std::sync::atomic::{AtomicU64, Ordering}; use std::sync::Arc; use tokio::sync::mpsc; @@ -59,6 +62,51 @@ use tokio::sync::mpsc; /// turned away. const SELF_CLOSENESS_GATE_WIDTH: usize = K_BUCKET_SIZE; +/// How many unversioned quote requests to serve between adoption log lines. +/// +/// One line per request would drown the log at production quote rates, and one +/// line total would say nothing about the trend. A running count emitted every +/// `N` gives the shape of client adoption, which is the number that decides +/// when unversioned requests can start being refused outright. +const UNVERSIONED_QUOTE_LOG_INTERVAL: u64 = 1_000; + +/// Unversioned quote requests served since start, by path. +/// +/// Indices are `[single_node, merkle]`. A plain counter rather than a metric +/// because the only consumer is the rollout decision, and that reads logs. +static UNVERSIONED_QUOTES_SERVED: [AtomicU64; 2] = [AtomicU64::new(0), AtomicU64::new(0)]; + +/// Refuse a quote when the requesting client settles under rules this node no +/// longer accepts. `None` means the request may proceed. +/// +/// This is the whole point of the settlement version. A merkle batch pays +/// on-chain **before** any storer sees a PUT, so refusing the *payment* is +/// refusing something already spent and unrefundable. Refusing the *quote* +/// costs the client nothing: no quote means no pool commitment, which means no +/// payment. +/// +/// A version NEWER than this node understands is deliberately allowed through. +/// The storer still verifies whatever payment actually arrives, so nothing is +/// weakened by letting it past, whereas rejecting it would let a stale node +/// veto a rule set the network has already moved to. +fn settlement_gate(client_settlement_version: u32, path: &str) -> Option { + if settlement_version_is_supported(client_settlement_version) { + return None; + } + + warn!( + target: "ant_node::quote::settlement", + "Refusing {path} quote: client settlement version {client_settlement_version} \ + is below the minimum {MIN_SUPPORTED_SETTLEMENT_VERSION}. No quote issued, \ + so the client has not been charged.", + ); + + Some(ProtocolError::ClientUpdateRequired { + client_settlement_version, + min_settlement_version: MIN_SUPPORTED_SETTLEMENT_VERSION, + }) +} + /// ANT protocol handler. /// /// Handles chunk PUT/GET/Quote requests using LMDB storage for persistence @@ -238,13 +286,23 @@ impl AntProtocol { ChunkMessageBody::GetResponse(self.handle_get(req).await) } ChunkMessageBody::QuoteRequest(ref req) => { + Self::note_unversioned_quote("single_node"); ChunkMessageBody::QuoteResponse(self.handle_quote(req)) } ChunkMessageBody::MerkleCandidateQuoteRequest(ref req) => { + Self::note_unversioned_quote("merkle"); ChunkMessageBody::MerkleCandidateQuoteResponse( self.handle_merkle_candidate_quote(req), ) } + ChunkMessageBody::QuoteRequestV2(ref req) => { + ChunkMessageBody::QuoteResponse(self.handle_quote_v2(req)) + } + ChunkMessageBody::MerkleCandidateQuoteRequestV2(ref req) => { + ChunkMessageBody::MerkleCandidateQuoteResponse( + self.handle_merkle_candidate_quote_v2(req), + ) + } // Anything else — response messages are handled by client // subscribers (e.g. send_and_await_chunk_response), not by the // protocol handler. Returning None prevents the caller from @@ -609,6 +667,66 @@ impl AntProtocol { } } + /// Handle a version-declaring storage quote request. + /// + /// Refuse first, then fall through to the existing unversioned handler + /// with the same request. The gate is the only difference between the two + /// paths, so nothing else is duplicated and the two cannot drift. + fn handle_quote_v2(&self, request: &ChunkQuoteRequestV2) -> ChunkQuoteResponse { + if let Some(refusal) = settlement_gate(request.settlement_version, "single_node") { + return ChunkQuoteResponse::Error(refusal); + } + self.handle_quote(&ChunkQuoteRequest { + address: request.address, + data_size: request.data_size, + data_type: request.data_type, + }) + } + + /// Handle a version-declaring merkle candidate quote request. + /// + /// The gate matters most here: a merkle batch settles on-chain before any + /// storer sees a PUT, so this is the last point at which refusing costs + /// the client nothing. + fn handle_merkle_candidate_quote_v2( + &self, + request: &MerkleCandidateQuoteRequestV2, + ) -> MerkleCandidateQuoteResponse { + if let Some(refusal) = settlement_gate(request.settlement_version, "merkle") { + return MerkleCandidateQuoteResponse::Error(refusal); + } + self.handle_merkle_candidate_quote(&MerkleCandidateQuoteRequest { + address: request.address, + data_type: request.data_type, + data_size: request.data_size, + merkle_payment_timestamp: request.merkle_payment_timestamp, + }) + } + + /// Record that a client asked for a quote without declaring a settlement + /// version, and periodically report the running count. + /// + /// Unversioned requests are still served. A node cannot tell a client that + /// settles correctly but predates the version field from one that does + /// not, and refusing both would break clients that are behaving. What the + /// count buys is the evidence for flipping that policy later: once + /// unversioned traffic has decayed, refusing it stops turning away only + /// the clients that were already going to lose their money. + fn note_unversioned_quote(path: &str) { + let slot = usize::from(path == "merkle"); + let Some(counter) = UNVERSIONED_QUOTES_SERVED.get(slot) else { + return; + }; + let served = counter.fetch_add(1, Ordering::Relaxed).saturating_add(1); + if served % UNVERSIONED_QUOTE_LOG_INTERVAL == 0 { + info!( + target: "ant_node::quote::settlement", + "Served {served} {path} quotes to clients that declare no settlement version. \ + These clients cannot be told to upgrade before they pay.", + ); + } + } + /// Handle a merkle candidate quote request. fn handle_merkle_candidate_quote( &self, @@ -1232,6 +1350,182 @@ mod tests { } } + /// A current client gets a quote through the versioned request, exactly as + /// it would through the legacy one. The gate must be invisible to clients + /// that can pay. + #[tokio::test] + async fn v2_merkle_quote_is_served_at_the_current_settlement_version() { + let (protocol, _temp) = create_test_protocol().await; + let timestamp = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .expect("system time") + .as_secs(); + + let msg = ChunkMessage { + request_id: 700, + body: ChunkMessageBody::MerkleCandidateQuoteRequestV2( + MerkleCandidateQuoteRequestV2::new([0x88; 32], 4096, timestamp), + ), + }; + let response_bytes = protocol + .try_handle_request(&msg.encode().expect("encode request")) + .await + .expect("handle v2 merkle candidate quote") + .expect("expected response"); + let response = ChunkMessage::decode(&response_bytes).expect("decode response"); + + assert_eq!(response.request_id, 700); + match response.body { + ChunkMessageBody::MerkleCandidateQuoteResponse( + MerkleCandidateQuoteResponse::Success { .. }, + ) => {} + other => panic!("expected Success, got: {other:?}"), + } + } + + /// The point of the whole change: a client that cannot settle correctly is + /// turned away at quote time, so it never reaches the on-chain payment it + /// would not be able to spend. + #[tokio::test] + async fn v2_merkle_quote_is_refused_below_the_minimum_settlement_version() { + let (protocol, _temp) = create_test_protocol().await; + let timestamp = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .expect("system time") + .as_secs(); + + let stale = MIN_SUPPORTED_SETTLEMENT_VERSION.saturating_sub(1); + let mut request = MerkleCandidateQuoteRequestV2::new([0x99; 32], 4096, timestamp); + request.settlement_version = stale; + + let msg = ChunkMessage { + request_id: 701, + body: ChunkMessageBody::MerkleCandidateQuoteRequestV2(request), + }; + let response_bytes = protocol + .try_handle_request(&msg.encode().expect("encode request")) + .await + .expect("handle v2 merkle candidate quote") + .expect("expected response"); + let response = ChunkMessage::decode(&response_bytes).expect("decode response"); + + match response.body { + ChunkMessageBody::MerkleCandidateQuoteResponse( + MerkleCandidateQuoteResponse::Error(ProtocolError::ClientUpdateRequired { + client_settlement_version, + min_settlement_version, + }), + ) => { + assert_eq!(client_settlement_version, stale); + assert_eq!(min_settlement_version, MIN_SUPPORTED_SETTLEMENT_VERSION); + // The refusal has to be actionable, not just correct. + let rendered = ProtocolError::ClientUpdateRequired { + client_settlement_version, + min_settlement_version, + } + .to_string(); + assert!(rendered.contains("ant update"), "{rendered}"); + } + other => panic!("expected ClientUpdateRequired, got: {other:?}"), + } + } + + /// Same gate on the single-node path, so a refused merkle client cannot + /// simply fall back to per-chunk quotes and burn money that way instead. + #[tokio::test] + async fn v2_single_node_quote_is_refused_below_the_minimum_settlement_version() { + let (protocol, _temp) = create_test_protocol().await; + + let mut request = ChunkQuoteRequestV2::new([0xAA; 32], 4096); + request.settlement_version = MIN_SUPPORTED_SETTLEMENT_VERSION.saturating_sub(1); + + let msg = ChunkMessage { + request_id: 702, + body: ChunkMessageBody::QuoteRequestV2(request), + }; + let response_bytes = protocol + .try_handle_request(&msg.encode().expect("encode request")) + .await + .expect("handle v2 quote") + .expect("expected response"); + let response = ChunkMessage::decode(&response_bytes).expect("decode response"); + + match response.body { + ChunkMessageBody::QuoteResponse(ChunkQuoteResponse::Error( + ProtocolError::ClientUpdateRequired { .. }, + )) => {} + other => panic!("expected ClientUpdateRequired, got: {other:?}"), + } + } + + /// A settlement version this build has never heard of is served, not + /// refused. The storer still verifies the payment that actually arrives, + /// so nothing is weakened, and refusing would let a node that has not been + /// upgraded veto a rule set the network has already moved to. + #[tokio::test] + async fn a_newer_settlement_version_is_not_treated_as_an_error() { + let (protocol, _temp) = create_test_protocol().await; + + let mut request = ChunkQuoteRequestV2::new([0xBB; 32], 4096); + request.settlement_version = CURRENT_SETTLEMENT_VERSION.saturating_add(1); + + let msg = ChunkMessage { + request_id: 703, + body: ChunkMessageBody::QuoteRequestV2(request), + }; + let response_bytes = protocol + .try_handle_request(&msg.encode().expect("encode request")) + .await + .expect("handle v2 quote") + .expect("expected response"); + let response = ChunkMessage::decode(&response_bytes).expect("decode response"); + + match response.body { + ChunkMessageBody::QuoteResponse(ChunkQuoteResponse::Error( + ProtocolError::ClientUpdateRequired { .. }, + )) => { + panic!("a newer client must not be refused by an older node") + } + ChunkMessageBody::QuoteResponse(_) => {} + other => panic!("expected QuoteResponse, got: {other:?}"), + } + } + + /// Legacy requests keep working. A node cannot tell a client that settles + /// correctly but predates the version field from one that does not, so + /// refusing both would break clients that are behaving. + #[tokio::test] + async fn unversioned_requests_are_still_served() { + let (protocol, _temp) = create_test_protocol().await; + let timestamp = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .expect("system time") + .as_secs(); + + let msg = ChunkMessage { + request_id: 704, + body: ChunkMessageBody::MerkleCandidateQuoteRequest(MerkleCandidateQuoteRequest { + address: [0xCC; 32], + data_type: DATA_TYPE_CHUNK, + data_size: 4096, + merkle_payment_timestamp: timestamp, + }), + }; + let response_bytes = protocol + .try_handle_request(&msg.encode().expect("encode request")) + .await + .expect("handle legacy merkle candidate quote") + .expect("expected response"); + let response = ChunkMessage::decode(&response_bytes).expect("decode response"); + + match response.body { + ChunkMessageBody::MerkleCandidateQuoteResponse( + MerkleCandidateQuoteResponse::Success { .. }, + ) => {} + other => panic!("expected Success for a legacy request, got: {other:?}"), + } + } + #[tokio::test] async fn test_handle_unexpected_response_message() { let (protocol, _temp) = create_test_protocol().await; From 9bc8760db2ef881cd947881513433ab47d49172e Mon Sep 17 00:00:00 2001 From: grumbach Date: Thu, 13 Aug 2026 15:55:41 +0900 Subject: [PATCH 02/11] fix(quote): refuse settlement versions this node cannot verify Review raised that inheriting "any version at or above the minimum" lets an older node quote an arbitrarily newer client, which forfeits the pre-payment guarantee the gate exists for. PUT-time verification is too late: by then the client has settled on-chain and cannot be refunded. Refuse both directions, using the bounded settlement_compatibility check. A version above this node's own is answered with StorerUpdateRequired rather than ClientUpdateRequired, because the client is fine and this node is the one behind. The client should route to another storer and tell its user nothing. Telling an up-to-date user to upgrade would be wrong, and during the client-first rollout ADR-0008 prescribes it would be wrong for most of the fleet at once. The test that pinned the previous behaviour is inverted rather than deleted, so the corrected policy is the one under regression cover. --- Cargo.lock | 12 +++--- src/ant_protocol/mod.rs | 8 ++-- src/storage/handler.rs | 88 +++++++++++++++++++++++++++-------------- 3 files changed, 68 insertions(+), 40 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 6a644637..11b062d8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -863,7 +863,7 @@ dependencies = [ [[package]] name = "ant-protocol" version = "2.4.0" -source = "git+https://github.com/grumbach/ant-protocol?branch=settlement-version-quote-gate#19845c8c20e1a3505cfbfc446b7e2c26bf5b0726" +source = "git+https://github.com/grumbach/ant-protocol?branch=settlement-version-quote-gate#620bb9971ba5669bdb23a38568f8d3a597987db1" dependencies = [ "blake3", "bytes", @@ -3105,7 +3105,7 @@ dependencies = [ "libc", "percent-encoding", "pin-project-lite", - "socket2 0.5.10", + "socket2 0.6.4", "tokio", "tower-service", "tracing", @@ -4292,7 +4292,7 @@ dependencies = [ "quinn-udp 0.5.15", "rustc-hash", "rustls", - "socket2 0.5.10", + "socket2 0.6.4", "thiserror 2.0.18", "tokio", "tracing", @@ -4331,7 +4331,7 @@ dependencies = [ "cfg_aliases", "libc", "once_cell", - "socket2 0.5.10", + "socket2 0.6.4", "tracing", "windows-sys 0.61.2", ] @@ -4344,7 +4344,7 @@ checksum = "76150b617afc75e6e21ac5f39bc196e80b65415ae48d62dbef8e2519d040ce42" dependencies = [ "cfg_aliases", "libc", - "socket2 0.5.10", + "socket2 0.6.4", "tracing", "windows-sys 0.61.2", ] @@ -6417,7 +6417,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.48.0", + "windows-sys 0.61.2", ] [[package]] diff --git a/src/ant_protocol/mod.rs b/src/ant_protocol/mod.rs index 150e1eb2..74921117 100644 --- a/src/ant_protocol/mod.rs +++ b/src/ant_protocol/mod.rs @@ -17,10 +17,10 @@ pub use ::ant_protocol::chunk; pub use ::ant_protocol::chunk::{ - settlement_version_is_supported, ChunkGetRequest, ChunkGetResponse, ChunkMessage, - ChunkMessageBody, ChunkPutRequest, ChunkPutResponse, ChunkQuoteRequest, ChunkQuoteRequestV2, - ChunkQuoteResponse, MerkleCandidateQuoteRequest, MerkleCandidateQuoteRequestV2, - MerkleCandidateQuoteResponse, ProtocolError, XorName, CHUNK_PROTOCOL_ID, CLOSE_GROUP_MAJORITY, + settlement_compatibility, ChunkGetRequest, ChunkGetResponse, ChunkMessage, ChunkMessageBody, + ChunkPutRequest, ChunkPutResponse, ChunkQuoteRequest, ChunkQuoteRequestV2, ChunkQuoteResponse, + MerkleCandidateQuoteRequest, MerkleCandidateQuoteRequestV2, MerkleCandidateQuoteResponse, + ProtocolError, SettlementCompatibility, XorName, CHUNK_PROTOCOL_ID, CLOSE_GROUP_MAJORITY, CLOSE_GROUP_SIZE, CURRENT_SETTLEMENT_VERSION, DATA_TYPE_CHUNK, MAX_CHUNK_SIZE, MAX_WIRE_MESSAGE_SIZE, MIN_SUPPORTED_SETTLEMENT_VERSION, PROOF_TAG_MERKLE, PROOF_TAG_SINGLE_NODE, PROTOCOL_VERSION, XORNAME_LEN, diff --git a/src/storage/handler.rs b/src/storage/handler.rs index 741f94d4..22c0e15a 100644 --- a/src/storage/handler.rs +++ b/src/storage/handler.rs @@ -27,15 +27,15 @@ //! └─────────────────────────────────────────────────────────┘ //! ``` +#[cfg(test)] +use crate::ant_protocol::DATA_TYPE_CHUNK; use crate::ant_protocol::{ - settlement_version_is_supported, ChunkGetRequest, ChunkGetResponse, ChunkMessage, - ChunkMessageBody, ChunkPutRequest, ChunkPutResponse, ChunkQuoteRequest, ChunkQuoteRequestV2, - ChunkQuoteResponse, MerkleCandidateQuoteRequest, MerkleCandidateQuoteRequestV2, - MerkleCandidateQuoteResponse, ProtocolError, CHUNK_PROTOCOL_ID, MAX_CHUNK_SIZE, - MIN_SUPPORTED_SETTLEMENT_VERSION, + settlement_compatibility, ChunkGetRequest, ChunkGetResponse, ChunkMessage, ChunkMessageBody, + ChunkPutRequest, ChunkPutResponse, ChunkQuoteRequest, ChunkQuoteRequestV2, ChunkQuoteResponse, + MerkleCandidateQuoteRequest, MerkleCandidateQuoteRequestV2, MerkleCandidateQuoteResponse, + ProtocolError, SettlementCompatibility, CHUNK_PROTOCOL_ID, CURRENT_SETTLEMENT_VERSION, + MAX_CHUNK_SIZE, MIN_SUPPORTED_SETTLEMENT_VERSION, }; -#[cfg(test)] -use crate::ant_protocol::{CURRENT_SETTLEMENT_VERSION, DATA_TYPE_CHUNK}; use crate::client::compute_address; use crate::error::{Error, Result}; use crate::logging::{debug, info, warn}; @@ -90,21 +90,38 @@ static UNVERSIONED_QUOTES_SERVED: [AtomicU64; 2] = [AtomicU64::new(0), AtomicU64 /// weakened by letting it past, whereas rejecting it would let a stale node /// veto a rule set the network has already moved to. fn settlement_gate(client_settlement_version: u32, path: &str) -> Option { - if settlement_version_is_supported(client_settlement_version) { - return None; + match settlement_compatibility(client_settlement_version) { + SettlementCompatibility::Compatible => None, + SettlementCompatibility::ClientTooOld => { + warn!( + target: "ant_node::quote::settlement", + "Refusing {path} quote: client settlement version {client_settlement_version} \ + is below the minimum {MIN_SUPPORTED_SETTLEMENT_VERSION}. No quote issued, \ + so the client has not been charged.", + ); + Some(ProtocolError::ClientUpdateRequired { + client_settlement_version, + min_settlement_version: MIN_SUPPORTED_SETTLEMENT_VERSION, + }) + } + // This node is the old one. Quoting would promise to accept a payment + // whose rules it does not know, and a promise broken at PUT time is + // broken after the client has settled on-chain. Refusing sends the + // client to a peer that can actually honour the quote, at no cost. + SettlementCompatibility::NodeTooOld => { + warn!( + target: "ant_node::quote::settlement", + "Refusing {path} quote: client settles under version \ + {client_settlement_version}, newer than this node's \ + {CURRENT_SETTLEMENT_VERSION}. This node needs upgrading; the client \ + has not been charged.", + ); + Some(ProtocolError::StorerUpdateRequired { + client_settlement_version, + node_settlement_version: CURRENT_SETTLEMENT_VERSION, + }) + } } - - warn!( - target: "ant_node::quote::settlement", - "Refusing {path} quote: client settlement version {client_settlement_version} \ - is below the minimum {MIN_SUPPORTED_SETTLEMENT_VERSION}. No quote issued, \ - so the client has not been charged.", - ); - - Some(ProtocolError::ClientUpdateRequired { - client_settlement_version, - min_settlement_version: MIN_SUPPORTED_SETTLEMENT_VERSION, - }) } /// ANT protocol handler. @@ -1458,12 +1475,17 @@ mod tests { } } - /// A settlement version this build has never heard of is served, not - /// refused. The storer still verifies the payment that actually arrives, - /// so nothing is weakened, and refusing would let a node that has not been - /// upgraded veto a rule set the network has already moved to. + /// A settlement version this node has never heard of is refused, because + /// quoting it would promise to accept a payment whose rules the verifier + /// does not know. That promise would be broken at PUT time, which is after + /// the client has settled on-chain and can no longer be refunded. + /// + /// It must be refused as `StorerUpdateRequired`, not `ClientUpdateRequired`. + /// The client is fine; this node is behind. Telling an up-to-date user to + /// upgrade would be wrong, and during a client-first rollout it would be + /// wrong for most of the fleet at once. #[tokio::test] - async fn a_newer_settlement_version_is_not_treated_as_an_error() { + async fn a_newer_settlement_version_is_refused_as_this_nodes_fault() { let (protocol, _temp) = create_test_protocol().await; let mut request = ChunkQuoteRequestV2::new([0xBB; 32], 4096); @@ -1482,12 +1504,18 @@ mod tests { match response.body { ChunkMessageBody::QuoteResponse(ChunkQuoteResponse::Error( - ProtocolError::ClientUpdateRequired { .. }, + ProtocolError::StorerUpdateRequired { + client_settlement_version, + node_settlement_version, + }, )) => { - panic!("a newer client must not be refused by an older node") + assert_eq!( + client_settlement_version, + CURRENT_SETTLEMENT_VERSION.saturating_add(1) + ); + assert_eq!(node_settlement_version, CURRENT_SETTLEMENT_VERSION); } - ChunkMessageBody::QuoteResponse(_) => {} - other => panic!("expected QuoteResponse, got: {other:?}"), + other => panic!("expected StorerUpdateRequired, got: {other:?}"), } } From 671235cba2214e61568a60b10dfbb7c5c84fbd2c Mon Sep 17 00:00:00 2001 From: grumbach Date: Thu, 13 Aug 2026 16:01:46 +0900 Subject: [PATCH 03/11] docs(adr): record the settlement-version compatibility policy Captures the decision the coordinated protocol/node/client change implements, as requested at review: the inclusive MIN..=CURRENT range and why the upper bound is load-bearing, the two refusal directions and why they must stay distinct during a client-first rollout, the unversioned retry as a bounded downgrade path with a compile-time cutover rule, and the residual that merkle storers are not exactly the peers a client quotes. Also records what is not fixed: the clients burning money today are too old to declare a version, so only the reworded error reaches them. --- ...t-version-and-pre-payment-compatibility.md | 122 ++++++++++++++++++ 1 file changed, 122 insertions(+) create mode 100644 docs/adr/ADR-0010-settlement-version-and-pre-payment-compatibility.md diff --git a/docs/adr/ADR-0010-settlement-version-and-pre-payment-compatibility.md b/docs/adr/ADR-0010-settlement-version-and-pre-payment-compatibility.md new file mode 100644 index 00000000..dfcb4b48 --- /dev/null +++ b/docs/adr/ADR-0010-settlement-version-and-pre-payment-compatibility.md @@ -0,0 +1,122 @@ +# ADR-0010: Settlement version and pre-payment compatibility + +- **Status:** Proposed +- **Date:** 2026-08-13 +- **Decision owners:** Anselme Grumbach +- **Reviewers:** David Irvine +- **Supersedes:** none +- **Superseded by:** none +- **Related:** ADR-0008 (storage economics and payment protocol); ant-protocol#23; ant-node#204; ant-client#171 + +## Context + +ADR-0008 raised the merkle settlement multiplier to 3x. It changed client code and node code and **changed no wire type**, so nothing tied the two together. A client built before the change could still collect quotes, still pay, and only then discover that every storer refused the upload. + +That ordering is what makes it expensive. A merkle batch settles on-chain **before any storer sees a PUT**, and merkle receipts are not refundable. So the check that mattered ran after the money was gone. Production has been rejecting a slow trickle of such uploads at an exact 3x shortfall, each one a user's payment destroyed, reported to them as two integers to compare. + +ADR-0008 anticipated this and listed it as a re-open trigger: *"a rise in refused batch uploads after the boundary, indicating clients that never upgraded."* This ADR is that trigger being answered. + +The general problem is that **`PROTOCOL_VERSION` describes what a peer can parse, and nothing described how a peer settles**. Those two move independently. Every future change to settlement arithmetic reproduces this failure unless something carries the second fact. + +## Decision Drivers + +- A refusal must land **before** payment. Merkle settlement is irreversible, so after-the-fact verification cannot be the only check. +- A client-first rollout must stay possible. ADR-0008 chose client-first deliberately, because a client paying more is accepted by an old node for free. Any policy that makes old nodes refuse new clients breaks that ordering. +- Old peers must not be misread. `ChunkMessage` is postcard-encoded and non-self-describing, so a new field silently changes how existing peers parse every message. +- The user must be told what to do. The failure is only expensive because it is silent. + +## Considered Options + +1. **Bump `CHUNK_PROTOCOL_ID` to v3.** The established mechanism for wire changes here (ADR-0004, ADR-0005). Clean end state, but a hard cutover: old peers cannot negotiate a stream at all, so they get a handshake failure rather than a diagnosis, and the whole fleet plus all clients must move together. +2. **Append versioned request variants.** No cutover. Existing discriminants are untouched, unknown variants are rejected cleanly, and nodes and clients can move independently. +3. **Move the multiplier into the contract.** Client submits the 1x price, the contract applies the multiplier. This removes the payer's ability to get it wrong at all, and is the only option that protects clients which never upgrade. Requires a contract upgrade. +4. **Do nothing; drive client adoption.** Cheapest, and closes the *current* burn faster than any of the above. Does nothing for the next settlement change. + +## Decision + +We will adopt **option 2**, and treat **option 3 as the eventual structural fix** rather than a competing one. + +A `settlement_version` travels in the quote request. It is a separate constant from `PROTOCOL_VERSION` and is bumped whenever a change makes an older client pay an amount storers will refuse. + +### The compatibility rule + +A storer quotes only when the client's version falls inside its own inclusive range: + +``` +MIN_SUPPORTED_SETTLEMENT_VERSION <= client_version <= CURRENT_SETTLEMENT_VERSION +``` + +**Both ends are bounded, and the upper bound is the part that is easy to get wrong.** An earlier revision of this work accepted everything at or above the minimum, reasoning that a storer verifies whatever payment actually arrives, so letting a newer client through weakens nothing. That is true only when a settlement change *raises* what is paid. ADR-0008's 3x cleared an old node's 1x minimum, so old nodes accepted new clients for free, and that special case was mistaken for the general rule. A change that redefines the median rule, or which field the contract pays from, produces a payment an older verifier rejects, after the client has already settled. + +### Two refusals, not one + +The two out-of-range directions need opposite handling and are therefore separate wire variants. + +| Condition | Wire error | Whose problem | Client behaviour | +|---|---|---|---| +| `client_version < MIN` | `ClientUpdateRequired` | the client | **Terminal.** Abort before payment, show the upgrade instruction. | +| `client_version > CURRENT` | `StorerUpdateRequired` | this node | **Skip the peer.** Say nothing to the user; use other storers. | + +Collapsing them would either tell up-to-date users to upgrade, or strand new clients whenever the node fleet lags. A lagging fleet is the *normal* state during the client-first rollout ADR-0008 prescribes, so this distinction is what keeps the two ADRs compatible. + +If too few peers remain after skipping, the operation fails for lack of quotes. That is the correct outcome: it fails before payment rather than after. + +### The legacy fallback, and the rule that retires it + +A storer built before the versioned requests cannot decode them and simply never answers. Clients therefore retry each silent peer once in the legacy shape, or nothing would work until the whole fleet had upgraded. + +**This is a downgrade path.** Silence is not proof a peer cannot parse the request: a dropped response, packet loss, an overloaded peer, and one deliberately discarding versioned requests are indistinguishable from the client side. So the retry can be provoked, and a provoked retry bypasses the gate. + +It is safe **only while no client can be refused on version grounds**, which holds exactly while `MIN_SUPPORTED_SETTLEMENT_VERSION` is the first declarable version. The rule is therefore: + +> The unversioned retry must be deleted **before** `MIN_SUPPORTED_SETTLEMENT_VERSION` is ever raised. + +This is enforced by a compile-time assertion in the client, not by review discipline: raising the minimum while the fallback exists fails the build. + +### Unversioned requests are still served + +A storer cannot distinguish a client that settles correctly but predates the version field (ant-core 0.5.1 through 0.6.0) from one that does not. Refusing both would break clients that are behaving, so unversioned requests are served and counted. Nodes log a running total per path under `ant_node::quote::settlement`. + +Flipping that to a refusal is a **follow-up**, gated on that count decaying, and should use a dated self-retiring boundary in the style of `MERKLE_PARITY_ENFORCED_FROM_UNIX`. + +## Consequences + +### Positive + +- A settlement change can no longer burn an outdated client's money. The refusal lands at quote time, where it costs nothing. +- The user is told what happened and what to do, in both the quote refusal and the PUT-time underpayment message. +- No protocol cutover. Nodes and clients roll independently. +- The wire discriminants of every pre-existing message and error are pinned by a regression test, so the append-only property this rests on is enforced rather than assumed. + +### Negative / Trade-offs + +- **It does not fix the current burn.** A client too old to settle correctly is also too old to declare a version, so the gate cannot see the population causing today's rejections. Only the reworded error reaches them. Driving client adoption remains the cheaper and faster remedy for the incident that prompted this. +- **Merkle storers are not exactly the quoted peers.** The gate covers the 16 candidates a client quotes, but the chunk is stored by each chunk's close group, which may include a peer that never quoted. A storer outside the candidate set can still refuse at PUT time. This narrows the exposure substantially without closing it, and only option 3 closes it fully. +- The fallback costs one extra timeout per silent peer during rollout. Requests run concurrently, so the worst case is 2x the quote timeout overall. +- Clients declaring a version are refused by nodes that have not upgraded. Harmless today because no node has a lower `CURRENT`, but it makes node rollout a prerequisite for the next settlement bump. + +### Neutral / Operational + +- Deploy order is **nodes before clients**, since a node on `ant-protocol` 2.3.x cannot decode versioned requests. The fallback makes this a preference rather than a hard gate. +- The unversioned-quote counter is the input to the follow-up decision above. +- `ant-protocol` moves 2.3.2 -> 2.4.0. Additive, minor. + +## Validation + +- **Wire safety.** `appending_v2_variants_leaves_existing_discriminants_untouched` and `client_update_required_is_appended_to_protocol_error` pin the discriminant of every pre-existing variant. If either fails, older peers are misreading current traffic. +- **Range policy.** `a_newer_settlement_version_is_refused_rather_than_assumed_compatible` pins the upper bound, which is the specific error this ADR corrects. +- **Refusal direction.** `a_newer_settlement_version_is_refused_as_this_nodes_fault` and `the_two_refusals_do_not_blame_the_same_party` pin that a lagging node never reports a client fault. +- **Terminality.** `a_refusal_aborts_quote_collection_instead_of_counting_as_one_bad_peer` pins that a refusal stops collection rather than joining the failure list, which is what would otherwise let the remaining peers form a quorum and pay anyway. +- **Downgrade bound.** The compile-time assertion in the client, plus `only_silence_triggers_the_legacy_retry`. +- **Still outstanding at the time of writing:** a mixed-version dev testnet exercising legacy node, upgraded node, structured refusal, lost refusal, and send failure against real peers. The unit tests pin the decisions; they do not prove the behaviour end to end. + +### Re-open triggers + +- The unversioned-quote count failing to decay, which would mean a long tail of clients the gate can never protect and would raise the priority of option 3. +- Any settlement change that is **not** a monotonic increase, which makes the upper bound load-bearing for the first time. +- Evidence of peers selectively dropping versioned requests, which would mean the downgrade path is being probed and the fallback should be retired early. +- A decision to move the multiplier on-chain, which would supersede most of this. + +## Notes for AI-assisted work + +AI tools may help draft this ADR, but **must not mark it Accepted without human review**. Accepted ADRs are immutable: create a new superseding ADR rather than editing an Accepted ADR. From 26b34e35f94c08c4ff513c4ad9ff1aa06bb67861 Mon Sep 17 00:00:00 2001 From: grumbach Date: Thu, 13 Aug 2026 16:03:56 +0900 Subject: [PATCH 04/11] chore(deps): resolve ant-protocol to the branch tip Keeps the lockfile at the commit CI resolves for the branch pin. No source change; picks up the ruint advisory bump made on the protocol branch. --- Cargo.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.lock b/Cargo.lock index 11b062d8..af7990aa 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -863,7 +863,7 @@ dependencies = [ [[package]] name = "ant-protocol" version = "2.4.0" -source = "git+https://github.com/grumbach/ant-protocol?branch=settlement-version-quote-gate#620bb9971ba5669bdb23a38568f8d3a597987db1" +source = "git+https://github.com/grumbach/ant-protocol?branch=settlement-version-quote-gate#439ed83d70ab3bbd14c1031c2e319f793141ea5a" dependencies = [ "blake3", "bytes", From 90799bae8d5a00035f85012876a0ced229db24a1 Mon Sep 17 00:00:00 2001 From: grumbach Date: Thu, 13 Aug 2026 20:03:25 +0900 Subject: [PATCH 05/11] chore: drop the named protocol version from the branch-pin comment Versioning is the release train's call, so the comment now points at 'a published version pin' rather than naming one that has not been decided. Lockfile follows the protocol branch, which no longer carries a bump. Also records in ADR-0010 that the semver impact is declared, not taken. --- Cargo.lock | 4 ++-- Cargo.toml | 4 ++-- ...R-0010-settlement-version-and-pre-payment-compatibility.md | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index af7990aa..326dbf27 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -862,8 +862,8 @@ dependencies = [ [[package]] name = "ant-protocol" -version = "2.4.0" -source = "git+https://github.com/grumbach/ant-protocol?branch=settlement-version-quote-gate#439ed83d70ab3bbd14c1031c2e319f793141ea5a" +version = "2.3.2" +source = "git+https://github.com/grumbach/ant-protocol?branch=settlement-version-quote-gate#83160b044bdd3f035b9a78ba214d9510ab3ca753" dependencies = [ "blake3", "bytes", diff --git a/Cargo.toml b/Cargo.toml index c250cc70..74feee3d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -40,8 +40,8 @@ mimalloc = "0.1" # (the rc-2026.4.2 branch) so Cargo can unify the wire types here # with ant-protocol's re-exports. # Branch pin while the settlement-version wire types are in review -# (WithAutonomi/ant-protocol#23). Swap back to `ant-protocol = "2.4.0"` once -# that PR merges and 2.4.0 is published. +# (WithAutonomi/ant-protocol#23). Swap back to a published `ant-protocol` +# version pin once that PR merges and the release train publishes it. ant-protocol = { git = "https://github.com/grumbach/ant-protocol", branch = "settlement-version-quote-gate" } # Core (provides EVERYTHING: networking, DHT, security, trust, storage) diff --git a/docs/adr/ADR-0010-settlement-version-and-pre-payment-compatibility.md b/docs/adr/ADR-0010-settlement-version-and-pre-payment-compatibility.md index dfcb4b48..c0e8208a 100644 --- a/docs/adr/ADR-0010-settlement-version-and-pre-payment-compatibility.md +++ b/docs/adr/ADR-0010-settlement-version-and-pre-payment-compatibility.md @@ -99,7 +99,7 @@ Flipping that to a refusal is a **follow-up**, gated on that count decaying, and - Deploy order is **nodes before clients**, since a node on `ant-protocol` 2.3.x cannot decode versioned requests. The fallback makes this a preference rather than a hard gate. - The unversioned-quote counter is the input to the follow-up decision above. -- `ant-protocol` moves 2.3.2 -> 2.4.0. Additive, minor. +- The `ant-protocol` change is additive, so it carries a minor semver impact. The version bump itself is left to the release train, not taken in the PR. ## Validation From c02b56bd7017d3f78d8a447d55433127e1c0ad75 Mon Sep 17 00:00:00 2001 From: grumbach Date: Thu, 13 Aug 2026 20:28:41 +0900 Subject: [PATCH 06/11] docs(adr): correct the downgrade-guard claim and record the ordering rule The document said the unversioned retry is build-enforced before the minimum can rise. That held for the merkle path only; the independent single-node retry was unguarded. Both now reference one shared constant, and the text says so. Also records why a refusal must not depend on which peers answered first: the collector drains every launched peer rather than stopping at the quote target, and the verdict is kept outside the collection timeout whose elapsed arm deliberately falls through. --- ...ment-version-and-pre-payment-compatibility.md | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/docs/adr/ADR-0010-settlement-version-and-pre-payment-compatibility.md b/docs/adr/ADR-0010-settlement-version-and-pre-payment-compatibility.md index c0e8208a..f83c7d15 100644 --- a/docs/adr/ADR-0010-settlement-version-and-pre-payment-compatibility.md +++ b/docs/adr/ADR-0010-settlement-version-and-pre-payment-compatibility.md @@ -71,7 +71,18 @@ It is safe **only while no client can be refused on version grounds**, which hol > The unversioned retry must be deleted **before** `MIN_SUPPORTED_SETTLEMENT_VERSION` is ever raised. -This is enforced by a compile-time assertion in the client, not by review discipline: raising the minimum while the fallback exists fails the build. +This is enforced by a compile-time assertion in the client, not by review discipline: raising the minimum while a fallback exists fails the build. + +There are **two** independent fallbacks, single-node and merkle. The guard is therefore a single shared constant that each site references, so deleting one path cannot orphan the check for the other. An earlier revision put the assertion beside the merkle path only, which left the single-node retry unguarded while this document claimed otherwise. + +### A refusal must not depend on who answered first + +A refusal is a verdict about the client, not about one peer, so it cannot be treated as one failed response among many. Two things follow, and both were wrong in the first implementation: + +- The collector stops **launching** new peers once it has enough quotes, but keeps **draining** those already launched. Otherwise a refusal still in flight is discarded because faster peers filled the quota, and the upload pays. +- The verdict is recorded outside the collection timeout. The elapsed arm deliberately falls through so quotes from fast peers stay usable, and would otherwise throw away a refusal observed just before the clock ran out. + +`StorerUpdateRequired` deliberately does **not** get this treatment. It is not a verdict about the client, so one lagging peer must not abort an upload the rest of the close group can serve. ### Unversioned requests are still served @@ -107,7 +118,8 @@ Flipping that to a refusal is a **follow-up**, gated on that count decaying, and - **Range policy.** `a_newer_settlement_version_is_refused_rather_than_assumed_compatible` pins the upper bound, which is the specific error this ADR corrects. - **Refusal direction.** `a_newer_settlement_version_is_refused_as_this_nodes_fault` and `the_two_refusals_do_not_blame_the_same_party` pin that a lagging node never reports a client fault. - **Terminality.** `a_refusal_aborts_quote_collection_instead_of_counting_as_one_bad_peer` pins that a refusal stops collection rather than joining the failure list, which is what would otherwise let the remaining peers form a quorum and pay anyway. -- **Downgrade bound.** The compile-time assertion in the client, plus `only_silence_triggers_the_legacy_retry`. +- **Order independence.** `a_refusal_is_recorded_where_the_collection_timeout_cannot_discard_it` and `meeting_the_target_stops_launching_without_stopping_collection` pin that the verdict does not depend on which peers answered first, and `a_lagging_storer_does_not_populate_the_refusal_slot` pins that a behind-the-times peer is not mistaken for one. +- **Downgrade bound.** The shared compile-time constant referenced from both fallback sites, plus `only_silence_triggers_the_legacy_retry`. - **Still outstanding at the time of writing:** a mixed-version dev testnet exercising legacy node, upgraded node, structured refusal, lost refusal, and send failure against real peers. The unit tests pin the decisions; they do not prove the behaviour end to end. ### Re-open triggers From dabe3c74cbc98cafd6cde2233c0ce0705c89c482 Mon Sep 17 00:00:00 2001 From: grumbach Date: Fri, 14 Aug 2026 11:05:41 +0900 Subject: [PATCH 07/11] docs(adr): record the mixed-version validation and what it found The new-client-against-old-fleet case is no longer outstanding. ant-client's merkle E2E spawns a 35-node testnet from the published ant-node, which predates the versioned requests, so the suite is a live mixed-version run. It passed functionally and failed on cost: the suite went from a 24-38 minute baseline to exceeding the 60-minute CI cap with 4 of 7 tests done, because a peer that cannot decode the versioned request never answers and the client waited a full quote timeout before falling back, on every request rather than once per peer. That is the kind of defect a unit test cannot surface, which is why the reviewer was right to ask for this. Records the two client-side bounds that came out of it, the misjudgement tradeoff the probe ceiling accepts, and what remains unproven: the reverse direction needs a testnet built from this branch's ant-node, which is not available until the coordinated set lands. --- ...t-version-and-pre-payment-compatibility.md | 24 +++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) diff --git a/docs/adr/ADR-0010-settlement-version-and-pre-payment-compatibility.md b/docs/adr/ADR-0010-settlement-version-and-pre-payment-compatibility.md index f83c7d15..58062289 100644 --- a/docs/adr/ADR-0010-settlement-version-and-pre-payment-compatibility.md +++ b/docs/adr/ADR-0010-settlement-version-and-pre-payment-compatibility.md @@ -103,7 +103,7 @@ Flipping that to a refusal is a **follow-up**, gated on that count decaying, and - **It does not fix the current burn.** A client too old to settle correctly is also too old to declare a version, so the gate cannot see the population causing today's rejections. Only the reworded error reaches them. Driving client adoption remains the cheaper and faster remedy for the incident that prompted this. - **Merkle storers are not exactly the quoted peers.** The gate covers the 16 candidates a client quotes, but the chunk is stored by each chunk's close group, which may include a peer that never quoted. A storer outside the candidate set can still refuse at PUT time. This narrows the exposure substantially without closing it, and only option 3 closes it fully. -- The fallback costs one extra timeout per silent peer during rollout. Requests run concurrently, so the worst case is 2x the quote timeout overall. +- The fallback costs one probe per silent peer, bounded by `VERSIONED_QUOTE_PROBE_CEILING` and paid once per peer rather than once per request. Before those two bounds it was one full quote timeout on **every** request, which took the merkle E2E suite from ~24 minutes past the 60-minute CI cap. See the validation section. - Clients declaring a version are refused by nodes that have not upgraded. Harmless today because no node has a lower `CURRENT`, but it makes node rollout a prerequisite for the next settlement bump. ### Neutral / Operational @@ -120,7 +120,27 @@ Flipping that to a refusal is a **follow-up**, gated on that count decaying, and - **Terminality.** `a_refusal_aborts_quote_collection_instead_of_counting_as_one_bad_peer` pins that a refusal stops collection rather than joining the failure list, which is what would otherwise let the remaining peers form a quorum and pay anyway. - **Order independence.** `a_refusal_is_recorded_where_the_collection_timeout_cannot_discard_it` and `meeting_the_target_stops_launching_without_stopping_collection` pin that the verdict does not depend on which peers answered first, and `a_lagging_storer_does_not_populate_the_refusal_slot` pins that a behind-the-times peer is not mistaken for one. - **Downgrade bound.** The shared compile-time constant referenced from both fallback sites, plus `only_silence_triggers_the_legacy_retry`. -- **Still outstanding at the time of writing:** a mixed-version dev testnet exercising legacy node, upgraded node, structured refusal, lost refusal, and send failure against real peers. The unit tests pin the decisions; they do not prove the behaviour end to end. +### Mixed-version validation, and what it found + +The new-client-against-old-fleet case has been exercised for real, not simulated. `ant-client`'s merkle E2E suite spawns a 35-node testnet from the **published** `ant-node`, which predates the versioned requests, so the suite is a live mixed-version run: every node logs `Failed to decode message: deserialization failed` for each versioned probe and answers only the unversioned retry. + +It passed functionally and **failed on cost**, which is exactly what a unit test could not have shown: + +| | Merkle E2E, ubuntu | +|---|---| +| Baseline on `main` | 24–38 min | +| First implementation | exceeded the 60-min CI cap with 4 of 7 tests done | + +The cause was that a peer which cannot decode the versioned request never answers, so the client waited the **full** quote timeout before falling back, and paid that on every request rather than once per peer. The suite runs `quote_timeout_secs = 120`, and a merkle pool asks sixteen candidates. + +Two changes came out of it, both in the client: + +- **Remember the answer.** A peer that fails to answer a versioned request is asked in the legacy shape from then on, so the probe is paid once per peer instead of once per request. +- **Cap the probe.** A capability probe does not need the patience of a real quote, so the versioned attempt is bounded by `VERSIONED_QUOTE_PROBE_CEILING`. Production's 10s timeout is already below it; the ceiling only binds in test configurations. + +Cutting a probe short can misjudge a slow but upgraded peer as legacy. The only consequence is a lost version declaration to that peer, because the fallback still obtains a quote, and it cannot matter while no client can be refused on version grounds. By the time it could, the fallback must already be deleted. + +**Still outstanding:** the reverse direction, old client against an upgraded node, and a fleet with both. Those need a testnet built from this branch's `ant-node` rather than the published one, which is not available until the coordinated set lands. Also unproven end to end: structured refusal, lost refusal, and send failure against real peers, which are pinned at unit level only. ### Re-open triggers From bb2095d16c9636c524f3e9c3d8672aa477117ecf Mon Sep 17 00:00:00 2001 From: grumbach Date: Fri, 14 Aug 2026 11:17:27 +0900 Subject: [PATCH 08/11] fix(payment): recognise a stale client at depths that do not divide evenly The upgrade advice on an underpayment was gated on `expected == paid * required_multiplier`. That assumes the expectation is linear in the multiplier, and it is not: `merkle_expected_per_node` floors after multiplying, so at median 901 and depth 7 the parity expectation is 49_426 while three times the bare expectation is 49_425. The check therefore asked `49_426 == 49_425` and stayed silent on a settlement that really was unmultiplied. That silence fell on exactly the population the advice exists for, at every depth whose leaf count is not divisible by the depth. Compare against the bare expectation instead, which is the same arithmetic the expectation was built with and holds everywhere. The predicate is extracted so the case can be pinned directly rather than needing a depth-7 proof fixture. Also corrects a comment in the quote gate that still described the previous policy, claiming a newer settlement version is "deliberately allowed through" immediately above the code that refuses it as StorerUpdateRequired. Records in ADR-0010 the guarantees the tests do and do not provide: no test drives a real collection to timeout, merkle storers are not exactly the quoted peers, a deployed client binary cannot be reached by a source-level guard, and a refusal in a later sub-batch arrives after earlier sub-batches have already paid. --- Cargo.lock | 2 +- ...t-version-and-pre-payment-compatibility.md | 12 ++-- src/payment/verifier.rs | 58 ++++++++++++++++++- src/storage/handler.rs | 10 ++-- 4 files changed, 70 insertions(+), 12 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 326dbf27..244cf9fd 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -863,7 +863,7 @@ dependencies = [ [[package]] name = "ant-protocol" version = "2.3.2" -source = "git+https://github.com/grumbach/ant-protocol?branch=settlement-version-quote-gate#83160b044bdd3f035b9a78ba214d9510ab3ca753" +source = "git+https://github.com/grumbach/ant-protocol?branch=settlement-version-quote-gate#e97901694cba5cad84da0b99ee8af02a791df2cf" dependencies = [ "blake3", "bytes", diff --git a/docs/adr/ADR-0010-settlement-version-and-pre-payment-compatibility.md b/docs/adr/ADR-0010-settlement-version-and-pre-payment-compatibility.md index 58062289..86b7fdb7 100644 --- a/docs/adr/ADR-0010-settlement-version-and-pre-payment-compatibility.md +++ b/docs/adr/ADR-0010-settlement-version-and-pre-payment-compatibility.md @@ -94,7 +94,7 @@ Flipping that to a refusal is a **follow-up**, gated on that count decaying, and ### Positive -- A settlement change can no longer burn an outdated client's money. The refusal lands at quote time, where it costs nothing. +- A settlement change no longer burns an outdated client's money **on the path the gate covers**: the refusal lands at quote time, where it costs nothing. It is a large reduction rather than an elimination, and the two holes are named under trade-offs below. Read this bullet with those. - The user is told what happened and what to do, in both the quote refusal and the PUT-time underpayment message. - No protocol cutover. Nodes and clients roll independently. - The wire discriminants of every pre-existing message and error are pinned by a regression test, so the append-only property this rests on is enforced rather than assumed. @@ -102,7 +102,9 @@ Flipping that to a refusal is a **follow-up**, gated on that count decaying, and ### Negative / Trade-offs - **It does not fix the current burn.** A client too old to settle correctly is also too old to declare a version, so the gate cannot see the population causing today's rejections. Only the reworded error reaches them. Driving client adoption remains the cheaper and faster remedy for the incident that prompted this. -- **Merkle storers are not exactly the quoted peers.** The gate covers the 16 candidates a client quotes, but the chunk is stored by each chunk's close group, which may include a peer that never quoted. A storer outside the candidate set can still refuse at PUT time. This narrows the exposure substantially without closing it, and only option 3 closes it fully. +- **Merkle storers are not exactly the quoted peers.** The gate covers the 16 candidates a client quotes, but the chunk is stored by each chunk's close group, which may include a peer that never quoted, and routing can move that group between quoting and storing. A storer outside the candidate set can still refuse at PUT time, after payment. This narrows the exposure substantially without closing it, and only option 3 closes it fully. +- **A client binary already in the field cannot be reached.** The cutover guard bounds what future builds may do; it does nothing about a released client that still carries the fallback when nodes later raise their minimum. That is inherent to shipping software, and it is the same reason the storer verifies every payment it is actually offered rather than trusting the declared version. +- **A refusal in a later merkle sub-batch arrives after earlier sub-batches have paid.** Batches above `MAX_LEAVES` settle sequentially, so the gate cannot be consulted for sub-batch two before sub-batch one's money is spent. The refusal is surfaced rather than folded into a partial success, but the earlier spend has already happened. - The fallback costs one probe per silent peer, bounded by `VERSIONED_QUOTE_PROBE_CEILING` and paid once per peer rather than once per request. Before those two bounds it was one full quote timeout on **every** request, which took the merkle E2E suite from ~24 minutes past the 60-minute CI cap. See the validation section. - Clients declaring a version are refused by nodes that have not upgraded. Harmless today because no node has a lower `CURRENT`, but it makes node rollout a prerequisite for the next settlement bump. @@ -114,12 +116,12 @@ Flipping that to a refusal is a **follow-up**, gated on that count decaying, and ## Validation -- **Wire safety.** `appending_v2_variants_leaves_existing_discriminants_untouched` and `client_update_required_is_appended_to_protocol_error` pin the discriminant of every pre-existing variant. If either fails, older peers are misreading current traffic. +- **Wire safety.** `appending_v2_variants_leaves_existing_discriminants_untouched` and `client_update_required_is_appended_to_protocol_error` pin the discriminant of every pre-existing message body **and** every pre-existing `ProtocolError`, responses included. An earlier revision pinned only the request half and the endpoints of the error enum, so a reordered response variant would have passed while breaking old peers. - **Range policy.** `a_newer_settlement_version_is_refused_rather_than_assumed_compatible` pins the upper bound, which is the specific error this ADR corrects. - **Refusal direction.** `a_newer_settlement_version_is_refused_as_this_nodes_fault` and `the_two_refusals_do_not_blame_the_same_party` pin that a lagging node never reports a client fault. - **Terminality.** `a_refusal_aborts_quote_collection_instead_of_counting_as_one_bad_peer` pins that a refusal stops collection rather than joining the failure list, which is what would otherwise let the remaining peers form a quorum and pay anyway. -- **Order independence.** `a_refusal_is_recorded_where_the_collection_timeout_cannot_discard_it` and `meeting_the_target_stops_launching_without_stopping_collection` pin that the verdict does not depend on which peers answered first, and `a_lagging_storer_does_not_populate_the_refusal_slot` pins that a behind-the-times peer is not mistaken for one. -- **Downgrade bound.** The shared compile-time constant referenced from both fallback sites, plus `only_silence_triggers_the_legacy_retry`. +- **Order independence.** `a_refusal_is_recorded_where_the_collection_timeout_cannot_discard_it` and `meeting_the_target_stops_launching_without_stopping_collection`, plus `a_lagging_storer_does_not_populate_the_refusal_slot`. Stated precisely, because it is easy to overclaim: these pin the *mechanism* — that the verdict is stored where the elapsed arm cannot reach it, and that the launch budget stops recruiting at the target. **No test drives a real collection to timeout**, so the end-to-end ordering behaviour is argued from those two pieces rather than observed. +- **Downgrade bound.** The shared compile-time constant referenced from both fallback sites, plus `only_silence_triggers_the_legacy_retry` and `only_silence_is_evidence_worth_caching`. The constant now bounds `CURRENT` as well as `MIN`: raising either opens a refusal the unversioned retry could route around, and an earlier revision guarded only `MIN`. ### Mixed-version validation, and what it found The new-client-against-old-fleet case has been exercised for real, not simulated. `ant-client`'s merkle E2E suite spawns a 35-node testnet from the **published** `ant-node`, which predates the versioned requests, so the suite is a live mixed-version run: every node logs `Failed to decode message: deserialization failed` for each versioned probe and answers only the unversioned retry. diff --git a/src/payment/verifier.rs b/src/payment/verifier.rs index da6cb75b..90a21942 100644 --- a/src/payment/verifier.rs +++ b/src/payment/verifier.rs @@ -354,6 +354,24 @@ fn merkle_required_multiplier(receipt_timestamp: u64, enforced_from: u64) -> u64 /// scaling a 1x result is wrong. The order here — multiply the total, then /// divide once — is deliberate, and matches the contract, which computes /// `totalAmount` before splitting it. +/// Does this underpayment look like a client that applied no multiplier at +/// all, as opposed to one that merely paid too little? +/// +/// Compares against the **bare** expectation rather than scaling the paid +/// amount up by the required multiplier. [`merkle_expected_per_node`] floors +/// after multiplying, so `expected(m)` is not generally `m * expected(1)`: at +/// median 901 and depth 7 they differ by one wei. A scaled comparison +/// therefore misses genuinely unmultiplied settlements at every depth that +/// does not divide its leaf count evenly, and those are exactly the clients +/// the upgrade advice exists for. +fn underpayment_looks_like_a_stale_client( + paid: Amount, + expected_bare: Amount, + required_multiplier: u64, +) -> bool { + required_multiplier > 1 && paid == expected_bare +} + fn merkle_expected_per_node( candidate_prices: &[Amount], depth: u8, @@ -3375,8 +3393,11 @@ impl PaymentVerifier { // pay, but only for clients new enough to declare a version, // which by construction excludes every client this branch // catches. - let looks_like_stale_client = expected_per_node - == paid_amount.saturating_mul(Amount::from(required_multiplier)); + let looks_like_stale_client = underpayment_looks_like_a_stale_client( + *paid_amount, + expected_per_node_bare, + required_multiplier, + ); let advice = if looks_like_stale_client { " This is exactly the pre-parity settlement amount, so the paying client is \ too old to settle correctly. Run `ant update` to upgrade, or reinstall from \ @@ -3654,6 +3675,39 @@ mod tests { ); } + /// The same non-linearity that test pins also breaks a naive check for + /// "did this client apply no multiplier". + /// + /// Scaling the paid amount up by the multiplier and comparing against the + /// parity expectation asks `49_426 == 49_425` at median 901 depth 7, which + /// is false, so a genuinely unmultiplied settlement is not recognised and + /// the payer never learns their client is too old. Comparing against the + /// bare expectation asks the same arithmetic the expectation was built + /// with, and holds at every depth. + #[test] + fn a_stale_client_is_recognised_where_depth_does_not_divide_its_leaves() { + let prices = prices_with_median(901); + let bare = merkle_expected_per_node(&prices, 7, 1).expect("bare expectation is payable"); + let parity = merkle_expected_per_node(&prices, 7, PAID_QUOTE_PAYMENT_MULTIPLIER) + .expect("parity expectation is payable"); + + // The scaled comparison this replaced would have missed it. + assert_ne!(parity, bare * Amount::from(PAID_QUOTE_PAYMENT_MULTIPLIER)); + + assert!( + underpayment_looks_like_a_stale_client(bare, bare, PAID_QUOTE_PAYMENT_MULTIPLIER), + "an exactly-1x settlement must be recognised at depth 7" + ); + // A merely-cheap payment still is not blamed on the client's version. + assert!(!underpayment_looks_like_a_stale_client( + bare.saturating_sub(Amount::from(1u64)), + bare, + PAID_QUOTE_PAYMENT_MULTIPLIER + )); + // And under the legacy regime there is nothing to upgrade away from. + assert!(!underpayment_looks_like_a_stale_client(bare, bare, 1)); + } + /// The economic invariant ADR-0008 restores: a merkle **leaf** settles /// for what the single-node path settles — 3x the median quote — up to /// the contract's integer division. diff --git a/src/storage/handler.rs b/src/storage/handler.rs index 22c0e15a..ba30d7fa 100644 --- a/src/storage/handler.rs +++ b/src/storage/handler.rs @@ -85,10 +85,12 @@ static UNVERSIONED_QUOTES_SERVED: [AtomicU64; 2] = [AtomicU64::new(0), AtomicU64 /// costs the client nothing: no quote means no pool commitment, which means no /// payment. /// -/// A version NEWER than this node understands is deliberately allowed through. -/// The storer still verifies whatever payment actually arrives, so nothing is -/// weakened by letting it past, whereas rejecting it would let a stale node -/// veto a rule set the network has already moved to. +/// A version NEWER than this node understands is refused too, as +/// `StorerUpdateRequired`. Quoting it would promise to accept a payment whose +/// rules this build does not know, and that promise can only be broken at PUT +/// time, once the client has settled on-chain. The client is told to use a +/// different storer rather than to upgrade, because it is this node that is +/// behind. fn settlement_gate(client_settlement_version: u32, path: &str) -> Option { match settlement_compatibility(client_settlement_version) { SettlementCompatibility::Compatible => None, From 1a8ff66408cae9915605ef39627dafac4528644f Mon Sep 17 00:00:00 2001 From: grumbach Date: Fri, 14 Aug 2026 11:41:14 +0900 Subject: [PATCH 09/11] docs(adr): correct the rollout guidance and record the corroboration rule Three corrections from a second review pass. The decision text said a client-first rollout must stay possible while the implemented upper bound makes an old node refuse a newer client. Both are true but the first was stated too strongly: the rollout stays usable because the refusal is not a client fault and a newer client routes elsewhere, but node rollout is now a prerequisite for the next settlement bump rather than merely desirable. Says so plainly. The cutover rule keyed deletion of the unversioned retry on MIN alone, while the compile guard requires both MIN and CURRENT. Raising CURRENT creates the node-behind refusal as soon as any node lags, and the retry routes around that too. Records why a refusal needs corroboration before it is believed, and why the verdict is then held client-wide, plus the two trade-offs that come with it: a two-peer denial of service replaces a one-peer one, and a later sub-batch still returns its already-paid proofs rather than stranding that spend. --- ...ent-version-and-pre-payment-compatibility.md | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/docs/adr/ADR-0010-settlement-version-and-pre-payment-compatibility.md b/docs/adr/ADR-0010-settlement-version-and-pre-payment-compatibility.md index 86b7fdb7..b6e9899e 100644 --- a/docs/adr/ADR-0010-settlement-version-and-pre-payment-compatibility.md +++ b/docs/adr/ADR-0010-settlement-version-and-pre-payment-compatibility.md @@ -21,7 +21,7 @@ The general problem is that **`PROTOCOL_VERSION` describes what a peer can parse ## Decision Drivers - A refusal must land **before** payment. Merkle settlement is irreversible, so after-the-fact verification cannot be the only check. -- A client-first rollout must stay possible. ADR-0008 chose client-first deliberately, because a client paying more is accepted by an old node for free. Any policy that makes old nodes refuse new clients breaks that ordering. +- A client-first rollout must stay **usable**, though this decision does narrow it. ADR-0008 chose client-first because a client paying more was accepted by an old node for free. Bounding the upper end ends that: an old node now refuses a newer client rather than quoting a payment it cannot promise to honour. The refusal is deliberately not a client fault, so a newer client routes to peers that can serve it and its user sees nothing, but **node rollout becomes a prerequisite for the next settlement bump** rather than merely desirable. That is the price of not paying before compatibility is established, and it is paid knowingly. - Old peers must not be misread. `ChunkMessage` is postcard-encoded and non-self-describing, so a new field silently changes how existing peers parse every message. - The user must be told what to do. The failure is only expensive because it is silent. @@ -69,12 +69,22 @@ A storer built before the versioned requests cannot decode them and simply never It is safe **only while no client can be refused on version grounds**, which holds exactly while `MIN_SUPPORTED_SETTLEMENT_VERSION` is the first declarable version. The rule is therefore: -> The unversioned retry must be deleted **before** `MIN_SUPPORTED_SETTLEMENT_VERSION` is ever raised. +> The unversioned retry must be deleted **before either `MIN_SUPPORTED_SETTLEMENT_VERSION` or `CURRENT_SETTLEMENT_VERSION` is raised.** + +Both, not just the minimum. Raising `MIN` creates the too-old refusal; raising `CURRENT` creates the node-behind refusal as soon as any node lags, and the retry routes around that one just as readily. This is enforced by a compile-time assertion in the client, not by review discipline: raising the minimum while a fallback exists fails the build. There are **two** independent fallbacks, single-node and merkle. The guard is therefore a single shared constant that each site references, so deleting one path cannot orphan the check for the other. An earlier revision put the assertion beside the merkle path only, which left the single-node retry unguarded while this document claimed otherwise. +### A refusal is a verdict about the client, and needs corroboration + +Nothing authenticates a refusal. Acting on one peer's word would let a single hostile or misconfigured storer answer `ClientUpdateRequired` to everything and deny every upload the client attempts, converting an over-query design that tolerates many bad peers into one that tolerates none. So a refusal is believed only once `SETTLEMENT_REFUSAL_QUORUM` distinct peers agree, and a refusal that does not describe this client (wrong echoed version, or a stated minimum this client already meets) is discarded as a bad peer rather than counted. + +A genuine incompatibility reaches the threshold immediately, because every peer enforcing the newer rule refuses and a client queries far more than two. + +The corroborated verdict is then held **client-wide and sticky**, not in one collector's local state. It concerns this build rather than this upload, so an upload that begins after another has already established it must not proceed to pay, and every payment entry point checks it first. + ### A refusal must not depend on who answered first A refusal is a verdict about the client, not about one peer, so it cannot be treated as one failed response among many. Two things follow, and both were wrong in the first implementation: @@ -104,7 +114,8 @@ Flipping that to a refusal is a **follow-up**, gated on that count decaying, and - **It does not fix the current burn.** A client too old to settle correctly is also too old to declare a version, so the gate cannot see the population causing today's rejections. Only the reworded error reaches them. Driving client adoption remains the cheaper and faster remedy for the incident that prompted this. - **Merkle storers are not exactly the quoted peers.** The gate covers the 16 candidates a client quotes, but the chunk is stored by each chunk's close group, which may include a peer that never quoted, and routing can move that group between quoting and storing. A storer outside the candidate set can still refuse at PUT time, after payment. This narrows the exposure substantially without closing it, and only option 3 closes it fully. - **A client binary already in the field cannot be reached.** The cutover guard bounds what future builds may do; it does nothing about a released client that still carries the fallback when nodes later raise their minimum. That is inherent to shipping software, and it is the same reason the storer verifies every payment it is actually offered rather than trusting the declared version. -- **A refusal in a later merkle sub-batch arrives after earlier sub-batches have paid.** Batches above `MAX_LEAVES` settle sequentially, so the gate cannot be consulted for sub-batch two before sub-batch one's money is spent. The refusal is surfaced rather than folded into a partial success, but the earlier spend has already happened. +- **A refusal in a later merkle sub-batch arrives after earlier sub-batches have paid.** Batches above `MAX_LEAVES` settle sequentially, so the gate cannot be consulted for sub-batch two before sub-batch one's money is spent. That call still returns its earlier proofs rather than failing: the caller writes the receipt cache only on the success path, so failing would strand a spend that has already settled on-chain, which is the destruction this work exists to prevent. Nothing is lost by returning them, because the verdict is latched client-wide and stops the *next* payment instead. +- **A single upload can still be denied by two colluding peers.** The corroboration threshold trades a lone-peer denial-of-service for a two-peer one. Two is chosen because a genuine incompatibility clears it instantly while a lone attacker cannot; raising it would blunt the real signal, and the failure mode is availability rather than lost funds. - The fallback costs one probe per silent peer, bounded by `VERSIONED_QUOTE_PROBE_CEILING` and paid once per peer rather than once per request. Before those two bounds it was one full quote timeout on **every** request, which took the merkle E2E suite from ~24 minutes past the 60-minute CI cap. See the validation section. - Clients declaring a version are refused by nodes that have not upgraded. Harmless today because no node has a lower `CURRENT`, but it makes node rollout a prerequisite for the next settlement bump. From 75fe967186229d08d6479b298ba383caf14ef7a1 Mon Sep 17 00:00:00 2001 From: grumbach Date: Fri, 14 Aug 2026 13:21:05 +0900 Subject: [PATCH 10/11] docs(adr): record why the probe ceiling must not bind in production A shorter probe ceiling was tried to bring the slower CI runner under its job cap, and independent review showed it would silently re-enable the loss this ADR exists to prevent: the probe wait is the only window in which a peer can refuse, and the fallback re-asks under a new request id, so a refusal arriving after the ceiling is answering a request nobody is listening to. Records the resulting rule, that the ceiling stays at or above the largest production quote timeout, and that the remaining suite cost is an artifact of the temporary fork-branch pin rather than of the design. --- ...10-settlement-version-and-pre-payment-compatibility.md | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/docs/adr/ADR-0010-settlement-version-and-pre-payment-compatibility.md b/docs/adr/ADR-0010-settlement-version-and-pre-payment-compatibility.md index b6e9899e..66a1f3d0 100644 --- a/docs/adr/ADR-0010-settlement-version-and-pre-payment-compatibility.md +++ b/docs/adr/ADR-0010-settlement-version-and-pre-payment-compatibility.md @@ -149,9 +149,13 @@ The cause was that a peer which cannot decode the versioned request never answer Two changes came out of it, both in the client: - **Remember the answer.** A peer that fails to answer a versioned request is asked in the legacy shape from then on, so the probe is paid once per peer instead of once per request. -- **Cap the probe.** A capability probe does not need the patience of a real quote, so the versioned attempt is bounded by `VERSIONED_QUOTE_PROBE_CEILING`. Production's 10s timeout is already below it; the ceiling only binds in test configurations. +- **Cap the probe**, with `VERSIONED_QUOTE_PROBE_CEILING`, but only far enough to bound a pathological timeout. Production's 10s sits below it, so **the ceiling never binds on a production client**. -Cutting a probe short can misjudge a slow but upgraded peer as legacy. The only consequence is a lost version declaration to that peer, because the fallback still obtains a quote, and it cannot matter while no client can be refused on version grounds. By the time it could, the fallback must already be deleted. +That last point is a safety property, not a tuning choice, and it was nearly got wrong. A shorter ceiling was tried to bring the slower CI runner under its job cap. It would have been a defect: the probe wait is the only window in which a peer can refuse, and the fallback re-asks under a *new* request id, so a refusal arriving after the ceiling answers a request nobody is listening to. It would never count toward corroboration, never set the latch, and the racing unversioned request could return a quote the client then pays against. Neither the never-demote rule nor the compile-time guard covers this: the first only stops a peer being *cached* as legacy, and the second binds future builds while the clients at risk are the ones already released. + +Cutting a probe short therefore does more than misjudge a slow peer. The rule is: **keep the ceiling at or above the largest production quote timeout.** + +The cost of that is roughly two minutes per merkle E2E test for as long as the suite's devnet speaks the pre-versioned dialect. That is an artifact of the temporary fork-branch protocol pin rather than of the design; when the fleet under test can answer a versioned request there are no probes to pay for and the suite returns to baseline. **Still outstanding:** the reverse direction, old client against an upgraded node, and a fleet with both. Those need a testnet built from this branch's `ant-node` rather than the published one, which is not available until the coordinated set lands. Also unproven end to end: structured refusal, lost refusal, and send failure against real peers, which are pinned at unit level only. From a4aa14e0c1133b677347033a7549a8daea509946 Mon Sep 17 00:00:00 2001 From: grumbach Date: Fri, 14 Aug 2026 15:23:39 +0900 Subject: [PATCH 11/11] docs(adr): record the release gates that block a fleet-ready verdict Merging puts this in the next release, so the bar is fleet-ready rather than code-complete, and green CI is evidence for the code gate alone. Enumerates what is still open: the mutable protocol branch pin, mixed-version proof over a real connection, deployment ordering, observability of the adoption counter, NAT/canary, rollback rehearsal and fleet safety. Calls out that deployment ordering is the one gate that is not inert. The refusal machinery cannot fire while MIN and CURRENT are both the first declarable version, but against a fleet that cannot answer a versioned request every first contact still costs a probe wait, so releasing the client ahead of the nodes adds real latency to cold uploads. --- ...ent-version-and-pre-payment-compatibility.md | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/docs/adr/ADR-0010-settlement-version-and-pre-payment-compatibility.md b/docs/adr/ADR-0010-settlement-version-and-pre-payment-compatibility.md index 66a1f3d0..39af0594 100644 --- a/docs/adr/ADR-0010-settlement-version-and-pre-payment-compatibility.md +++ b/docs/adr/ADR-0010-settlement-version-and-pre-payment-compatibility.md @@ -159,6 +159,23 @@ The cost of that is roughly two minutes per merkle E2E test for as long as the s **Still outstanding:** the reverse direction, old client against an upgraded node, and a fleet with both. Those need a testnet built from this branch's `ant-node` rather than the published one, which is not available until the coordinated set lands. Also unproven end to end: structured refusal, lost refusal, and send failure against real peers, which are pinned at unit level only. +### Release gates + +Merging puts this in the next release, so the bar is fleet-ready, not code-complete. Green CI covers the code gate and nothing else. **The set is not production ready while any of these is open:** + +| Gate | Status | +|---|---| +| Code / CI | Proven across all three repos | +| Dependency | Open: both downstream crates pin a mutable protocol branch | +| Mixed-version | Partial: new-client against an old fleet is proven by the client suite; the gate itself has never run over a real connection, because the devnet speaks the pre-versioned dialect | +| Deployment ordering | Open, and **not inert**: against a fleet that cannot answer a versioned request every first contact costs a probe wait, so releasing the client ahead of the nodes adds real latency to cold uploads | +| Observability | Open: the unversioned-quote counter is the signal that retires the legacy path and has never been read in production | +| NAT / canary | Open: no canary; relayed and NAT'd peers are the paths this adds work to | +| Rollback | Argued, not rehearsed | +| Fleet safety | Open: the corroboration quorum and the client-wide latch have never met a real fleet | + +The refusal machinery is inert on arrival, since `MIN` and `CURRENT` are both the first declarable version, so nothing can be refused yet. That lowers the risk. It does not close a gate, and the client-side cost above is live regardless. + ### Re-open triggers - The unversioned-quote count failing to decay, which would mean a long tail of clients the gate can never protect and would raise the priority of option 3.