From 835416264910302c2dd232a202567e81d2c06c0c Mon Sep 17 00:00:00 2001 From: grumbach Date: Tue, 28 Jul 2026 15:19:37 +0900 Subject: [PATCH 1/3] Pay the 3x settlement multiplier on the merkle batch path A chunk stored through a merkle batch earned its node one third of what the identical chunk earned through a single-node upload. The single-node path pays the median-priced issuer 3x its quote; the merkle path copied the bare quoted price into the on-chain payable amount, so the contract's median16(amount) x 2^depth settled 1x the median per leaf. Apply the multiplier where the payable amount is built. The signed candidate prices and the pool hash are untouched: the hash is the key a storer resolves the on-chain payment record under, and it must keep committing to the quotes the nodes actually signed. Batch uploads get ~3x more expensive. The alternative, lowering the single-node path to 1x, would cut per-chunk revenue network-wide by two thirds instead. --- ant-core/src/data/client/merkle.rs | 156 ++++++++++++++++++++++++++++- 1 file changed, 151 insertions(+), 5 deletions(-) diff --git a/ant-core/src/data/client/merkle.rs b/ant-core/src/data/client/merkle.rs index af0e44b..2fbf60f 100644 --- a/ant-core/src/data/client/merkle.rs +++ b/ant-core/src/data/client/merkle.rs @@ -37,6 +37,28 @@ use xor_name::XorName; /// Default threshold: use merkle payments when chunk count >= this value. pub const DEFAULT_MERKLE_THRESHOLD: usize = 64; +/// Payment multiplier applied to a quoted price before settlement. +/// +/// The single-node path has always paid the median-priced issuer **3× its +/// quoted price** (`SingleNodePayment::from_quotes`), so that the network +/// receives the same revenue as paying three members of the close group while +/// costing one transaction's gas. The merkle path never applied it: it +/// submitted the raw quoted price as the on-chain payable amount, so the +/// contract's `median16(amount) × 2^depth` came to exactly **1×** the median +/// per chunk — a third of what the same chunk earns on the single-node path, +/// for identical storage and replication. +/// +/// The on-chain field is `CandidateNode.amount`, the sum the vault pays out, +/// not the quote itself; the signed candidate keeps its 1× quoted `price` and +/// the pool hash is unchanged, so every proof still verifies against the +/// quotes the nodes actually signed. +/// +/// Must stay equal to the single-node multiplier in +/// `ant_protocol::payment::single_node` and to the storer's +/// `PAID_QUOTE_PAYMENT_MULTIPLIER`. Consolidating all three into one +/// `ant-protocol` constant is a follow-up. +const MERKLE_PAYMENT_MULTIPLIER: u64 = 3; + /// ADR-0004 resolve-before-pay gate for a merkle candidate — the merkle-path /// equivalent of the single-node `quote_commitment_binding_is_valid`. Runs the /// FULL binding check (shape, cap, exact price, and for bound candidates the @@ -111,6 +133,34 @@ fn merkle_candidate_binding_is_valid( Ok(()) } +/// Build the on-chain [`PoolCommitment`] for a candidate pool, applying +/// [`MERKLE_PAYMENT_MULTIPLIER`] to every candidate's payable amount. +/// +/// The contract derives what it pays out from the amounts submitted here +/// (`total = median16(amount) × 2^depth`, split evenly across `depth` +/// winners), so multiplying here — and only here — brings the merkle path to +/// the same per-chunk revenue as the single-node path. +/// +/// The pool hash is deliberately left as `pool.hash()`, computed over the +/// candidates' **signed** 1× prices. It is the key the storer resolves the +/// on-chain payment record under, and it commits to the quotes the nodes +/// actually signed; multiplying the payable amount must not disturb it. +fn pool_commitment_with_payment_multiplier( + pool: &MerklePaymentCandidatePool, +) -> Result { + let mut commitment = pool.to_commitment(); + let multiplier = Amount::from(MERKLE_PAYMENT_MULTIPLIER); + for candidate in &mut commitment.candidates { + candidate.price = candidate.price.checked_mul(multiplier).ok_or_else(|| { + Error::Payment(format!( + "Merkle candidate amount overflow applying {MERKLE_PAYMENT_MULTIPLIER}x to price {}", + candidate.price + )) + })?; + } + Ok(commitment) +} + /// Payment mode for uploads. #[derive(Debug, Clone, Copy, Default, PartialEq, Eq, serde::Serialize, serde::Deserialize)] #[serde(rename_all = "snake_case")] @@ -537,11 +587,14 @@ impl Client { ) .await?; - // 4. Build pool commitments for on-chain payment + // 4. Build pool commitments for on-chain payment. Every candidate's + // payable amount carries MERKLE_PAYMENT_MULTIPLIER so a merkle + // chunk settles for the same amount a single-node chunk does; the + // signed candidate prices and the pool hashes are untouched. let pool_commitments: Vec = candidate_pools .iter() - .map(MerklePaymentCandidatePool::to_commitment) - .collect(); + .map(pool_commitment_with_payment_multiplier) + .collect::>>()?; Ok(PreparedMerkleBatch { depth, @@ -1825,8 +1878,9 @@ mod tests { let pool_commitments = candidate_pools .iter() - .map(MerklePaymentCandidatePool::to_commitment) - .collect(); + .map(pool_commitment_with_payment_multiplier) + .collect::>>() + .unwrap(); PreparedMerkleBatch { depth: tree.depth(), @@ -1838,6 +1892,98 @@ mod tests { } } + /// Candidate pool with distinct prices, so the median is a specific + /// candidate rather than an artifact of every price being equal. + fn pool_with_varied_prices(timestamp: u64) -> MerklePaymentCandidatePool { + let addrs = make_test_addresses(4); + let xornames: Vec = addrs.iter().map(|a| XorName(*a)).collect(); + let tree = MerkleTree::from_xornames(xornames).unwrap(); + let midpoint = tree + .reward_candidates(timestamp) + .unwrap() + .into_iter() + .next() + .unwrap(); + + let candidate_nodes = std::array::from_fn(|i| MerklePaymentCandidateNode { + pub_key: vec![i as u8; 32], + // 100, 200, ... 1600 — upper median (index 8 of 16) is 900. + price: Amount::from((i as u64 + 1) * 100), + reward_address: RewardsAddress::new([i as u8; 20]), + merkle_payment_timestamp: timestamp, + signature: vec![i as u8; 64], + committed_key_count: 0, + commitment_pin: None, + }); + + MerklePaymentCandidatePool { + midpoint_proof: midpoint, + candidate_nodes, + } + } + + /// The contract's `median16`: upper median, index 8 of 16 ascending. + fn median16(mut amounts: Vec) -> Amount { + amounts.sort_unstable(); + *amounts.get(amounts.len() / 2).unwrap() + } + + #[test] + fn pool_commitment_applies_payment_multiplier_to_every_candidate() { + let pool = pool_with_varied_prices(1_700_000_000); + let commitment = pool_commitment_with_payment_multiplier(&pool).unwrap(); + + for (candidate, signed) in commitment + .candidates + .iter() + .zip(pool.candidate_nodes.iter()) + { + assert_eq!( + candidate.price, + signed.price * Amount::from(MERKLE_PAYMENT_MULTIPLIER), + "on-chain payable amount must be {MERKLE_PAYMENT_MULTIPLIER}x the quoted price" + ); + } + } + + #[test] + fn pool_commitment_multiplier_leaves_signed_prices_and_pool_hash_untouched() { + let pool = pool_with_varied_prices(1_700_000_000); + let before: Vec = pool.candidate_nodes.iter().map(|c| c.price).collect(); + + let commitment = pool_commitment_with_payment_multiplier(&pool).unwrap(); + + let after: Vec = pool.candidate_nodes.iter().map(|c| c.price).collect(); + assert_eq!(before, after, "signed candidate prices must not change"); + assert_eq!( + commitment.pool_hash, + pool.hash(), + "pool hash is the storer's on-chain lookup key and must be \ + computed over the signed 1x prices" + ); + } + + /// The invariant the fix exists for: a chunk paid through the merkle path + /// settles for the same amount as a chunk paid through the single-node + /// path. Contract formula: `total = median16(amount) * 2^depth`, spread + /// over `2^depth` leaves, so per chunk it is exactly `median16(amount)`. + #[test] + fn merkle_per_chunk_settlement_matches_single_node_multiplier() { + let pool = pool_with_varied_prices(1_700_000_000); + let commitment = pool_commitment_with_payment_multiplier(&pool).unwrap(); + + let quoted_median = median16(pool.candidate_nodes.iter().map(|c| c.price).collect()); + let per_chunk = median16(commitment.candidates.iter().map(|c| c.price).collect()); + + assert_eq!(quoted_median, Amount::from(900u64)); + assert_eq!( + per_chunk, + quoted_median * Amount::from(MERKLE_PAYMENT_MULTIPLIER), + "merkle per-chunk settlement must equal the single-node \ + {MERKLE_PAYMENT_MULTIPLIER}x median, not the bare quoted price" + ); + } + #[test] fn test_finalize_merkle_batch_with_valid_winner() { let prepared = make_prepared_merkle_batch(4); From b8c1a35c398aad3be4dc8766cf4d343870000984 Mon Sep 17 00:00:00 2001 From: grumbach Date: Tue, 28 Jul 2026 16:46:57 +0900 Subject: [PATCH 2/3] Bill merkle uploads for the padded tree in cost estimates Two follow-ups to paying the 3x multiplier on the merkle path. Read the multiplier from the single-node constant instead of a second literal 3. The defect being fixed was the two payment paths disagreeing about the multiplier, so they now read it from one place. Bill the estimate for padded leaves. The vault charges median16 x 2^depth per batch and the tree rounds its leaf count up to a power of two, so a 65-chunk batch pays for 128 leaves. Estimating from the raw chunk count was already wrong before this change, but it erred high and was harmless; now that merkle settles at 3x it errs low, by up to 2x on a non-power-of-two batch. An under-quote is the harmful direction, because a caller sizing its wallet from the estimate runs dry mid-upload. --- ant-core/src/data/client/file.rs | 76 ++++++++++++++++++++++++++++-- ant-core/src/data/client/merkle.rs | 29 ++++++------ 2 files changed, 88 insertions(+), 17 deletions(-) diff --git a/ant-core/src/data/client/file.rs b/ant-core/src/data/client/file.rs index f2fe43c..7978f7e 100644 --- a/ant-core/src/data/client/file.rs +++ b/ant-core/src/data/client/file.rs @@ -346,6 +346,38 @@ const GAS_PER_WAVE_TX: u128 = 1_500_000; /// and posts a pool commitment, so budget higher than a plain transfer. const GAS_PER_MERKLE_TX: u128 = 500_000; +/// Leaves a merkle upload of `chunk_count` chunks is actually billed for. +/// +/// The vault charges `median16 × 2^depth` per batch and `MerkleTree` rounds its +/// leaf count up to a power of two, so a batch of 65 chunks pays for 128 +/// leaves. Uploads longer than `MAX_LEAVES` split into full batches plus a +/// remainder batch, and only the remainder is padded. +/// +/// Rounding up matters in one direction only: a caller sizing its wallet from +/// an under-quote runs dry mid-upload, so the padding is billed rather than +/// hidden. +fn merkle_billable_leaves(chunk_count: u64) -> u64 { + let per_batch = u64::try_from(MAX_LEAVES).unwrap_or(u64::MAX); + if per_batch == 0 { + return chunk_count; + } + + let full_batches = chunk_count / per_batch; + let remainder = chunk_count % per_batch; + + // A merkle tree needs at least two leaves, so a one-chunk remainder is + // still billed as a two-leaf tree. + let padded_remainder = match remainder { + 0 => 0, + 1 => 2, + n => n.checked_next_power_of_two().unwrap_or(u64::MAX), + }; + + full_batches + .saturating_mul(per_batch) + .saturating_add(padded_remainder) +} + /// Advisory gas price (wei/gas) used to turn the gas estimate into an ETH /// figure when no live gas oracle is consulted. /// @@ -1343,8 +1375,8 @@ impl Client { } }; - // Use the median price × 3 (matches the single-node payment builder - // which pays 3x the median to incentivize reliable storage). + // Use the median price × 3, matching the settlement multiplier both + // payment paths now apply. let mut prices: Vec = quotes.iter().map(|(_, _, _, price, _)| *price).collect(); prices.sort(); let median_price = prices @@ -1354,7 +1386,18 @@ impl Client { let per_chunk_cost = median_price * Amount::from(SINGLE_NODE_PAYMENT_MULTIPLIER); let chunk_count_u64 = u64::try_from(chunk_count).unwrap_or(u64::MAX); - let total_storage = per_chunk_cost * Amount::from(chunk_count_u64); + // Merkle settles per *padded* leaf, not per chunk: the contract charges + // `median16 × 2^depth` per batch and the tree rounds up to a power of + // two, so a 65-chunk batch pays for 128 leaves. Billing the raw chunk + // count here would under-quote a non-power-of-two batch by up to 2x — + // and an under-quote is the harmful direction, because a caller sizing + // its wallet from this estimate would run dry mid-upload. + let billable_units = if uses_merkle { + merkle_billable_leaves(chunk_count_u64) + } else { + chunk_count_u64 + }; + let total_storage = per_chunk_cost * Amount::from(billable_units); // Estimate gas cost from realistic per-transaction budgets rather // than a flat per-chunk or per-wave number. @@ -3451,6 +3494,33 @@ mod tests { assert_eq!(distributed_sample_indices(5, 5), vec![0, 1, 2, 3, 4]); } + #[test] + fn merkle_billable_leaves_bills_the_padded_tree() { + // Powers of two are billed exactly. + assert_eq!(merkle_billable_leaves(64), 64); + assert_eq!(merkle_billable_leaves(256), 256); + // Anything else pays for the padding the contract charges for. + assert_eq!(merkle_billable_leaves(65), 128); + assert_eq!(merkle_billable_leaves(100), 128); + assert_eq!(merkle_billable_leaves(129), 256); + // A tree needs two leaves minimum. + assert_eq!(merkle_billable_leaves(1), 2); + // Beyond MAX_LEAVES only the remainder batch is padded. + assert_eq!(merkle_billable_leaves(257), 256 + 2); + assert_eq!(merkle_billable_leaves(300), 256 + 64); + assert_eq!(merkle_billable_leaves(512), 512); + } + + #[test] + fn merkle_billable_leaves_never_under_quotes() { + for chunks in 1..2000u64 { + assert!( + merkle_billable_leaves(chunks) >= chunks, + "{chunks} chunks must never be billed as fewer leaves" + ); + } + } + #[test] fn distributed_sample_indices_is_in_range_and_increasing() { assert!(distributed_sample_indices(0, 5).is_empty()); diff --git a/ant-core/src/data/client/merkle.rs b/ant-core/src/data/client/merkle.rs index 2fbf60f..22ad385 100644 --- a/ant-core/src/data/client/merkle.rs +++ b/ant-core/src/data/client/merkle.rs @@ -39,25 +39,26 @@ pub const DEFAULT_MERKLE_THRESHOLD: usize = 64; /// Payment multiplier applied to a quoted price before settlement. /// -/// The single-node path has always paid the median-priced issuer **3× its -/// quoted price** (`SingleNodePayment::from_quotes`), so that the network -/// receives the same revenue as paying three members of the close group while -/// costing one transaction's gas. The merkle path never applied it: it -/// submitted the raw quoted price as the on-chain payable amount, so the -/// contract's `median16(amount) × 2^depth` came to exactly **1×** the median -/// per chunk — a third of what the same chunk earns on the single-node path, -/// for identical storage and replication. +/// Deliberately the **same constant** the single-node path uses rather than a +/// second copy of `3`: the whole defect this fixes was the two paths disagreeing +/// about the multiplier, so they now read it from one place. (The storer's +/// `PAID_QUOTE_PAYMENT_MULTIPLIER` and `ant_protocol`'s single-node builder are +/// still separate literals; folding all of them into one `ant-protocol` +/// constant is a follow-up.) +/// +/// The single-node path pays the median-priced issuer 3× its quoted price, so +/// the network receives the same revenue as paying three members of the close +/// group while costing one transaction's gas. The merkle path never applied it: +/// it submitted the raw quoted price as the on-chain payable amount, so the +/// contract's `median16(amount) × 2^depth` came to **1×** the median per padded +/// leaf — a third of what the same chunk earns on the single-node path, for +/// identical storage and replication. /// /// The on-chain field is `CandidateNode.amount`, the sum the vault pays out, /// not the quote itself; the signed candidate keeps its 1× quoted `price` and /// the pool hash is unchanged, so every proof still verifies against the /// quotes the nodes actually signed. -/// -/// Must stay equal to the single-node multiplier in -/// `ant_protocol::payment::single_node` and to the storer's -/// `PAID_QUOTE_PAYMENT_MULTIPLIER`. Consolidating all three into one -/// `ant-protocol` constant is a follow-up. -const MERKLE_PAYMENT_MULTIPLIER: u64 = 3; +use crate::data::client::payment::SINGLE_NODE_PAYMENT_MULTIPLIER as MERKLE_PAYMENT_MULTIPLIER; /// ADR-0004 resolve-before-pay gate for a merkle candidate — the merkle-path /// equivalent of the single-node `quote_commitment_binding_is_valid`. Runs the From ffb7567482833b5c92699385b387e6605fb1da73 Mon Sep 17 00:00:00 2001 From: Chris O'Neil Date: Tue, 28 Jul 2026 14:25:29 +0100 Subject: [PATCH 3/3] Partition merkle batches so no remainder is a one-leaf tree Payment split oversized uploads with `addresses.chunks(MAX_LEAVES)`, which respects the 256-leaf ceiling but not the two-leaf floor: 257 addresses came out as [256, 1]. The 256-address batch was paid on-chain and the singleton remainder could not build a tree, so the upload returned a paid partial failure. Every post-preflight count congruent to 1 modulo 256 hits it (257, 513, 769, ...), and preflight leaves an arbitrary count behind. Add one partitioning helper, `merkle_batch_sizes`, and route both payment and cost estimation through it. It borrows one address from the preceding batch rather than stranding it: 257 becomes [255, 2], 513 becomes [256, 255, 2]. Order is preserved and no address is duplicated or synthesised, so a partition is a plain in-order cover whose every batch is a buildable 2..=256-leaf tree. Billable leaves are now summed over those same partitions instead of a second batching model that could drift from what payment does. The leaf totals are unchanged (255 pads to 256), but they are now derived rather than restated. The external-signer path passed its whole address set to `prepare_merkle_batch_external`, which cannot span transactions: one prepared batch is one signature and one payment. It now returns a typed `MerkleBatchTooLarge` before any candidate collection, rather than failing opaquely in the tree build or silently switching payment model. Also renames `merkle_per_chunk_settlement_matches_single_node_multiplier`, which proves 3x settlement per *padded leaf* against the pool median, not equal cost per actual chunk around padding boundaries. Co-Authored-By: Claude Opus 5 (1M context) --- ant-core/src/data/client/file.rs | 92 +++----- ant-core/src/data/client/merkle.rs | 325 ++++++++++++++++++++++++++-- ant-core/src/data/client/mod.rs | 14 ++ ant-core/src/data/error.rs | 32 +++ ant-core/tests/e2e_cost_estimate.rs | 65 +++++- ant-core/tests/e2e_merkle.rs | 126 ++++++++++- ant-core/tests/merkle_unit.rs | 96 +++++++- 7 files changed, 665 insertions(+), 85 deletions(-) diff --git a/ant-core/src/data/client/file.rs b/ant-core/src/data/client/file.rs index 7978f7e..42bc60d 100644 --- a/ant-core/src/data/client/file.rs +++ b/ant-core/src/data/client/file.rs @@ -17,14 +17,14 @@ use crate::data::client::batch::{ use crate::data::client::chunk::ChunkPeerGetResult; use crate::data::client::classify_error; use crate::data::client::merkle::{ - chunk_contents_for_upload_addresses, finalize_merkle_batch, merkle_deferred_retry, - merkle_store_with_retry, should_use_merkle, MerkleBatchPaymentResult, PaymentMode, - PreparedMerkleBatch, DEFERRED_ROUND_DELAYS_SECS, + chunk_contents_for_upload_addresses, finalize_merkle_batch, merkle_batch_sizes, + merkle_billable_leaves, merkle_deferred_retry, merkle_store_with_retry, should_use_merkle, + MerkleBatchPaymentResult, PaymentMode, PreparedMerkleBatch, DEFERRED_ROUND_DELAYS_SECS, }; use crate::data::client::payment::SINGLE_NODE_PAYMENT_MULTIPLIER; use crate::data::client::Client; use crate::data::error::{Error, PartialUploadSpend, Result}; -use ant_protocol::evm::{Amount, PaymentQuote, QuoteHash, TxHash, MAX_LEAVES}; +use ant_protocol::evm::{Amount, PaymentQuote, QuoteHash, TxHash}; use ant_protocol::transport::{MultiAddr, PeerId}; use ant_protocol::{compute_address, XorName as ChunkAddress, DATA_TYPE_CHUNK}; use bytes::Bytes; @@ -346,38 +346,6 @@ const GAS_PER_WAVE_TX: u128 = 1_500_000; /// and posts a pool commitment, so budget higher than a plain transfer. const GAS_PER_MERKLE_TX: u128 = 500_000; -/// Leaves a merkle upload of `chunk_count` chunks is actually billed for. -/// -/// The vault charges `median16 × 2^depth` per batch and `MerkleTree` rounds its -/// leaf count up to a power of two, so a batch of 65 chunks pays for 128 -/// leaves. Uploads longer than `MAX_LEAVES` split into full batches plus a -/// remainder batch, and only the remainder is padded. -/// -/// Rounding up matters in one direction only: a caller sizing its wallet from -/// an under-quote runs dry mid-upload, so the padding is billed rather than -/// hidden. -fn merkle_billable_leaves(chunk_count: u64) -> u64 { - let per_batch = u64::try_from(MAX_LEAVES).unwrap_or(u64::MAX); - if per_batch == 0 { - return chunk_count; - } - - let full_batches = chunk_count / per_batch; - let remainder = chunk_count % per_batch; - - // A merkle tree needs at least two leaves, so a one-chunk remainder is - // still billed as a two-leaf tree. - let padded_remainder = match remainder { - 0 => 0, - 1 => 2, - n => n.checked_next_power_of_two().unwrap_or(u64::MAX), - }; - - full_batches - .saturating_mul(per_batch) - .saturating_add(padded_remainder) -} - /// Advisory gas price (wei/gas) used to turn the gas estimate into an ETH /// figure when no live gas oracle is consulted. /// @@ -1388,10 +1356,9 @@ impl Client { let chunk_count_u64 = u64::try_from(chunk_count).unwrap_or(u64::MAX); // Merkle settles per *padded* leaf, not per chunk: the contract charges // `median16 × 2^depth` per batch and the tree rounds up to a power of - // two, so a 65-chunk batch pays for 128 leaves. Billing the raw chunk - // count here would under-quote a non-power-of-two batch by up to 2x — - // and an under-quote is the harmful direction, because a caller sizing - // its wallet from this estimate would run dry mid-upload. + // two, so a 65-chunk batch pays for 128 leaves. The leaf total is + // summed over the batches the payment path really builds + // (`merkle_batch_sizes`), so the estimate cannot drift from execution. let billable_units = if uses_merkle { merkle_billable_leaves(chunk_count_u64) } else { @@ -1415,7 +1382,10 @@ impl Client { // Gas is priced at ARBITRUM_GAS_PRICE_WEI (~0.1 gwei, a typical // Arbitrum baseline). Treat the result as advisory, not a commitment. let waves = u128::try_from(chunk_count.div_ceil(UPLOAD_WAVE_SIZE)).unwrap_or(u128::MAX); - let merkle_batches = u128::try_from(chunk_count.div_ceil(MAX_LEAVES)).unwrap_or(u128::MAX); + // One tx per batch the payment path builds — same partition the leaf + // total above is derived from. + let merkle_batches = + u128::try_from(merkle_batch_sizes(chunk_count).len()).unwrap_or(u128::MAX); let estimated_gas: u128 = if uses_merkle { merkle_batches .saturating_mul(GAS_PER_MERKLE_TX) @@ -1610,6 +1580,11 @@ impl Client { already_stored.append(&mut wave_already_stored); (payment_info, already_stored) } else { + // One prepared batch is one signature and one payment, so + // more than MAX_LEAVES addresses is refused with + // `MerkleBatchTooLarge` before any candidate collection — + // the wallet path's multi-transaction split has no + // external-signing equivalent to fall back on. match self .prepare_merkle_batch_external( &merkle_plan.to_upload, @@ -3494,29 +3469,20 @@ mod tests { assert_eq!(distributed_sample_indices(5, 5), vec![0, 1, 2, 3, 4]); } + /// The estimator bills the padded tree, and the leaf total it bills comes + /// from the batches the payment path really builds — a `[255, 2]` split of + /// 257 chunks, not a `[256, 1]` one that could never be paid. #[test] - fn merkle_billable_leaves_bills_the_padded_tree() { - // Powers of two are billed exactly. - assert_eq!(merkle_billable_leaves(64), 64); - assert_eq!(merkle_billable_leaves(256), 256); - // Anything else pays for the padding the contract charges for. - assert_eq!(merkle_billable_leaves(65), 128); - assert_eq!(merkle_billable_leaves(100), 128); - assert_eq!(merkle_billable_leaves(129), 256); - // A tree needs two leaves minimum. - assert_eq!(merkle_billable_leaves(1), 2); - // Beyond MAX_LEAVES only the remainder batch is padded. - assert_eq!(merkle_billable_leaves(257), 256 + 2); - assert_eq!(merkle_billable_leaves(300), 256 + 64); - assert_eq!(merkle_billable_leaves(512), 512); - } - - #[test] - fn merkle_billable_leaves_never_under_quotes() { - for chunks in 1..2000u64 { - assert!( - merkle_billable_leaves(chunks) >= chunks, - "{chunks} chunks must never be billed as fewer leaves" + fn estimator_leaf_total_is_the_padded_payment_partition() { + for chunks in [2u64, 64, 65, 100, 129, 255, 256, 257, 300, 512, 513, 769] { + let from_partition: u64 = merkle_batch_sizes(chunks as usize) + .into_iter() + .map(|size| size.next_power_of_two() as u64) + .sum(); + assert_eq!( + merkle_billable_leaves(chunks), + from_partition, + "{chunks} chunks must be billed for the partition the payment path pays" ); } } diff --git a/ant-core/src/data/client/merkle.rs b/ant-core/src/data/client/merkle.rs index 22ad385..2973c7e 100644 --- a/ant-core/src/data/client/merkle.rs +++ b/ant-core/src/data/client/merkle.rs @@ -355,6 +355,118 @@ fn preflight_stored_status(result: Result) -> Result { } } +/// Split `total` addresses into the merkle batches an upload is actually paid in. +/// +/// Every batch becomes one `MerkleTree`, and a tree is only valid with +/// `2..=MAX_LEAVES` leaves. The obvious `addresses.chunks(MAX_LEAVES)` split +/// respects the upper bound but not the lower one: 257 addresses come out as +/// `[256, 1]`, the 256-address batch is paid for on-chain, and the singleton +/// remainder cannot build a tree — so the upload fails as a *paid* partial. +/// Every count congruent to 1 modulo `MAX_LEAVES` (257, 513, 769, …) hits it, +/// and the merkle preflight can leave an arbitrary count behind. +/// +/// Borrowing one address from the preceding batch removes the case entirely: +/// 257 splits as `[255, 2]` and 513 as `[256, 255, 2]`. Order is preserved and +/// no address is duplicated or synthesised, so the partition is a plain +/// in-order cover of the input. +/// +/// Returns an empty vector for `total < 2`, which no merkle path may pay for — +/// `pay_for_merkle_batch` rejects those counts up front. +#[must_use] +pub fn merkle_batch_sizes(total: usize) -> Vec { + if total < 2 { + return Vec::new(); + } + + let mut sizes = Vec::with_capacity(total.div_ceil(MAX_LEAVES)); + let mut remaining = total; + while remaining > MAX_LEAVES { + // Taking a full MAX_LEAVES here would strand a single address as the + // final batch; take one fewer so the tail is a payable two-leaf tree. + let take = if remaining - MAX_LEAVES == 1 { + MAX_LEAVES - 1 + } else { + MAX_LEAVES + }; + sizes.push(take); + remaining -= take; + } + sizes.push(remaining); + sizes +} + +/// Split `addresses` into the sub-batches [`merkle_batch_sizes`] describes. +/// +/// The slices borrow `addresses` in order, so the partition cannot introduce a +/// duplicate or a synthetic address. +#[must_use] +pub fn merkle_batch_partitions(addresses: &[[u8; 32]]) -> Vec<&[[u8; 32]]> { + let mut partitions = Vec::new(); + let mut rest = addresses; + for size in merkle_batch_sizes(addresses.len()) { + let (batch, tail) = rest.split_at(size); + partitions.push(batch); + rest = tail; + } + partitions +} + +/// Leaves one merkle batch of `batch_size` addresses is billed for. +/// +/// `MerkleTree` pads its leaf count up to a power of two and the vault charges +/// `median16 × 2^depth`, so a 65-address batch pays for 128 leaves. +fn padded_leaf_count(batch_size: usize) -> u64 { + // Saturate rather than wrap: a batch is at most MAX_LEAVES, so the + // overflow arm is unreachable, and erring high never under-quotes. + let padded = batch_size + .max(2) + .checked_next_power_of_two() + .unwrap_or(usize::MAX); + u64::try_from(padded).unwrap_or(u64::MAX) +} + +/// Leaves a merkle upload of `chunk_count` chunks is actually billed for. +/// +/// Summed over the batches [`merkle_batch_sizes`] will really pay for, so the +/// estimate and the execution path share one batching model rather than two +/// that can drift. Billing the raw chunk count would under-quote a +/// non-power-of-two batch by up to 2×, and an under-quote is the harmful +/// direction: a caller sizing its wallet from the estimate runs dry +/// mid-upload. +#[must_use] +pub fn merkle_billable_leaves(chunk_count: u64) -> u64 { + let total = usize::try_from(chunk_count).unwrap_or(usize::MAX); + let batches = merkle_batch_sizes(total); + if batches.is_empty() { + // No payable partition exists below two chunks. Nothing costs nothing; + // a lone chunk is still quoted as the two-leaf minimum a tree needs. + return if total == 0 { 0 } else { 2 }; + } + + batches + .into_iter() + .map(padded_leaf_count) + .fold(0u64, u64::saturating_add) +} + +/// Reject an address set that cannot be prepared as a single merkle tree. +/// +/// The wallet path splits an oversized upload with [`merkle_batch_sizes`] and +/// pays each batch in its own transaction. The external-signer contract is one +/// prepared batch → one signature → one payment, so it has no way to express +/// that split. Refusing before any candidate collection or on-chain spend is +/// the honest answer; silently switching the caller to a different payment +/// model is not. +fn ensure_single_merkle_tree_batch(address_count: usize) -> Result<()> { + if address_count > MAX_LEAVES { + return Err(Error::MerkleBatchTooLarge { + addresses: address_count, + max_leaves: MAX_LEAVES, + }); + } + Ok(()) +} + /// Determine whether to use merkle payments for a given batch size. /// Free function — no Client needed. #[must_use] @@ -376,7 +488,8 @@ impl Client { /// Pay for a batch of chunks using merkle batch payment. /// /// Builds a merkle tree, collects candidate pools, pays on-chain in one tx, - /// and returns per-chunk proofs. Splits into sub-batches if > `MAX_LEAVES`. + /// and returns per-chunk proofs. Anything longer than `MAX_LEAVES` is split + /// by [`merkle_batch_sizes`] and paid one transaction per sub-batch. /// /// This low-level helper assumes the caller has already selected the /// addresses that need payment. User-facing upload paths first run the @@ -541,12 +654,22 @@ impl Client { /// Builds the merkle tree, collects candidate pools from the network, /// and returns the data needed for the on-chain payment call. /// Requires `EvmNetwork` but NOT a wallet. + /// + /// # Errors + /// + /// Returns [`Error::MerkleBatchTooLarge`] if `addresses` holds more than + /// `MAX_LEAVES` entries. One prepared batch is one signature and one + /// payment, so an oversized set has no valid external-signing form; the + /// wallet path splits it across transactions instead. The check runs + /// before any candidate collection, so nothing is spent. pub async fn prepare_merkle_batch_external( &self, addresses: &[[u8; 32]], data_type: u32, data_size: u64, ) -> Result { + ensure_single_merkle_tree_batch(addresses.len())?; + let chunk_count = addresses.len(); let xornames: Vec = addresses.iter().map(|a| XorName(*a)).collect(); @@ -650,7 +773,11 @@ impl Client { data_type: u32, data_size: u64, ) -> Result { - let sub_batches: Vec<&[[u8; 32]]> = addresses.chunks(MAX_LEAVES).collect(); + // Partition with the shared helper, NOT `chunks(MAX_LEAVES)`: the naive + // split leaves a one-address final batch for every count congruent to 1 + // modulo MAX_LEAVES, which cannot build a tree and so turns a paid + // upload into a partial failure. + let sub_batches = merkle_batch_partitions(addresses); let total_sub_batches = sub_batches.len(); let mut all_proofs = HashMap::with_capacity(addresses.len()); let mut total_storage = Amount::ZERO; @@ -1964,12 +2091,16 @@ mod tests { ); } - /// The invariant the fix exists for: a chunk paid through the merkle path - /// settles for the same amount as a chunk paid through the single-node - /// path. Contract formula: `total = median16(amount) * 2^depth`, spread - /// over `2^depth` leaves, so per chunk it is exactly `median16(amount)`. + /// The invariant the fix exists for, stated at the level it actually + /// holds: the median payable amount over the winning pool is + /// `MERKLE_PAYMENT_MULTIPLIER x` the median *quoted* price. The contract + /// spends `median16(amount) * 2^depth` over `2^depth` **padded** leaves, so + /// this is the settlement per padded leaf — the same figure the single-node + /// path pays its median-priced issuer per chunk. It says nothing about cost + /// per *actual* chunk: a batch whose size is not a power of two pays for + /// padding leaves too (that is what `merkle_billable_leaves` bills for). #[test] - fn merkle_per_chunk_settlement_matches_single_node_multiplier() { + fn merkle_settlement_per_padded_leaf_is_the_multiplied_pool_median() { let pool = pool_with_varied_prices(1_700_000_000); let commitment = pool_commitment_with_payment_multiplier(&pool).unwrap(); @@ -2074,19 +2205,179 @@ mod tests { // Batch splitting edge cases // ========================================================================= + /// Counts spanning every interesting case: the minimum tree, the merkle + /// threshold and just past it, either side of a full batch, and the + /// `1 mod MAX_LEAVES` counts the naive `chunks(MAX_LEAVES)` split turned + /// into an unpayable `[..., 1]` tail. + const PARTITION_CASES: [(usize, &[usize]); 10] = [ + (2, &[2]), + (64, &[64]), + (65, &[65]), + (255, &[255]), + (256, &[256]), + (257, &[255, 2]), + (300, &[256, 44]), + (512, &[256, 256]), + (513, &[256, 255, 2]), + (769, &[256, 256, 255, 2]), + ]; + #[test] - fn test_batch_split_calculation() { - // MAX_LEAVES chunks should fit in 1 batch - let addrs = make_test_addresses(MAX_LEAVES); - assert_eq!(addrs.chunks(MAX_LEAVES).count(), 1); + fn merkle_batch_sizes_rebalance_singleton_remainders() { + for (total, expected) in PARTITION_CASES { + assert_eq!( + merkle_batch_sizes(total), + expected, + "{total} addresses must partition as {expected:?}" + ); + } + } - // MAX_LEAVES + 1 should split into 2 - let addrs = make_test_addresses(MAX_LEAVES + 1); - assert_eq!(addrs.chunks(MAX_LEAVES).count(), 2); + /// The defect: `[256, 1]` pays the first batch on-chain and then hands a + /// single address to a tree that needs two, so the upload fails *after* + /// spending. Every count must produce trees that can all be built. + #[test] + fn merkle_batch_sizes_are_always_buildable_trees() { + for total in 2..=(4 * MAX_LEAVES + 3) { + let sizes = merkle_batch_sizes(total); + assert!(!sizes.is_empty(), "{total} addresses must produce batches"); + assert_eq!( + sizes.iter().sum::(), + total, + "{total} addresses: partition must cover every address" + ); + for size in sizes { + assert!( + (2..=MAX_LEAVES).contains(&size), + "{total} addresses produced a batch of {size}, outside 2..={MAX_LEAVES}" + ); + } + } + } - // 3 * MAX_LEAVES should split into 3 - let addrs = make_test_addresses(3 * MAX_LEAVES); - assert_eq!(addrs.chunks(MAX_LEAVES).count(), 3); + #[test] + fn merkle_batch_sizes_below_two_have_no_payable_partition() { + assert!(merkle_batch_sizes(0).is_empty()); + assert!(merkle_batch_sizes(1).is_empty()); + } + + #[test] + fn merkle_batch_partitions_preserve_order_and_use_each_address_once() { + for (total, _) in PARTITION_CASES { + let addrs = make_test_addresses(total); + let partitions = merkle_batch_partitions(&addrs); + + let flattened: Vec<[u8; 32]> = partitions.concat(); + assert_eq!( + flattened, addrs, + "{total} addresses: partitions must concatenate back to the input in order" + ); + + let unique: std::collections::HashSet<[u8; 32]> = flattened.iter().copied().collect(); + assert_eq!( + unique.len(), + total, + "{total} addresses: no address may be duplicated or synthesised" + ); + } + } + + /// A merkle upload of 257 chunks — the count the old split could not pay — + /// is the shape preflight routinely leaves behind, since `to_upload` is + /// whatever the network did not already hold. + #[test] + fn post_preflight_plan_of_257_partitions_into_payable_batches() { + let plan = MerkleUploadPlan { + already_stored: make_test_addresses(3), + to_upload: make_test_addresses(257), + to_upload_total_bytes: 257 * 1024, + }; + assert_eq!(plan.to_upload.len(), 257); + + let partitions = merkle_batch_partitions(&plan.to_upload); + let sizes: Vec = partitions.iter().map(|batch| batch.len()).collect(); + assert_eq!(sizes, vec![255, 2]); + for batch in partitions { + let xornames: Vec = batch.iter().map(|a| XorName(*a)).collect(); + assert!( + MerkleTree::from_xornames(xornames).is_ok(), + "every partition of a 257-chunk plan must build a tree" + ); + } + } + + /// No batch may be paid for and then fail to build its tree: the partition + /// is what payment iterates, so proving every batch builds proves no + /// on-chain payment can be followed by a singleton-tree failure. + #[test] + fn no_partition_pays_before_a_singleton_tree_failure() { + for total in [257usize, 513, 769] { + let addrs = make_test_addresses(total); + for batch in merkle_batch_partitions(&addrs) { + let xornames: Vec = batch.iter().map(|a| XorName(*a)).collect(); + assert!( + MerkleTree::from_xornames(xornames).is_ok(), + "{total} addresses: batch of {} is unpayable", + batch.len() + ); + } + } + } + + #[test] + fn merkle_billable_leaves_sum_the_padded_partitions() { + for (total, expected) in PARTITION_CASES { + let padded: u64 = expected + .iter() + .map(|size| size.next_power_of_two() as u64) + .sum(); + assert_eq!( + merkle_billable_leaves(total as u64), + padded, + "{total} chunks must bill for the padded partition {expected:?}" + ); + } + + // Known figures, spelled out: padding is billed, never hidden. + assert_eq!(merkle_billable_leaves(65), 128); + assert_eq!(merkle_billable_leaves(257), 256 + 2); + assert_eq!(merkle_billable_leaves(300), 256 + 64); + // Nothing to upload costs nothing; a lone chunk still quotes the + // two-leaf minimum a tree needs. + assert_eq!(merkle_billable_leaves(0), 0); + assert_eq!(merkle_billable_leaves(1), 2); + } + + #[test] + fn merkle_billable_leaves_never_under_quote() { + for chunks in 1..2000u64 { + assert!( + merkle_billable_leaves(chunks) >= chunks, + "{chunks} chunks must never be billed as fewer leaves" + ); + } + } + + /// The external signer prepares one tree, signs once, and pays once, so an + /// oversized set is refused up front rather than being quietly paid under a + /// different model. + #[test] + fn external_preparation_refuses_more_than_one_tree_of_addresses() { + assert!(ensure_single_merkle_tree_batch(2).is_ok()); + assert!(ensure_single_merkle_tree_batch(MAX_LEAVES).is_ok()); + + for oversized in [MAX_LEAVES + 1, 300, 513] { + match ensure_single_merkle_tree_batch(oversized) { + Err(Error::MerkleBatchTooLarge { + addresses, + max_leaves, + }) => { + assert_eq!(addresses, oversized); + assert_eq!(max_leaves, MAX_LEAVES); + } + other => panic!("{oversized} addresses should be refused, got {other:?}"), + } + } } // ========================================================================= diff --git a/ant-core/src/data/client/mod.rs b/ant-core/src/data/client/mod.rs index 224afaf..7a56edf 100644 --- a/ant-core/src/data/client/mod.rs +++ b/ant-core/src/data/client/mod.rs @@ -95,6 +95,10 @@ pub(crate) fn classify_error(err: &Error) -> Outcome { | Error::Cancelled(_) | Error::BadQuoteBinding { .. } | Error::BadQuoteCommitment { .. } + // An external-signer merkle batch larger than one tree can hold — + // a caller-shape refusal raised before any network work, so it says + // nothing about link capacity. + | Error::MerkleBatchTooLarge { .. } // A remote node responded with a structured rejection — the // transport round-trip succeeded, so the node declined at the // application layer (payment/disk/quote/pool). Not a local @@ -702,6 +706,15 @@ mod tests { Error::CloseGroupShortfall("Stored on 3 peers, need 4".to_string()), Outcome::ApplicationError, ), + // Refusing an oversized external-signer merkle batch happens + // before any network work, so it is not a capacity signal. + ( + Error::MerkleBatchTooLarge { + addresses: 257, + max_leaves: 256, + }, + Outcome::ApplicationError, + ), ]; for (err, expected) in &cases { let got = classify_error(err); @@ -785,6 +798,7 @@ mod tests { | Error::PartialUpload { .. } | Error::BadQuoteBinding { .. } | Error::BadQuoteCommitment { .. } + | Error::MerkleBatchTooLarge { .. } | Error::RemotePut { .. } | Error::CloseGroupShortfall(_) => (), }; diff --git a/ant-core/src/data/error.rs b/ant-core/src/data/error.rs index 8039e30..552277f 100644 --- a/ant-core/src/data/error.rs +++ b/ant-core/src/data/error.rs @@ -147,6 +147,25 @@ pub enum Error { #[error("insufficient disk space: {0}")] InsufficientDiskSpace(String), + /// An external-signer merkle preparation was handed more addresses than a + /// single merkle tree can hold. + /// + /// The wallet path splits an oversized upload into several trees and pays + /// each in its own transaction. The external-signer contract is one + /// prepared batch → one signature → one payment, so that split has no + /// representation there. Raised before any candidate collection or + /// on-chain spend, rather than silently paying under a different model. + #[error( + "merkle batch of {addresses} addresses exceeds the {max_leaves}-leaf limit of a single \ + merkle tree; external signing cannot span multiple payment transactions" + )] + MerkleBatchTooLarge { + /// Number of addresses the caller asked to prepare. + addresses: usize, + /// Maximum leaves one merkle tree can hold (`MAX_LEAVES`). + max_leaves: usize, + }, + /// Cost estimation could not reach a representative quote. /// /// Returned by [`crate::data::Client::estimate_upload_cost`] when every @@ -329,6 +348,19 @@ mod tests { ); } + #[test] + fn test_display_merkle_batch_too_large() { + let err = Error::MerkleBatchTooLarge { + addresses: 257, + max_leaves: 256, + }; + assert_eq!( + err.to_string(), + "merkle batch of 257 addresses exceeds the 256-leaf limit of a single merkle tree; \ + external signing cannot span multiple payment transactions" + ); + } + #[test] fn test_display_cost_estimation_inconclusive() { let err = Error::CostEstimationInconclusive( diff --git a/ant-core/tests/e2e_cost_estimate.rs b/ant-core/tests/e2e_cost_estimate.rs index 40cb299..c704e89 100644 --- a/ant-core/tests/e2e_cost_estimate.rs +++ b/ant-core/tests/e2e_cost_estimate.rs @@ -10,7 +10,7 @@ mod support; -use ant_core::data::client::merkle::PaymentMode; +use ant_core::data::client::merkle::{merkle_billable_leaves, PaymentMode}; use ant_core::data::{Client, ClientConfig, CostEstimateConfidence}; use serial_test::serial; use std::io::Write; @@ -341,3 +341,66 @@ async fn test_estimate_all_stored_full_sample_is_verified() { CostEstimateConfidence::VerifiedAllAlreadyStored ); } + +/// Merkle estimate vs actual on a batch whose leaf count is padded. +/// +/// The estimator bills `median x 3` per *padded* leaf, so a 9-chunk merkle +/// batch must quote — and the contract must settle — 16 leaves. Sizing the +/// file at `6 x MAX_CHUNK_SIZE` makes the count exact: self-encryption emits +/// 6 data chunks, and `shrink_data_map` adds 3 more for any file above 3 +/// chunks, giving 9. `PaymentMode::Merkle` forces the merkle path at a chunk +/// count small enough to upload in ~1 minute. +/// +/// The 65- and 257-chunk batch-boundary cases are proved directly against +/// on-chain payment in `e2e_merkle::test_merkle_payment_across_batch_boundary`, +/// which needs no multi-hundred-megabyte file to reach those counts. +#[tokio::test(flavor = "multi_thread")] +#[serial] +async fn test_merkle_estimate_vs_actual_pads_to_the_billed_tree() { + // 20+ nodes so merkle candidate pools (CANDIDATES_PER_POOL = 16) fill. + let testnet = MiniTestnet::start(20).await; + let node = testnet.node(3).expect("Node 3 should exist"); + let client = Client::from_node(Arc::clone(&node), ClientConfig::default()) + .with_wallet(testnet.wallet().clone()); + + let work_dir = TempDir::new().expect("create work dir"); + let file_size = 6 * self_encryption::MAX_CHUNK_SIZE as u64; + let path = create_test_file(work_dir.path(), file_size, "merkle_padded.bin", 0xEE09_0001); + + let (est_atto, act_atto, est_chunks, act_chunks) = + compare_estimate_vs_actual(&client, &path, PaymentMode::Merkle).await; + + eprintln!( + "merkle padded batch: est_chunks={est_chunks}, act_chunks={act_chunks}, \ + billable_leaves={}, est={est_atto} atto, actual={act_atto} atto", + merkle_billable_leaves(est_chunks as u64) + ); + + assert_eq!( + est_chunks, 9, + "6 data chunks + 3 shrunk DataMap chunks should give 9 chunks" + ); + assert_eq!( + est_chunks, act_chunks, + "estimate and upload must agree on the chunk count" + ); + assert_eq!( + merkle_billable_leaves(est_chunks as u64), + 16, + "9 chunks must be billed as a padded 16-leaf tree" + ); + + assert!( + act_atto > 0, + "a merkle upload must settle a non-zero amount" + ); + let ratio = if est_atto > act_atto { + est_atto as f64 / act_atto as f64 + } else { + act_atto as f64 / est_atto as f64 + }; + assert!( + ratio < 1.15, + "merkle estimate too far from actual: est={est_atto}, actual={act_atto}, ratio={ratio:.2}" + ); +} diff --git a/ant-core/tests/e2e_merkle.rs b/ant-core/tests/e2e_merkle.rs index 9c33cc9..eebc947 100644 --- a/ant-core/tests/e2e_merkle.rs +++ b/ant-core/tests/e2e_merkle.rs @@ -12,7 +12,7 @@ mod support; -use ant_core::data::client::merkle::PaymentMode; +use ant_core::data::client::merkle::{merkle_billable_leaves, PaymentMode}; use ant_core::data::{compute_address, Client, ClientConfig}; use serial_test::serial; use std::io::Write; @@ -281,6 +281,130 @@ async fn test_attack_merkle_proof_swap_within_batch() { testnet.teardown().await; } +/// Merkle payment either side of the 256-leaf batch boundary, against a real +/// on-chain settlement. +/// +/// 257 is the count the old `addresses.chunks(MAX_LEAVES)` split partitioned +/// as `[256, 1]`: the 256-address batch was paid for on-chain and the +/// singleton remainder could not build a tree, so the call returned a *paid* +/// partial result carrying 256 proofs. It now splits as `[255, 2]` and every +/// address comes back with a proof. +/// +/// Paying directly is what makes the boundary reachable: getting 257 chunks +/// out of self-encryption needs a ~1 GB file, while `pay_for_merkle_batch` +/// takes the address set straight. +/// +/// Settlement is checked against the same padded-leaf model the estimator +/// bills with — 65 addresses settle 128 leaves, 257 settle 256 + 2 — so the +/// two counts must cost in that ratio. +#[tokio::test(flavor = "multi_thread")] +#[serial] +async fn test_merkle_payment_across_batch_boundary() { + let (client, testnet) = setup_merkle_testnet().await; + + // Distinct, deterministic addresses. Payment binds to addresses only; the + // chunks themselves are never stored by this test. + let addresses = |count: usize, tag: u8| -> Vec<[u8; 32]> { + (0..count) + .map(|i| { + let mut addr = [0u8; 32]; + addr[0] = tag; + addr[1..9].copy_from_slice(&(i as u64).to_be_bytes()); + addr + }) + .collect() + }; + + let mut paid: Vec<(usize, u128)> = Vec::new(); + + for (count, tag) in [(65usize, 0xA1u8), (257usize, 0xB2u8)] { + let addrs = addresses(count, tag); + + // A 65/257-address tree collects far more candidate pools than the + // small-tree tests above, and an in-process 35-node testnet on a loaded + // CI runner can leave one pool a candidate short ("Got 15 merkle + // candidates, need 16"). That shortfall is transient, so retry it. + // The bug this test guards is not: a `[256, 1]` partition fails to + // build its second tree on every attempt, so a short proof set + // survives all three and still fails the assertion below. + let mut result = None; + let mut last_shortfall = String::new(); + for attempt in 1..=3 { + eprintln!("Paying for {count} addresses via merkle batch (attempt {attempt}/3)..."); + match client + .pay_for_merkle_batch(&addrs, 0, TEST_CHUNK_SIZE as u64) + .await + { + Ok(full) if full.proofs.len() == count => { + result = Some(full); + break; + } + Ok(partial) => { + last_shortfall = format!( + "paid but returned {} of {count} proofs", + partial.proofs.len() + ); + } + Err(e) => last_shortfall = e.to_string(), + } + eprintln!(" attempt {attempt}/3 fell short: {last_shortfall}"); + } + let result = result.unwrap_or_else(|| { + panic!( + "merkle payment for {count} addresses never returned a full proof set \ + after 3 attempts: {last_shortfall}" + ) + }); + + assert_eq!( + result.proofs.len(), + count, + "every one of the {count} addresses must come back with a proof, not just the \ + sub-batches that fit a tree" + ); + assert_eq!(result.chunk_count, count); + for addr in &addrs { + assert!( + result.proofs.contains_key(addr), + "missing proof for {}", + hex::encode(addr) + ); + } + + let settled: u128 = result + .storage_cost_atto + .parse() + .expect("settled amount should parse"); + assert!( + settled > 0, + "{count} addresses must settle a non-zero amount" + ); + eprintln!( + " {count} addresses: {} leaves billed, settled {settled} atto", + merkle_billable_leaves(count as u64) + ); + paid.push((count, settled)); + } + + // Prices are uniform across a freshly started local testnet and this test + // stores nothing, so the only thing separating the two settlements is the + // padded leaf count each partition pays for. + let [(small, small_atto), (large, large_atto)] = paid[..] else { + panic!("expected two payments"); + }; + let expected = + merkle_billable_leaves(large as u64) as f64 / merkle_billable_leaves(small as u64) as f64; + let observed = large_atto as f64 / small_atto as f64; + assert!( + (observed - expected).abs() / expected < 0.15, + "settlement should scale with padded leaves: expected ~{expected:.3}x \ + ({small} -> {large} addresses), observed {observed:.3}x" + ); + + drop(client); + testnet.teardown().await; +} + // Single-node coexistence is tested in e2e_file.rs (DEFAULT_NODE_COUNT testnet). // The 35-node testnet's DHT can have sparse XOR regions where single-node // quotes can't find 5 peers for a random chunk address, making that test diff --git a/ant-core/tests/merkle_unit.rs b/ant-core/tests/merkle_unit.rs index dd9138d..335b9e7 100644 --- a/ant-core/tests/merkle_unit.rs +++ b/ant-core/tests/merkle_unit.rs @@ -1,11 +1,18 @@ //! Merkle payment unit tests. //! -//! These tests use the free function `should_use_merkle` — no Client or network needed. -//! The real tests are in `src/client/merkle.rs` (inline test module). +//! These tests exercise the free functions — `should_use_merkle` and the +//! batch partitioning that payment and cost estimation share — so no Client or +//! network is needed. The rest live in `src/data/client/merkle.rs` (inline test +//! module). #![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)] -use ant_core::data::client::merkle::{should_use_merkle, PaymentMode, DEFAULT_MERKLE_THRESHOLD}; +use ant_core::data::client::merkle::{ + merkle_batch_partitions, merkle_batch_sizes, merkle_billable_leaves, should_use_merkle, + PaymentMode, DEFAULT_MERKLE_THRESHOLD, +}; +use ant_protocol::evm::MAX_LEAVES; +use std::collections::HashSet; #[test] fn test_threshold_constant() { @@ -28,3 +35,86 @@ fn test_merkle_mode() { fn test_single_mode() { assert!(!should_use_merkle(1000, PaymentMode::Single)); } + +/// Counts covering the minimum tree, the merkle threshold, either side of a +/// full batch, and the `1 mod MAX_LEAVES` counts a naive `chunks(MAX_LEAVES)` +/// split turned into an unpayable one-address tail. +const PARTITION_CASES: [(usize, &[usize]); 10] = [ + (2, &[2]), + (64, &[64]), + (65, &[65]), + (255, &[255]), + (256, &[256]), + (257, &[255, 2]), + (300, &[256, 44]), + (512, &[256, 256]), + (513, &[256, 255, 2]), + (769, &[256, 256, 255, 2]), +]; + +fn addresses(count: usize) -> Vec<[u8; 32]> { + (0..count) + .map(|i| { + let mut addr = [0u8; 32]; + addr[..8].copy_from_slice(&(i as u64).to_be_bytes()); + addr + }) + .collect() +} + +#[test] +fn batch_sizes_rebalance_singleton_remainders() { + for (total, expected) in PARTITION_CASES { + assert_eq!( + merkle_batch_sizes(total), + expected, + "{total} addresses must partition as {expected:?}, never with a one-address batch" + ); + } +} + +#[test] +fn batch_sizes_are_valid_tree_sizes_and_cover_the_input() { + for total in 2..=(3 * MAX_LEAVES + 5) { + let sizes = merkle_batch_sizes(total); + assert_eq!(sizes.iter().sum::(), total); + for size in sizes { + assert!( + (2..=MAX_LEAVES).contains(&size), + "{total} addresses produced a batch of {size}, outside 2..={MAX_LEAVES}" + ); + } + } +} + +#[test] +fn partitions_preserve_order_and_use_each_address_once() { + for (total, expected) in PARTITION_CASES { + let addrs = addresses(total); + let partitions = merkle_batch_partitions(&addrs); + + let sizes: Vec = partitions.iter().map(|batch| batch.len()).collect(); + assert_eq!(sizes, expected); + + let flattened: Vec<[u8; 32]> = partitions.concat(); + assert_eq!(flattened, addrs, "{total}: order must be preserved"); + + let unique: HashSet<[u8; 32]> = flattened.iter().copied().collect(); + assert_eq!(unique.len(), total, "{total}: no address may repeat"); + } +} + +#[test] +fn billable_leaves_are_the_padded_partitions() { + for (total, expected) in PARTITION_CASES { + let padded: u64 = expected + .iter() + .map(|size| size.next_power_of_two() as u64) + .sum(); + assert_eq!( + merkle_billable_leaves(total as u64), + padded, + "{total} chunks must bill for the partition {expected:?}, padded" + ); + } +}