diff --git a/ant-core/src/data/client/file.rs b/ant-core/src/data/client/file.rs index f2fe43c..42bc60d 100644 --- a/ant-core/src/data/client/file.rs +++ b/ant-core/src/data/client/file.rs @@ -17,14 +17,14 @@ use crate::data::client::batch::{ use crate::data::client::chunk::ChunkPeerGetResult; use crate::data::client::classify_error; use crate::data::client::merkle::{ - chunk_contents_for_upload_addresses, finalize_merkle_batch, merkle_deferred_retry, - merkle_store_with_retry, should_use_merkle, MerkleBatchPaymentResult, PaymentMode, - PreparedMerkleBatch, DEFERRED_ROUND_DELAYS_SECS, + chunk_contents_for_upload_addresses, finalize_merkle_batch, merkle_batch_sizes, + merkle_billable_leaves, merkle_deferred_retry, merkle_store_with_retry, should_use_merkle, + MerkleBatchPaymentResult, PaymentMode, PreparedMerkleBatch, DEFERRED_ROUND_DELAYS_SECS, }; use crate::data::client::payment::SINGLE_NODE_PAYMENT_MULTIPLIER; use crate::data::client::Client; use crate::data::error::{Error, PartialUploadSpend, Result}; -use ant_protocol::evm::{Amount, PaymentQuote, QuoteHash, TxHash, MAX_LEAVES}; +use ant_protocol::evm::{Amount, PaymentQuote, QuoteHash, TxHash}; use ant_protocol::transport::{MultiAddr, PeerId}; use ant_protocol::{compute_address, XorName as ChunkAddress, DATA_TYPE_CHUNK}; use bytes::Bytes; @@ -1343,8 +1343,8 @@ impl Client { } }; - // Use the median price × 3 (matches the single-node payment builder - // which pays 3x the median to incentivize reliable storage). + // Use the median price × 3, matching the settlement multiplier both + // payment paths now apply. let mut prices: Vec = quotes.iter().map(|(_, _, _, price, _)| *price).collect(); prices.sort(); let median_price = prices @@ -1354,7 +1354,17 @@ impl Client { let per_chunk_cost = median_price * Amount::from(SINGLE_NODE_PAYMENT_MULTIPLIER); let chunk_count_u64 = u64::try_from(chunk_count).unwrap_or(u64::MAX); - let total_storage = per_chunk_cost * Amount::from(chunk_count_u64); + // Merkle settles per *padded* leaf, not per chunk: the contract charges + // `median16 × 2^depth` per batch and the tree rounds up to a power of + // two, so a 65-chunk batch pays for 128 leaves. The leaf total is + // summed over the batches the payment path really builds + // (`merkle_batch_sizes`), so the estimate cannot drift from execution. + let billable_units = if uses_merkle { + merkle_billable_leaves(chunk_count_u64) + } else { + chunk_count_u64 + }; + let total_storage = per_chunk_cost * Amount::from(billable_units); // Estimate gas cost from realistic per-transaction budgets rather // than a flat per-chunk or per-wave number. @@ -1372,7 +1382,10 @@ impl Client { // Gas is priced at ARBITRUM_GAS_PRICE_WEI (~0.1 gwei, a typical // Arbitrum baseline). Treat the result as advisory, not a commitment. let waves = u128::try_from(chunk_count.div_ceil(UPLOAD_WAVE_SIZE)).unwrap_or(u128::MAX); - let merkle_batches = u128::try_from(chunk_count.div_ceil(MAX_LEAVES)).unwrap_or(u128::MAX); + // One tx per batch the payment path builds — same partition the leaf + // total above is derived from. + let merkle_batches = + u128::try_from(merkle_batch_sizes(chunk_count).len()).unwrap_or(u128::MAX); let estimated_gas: u128 = if uses_merkle { merkle_batches .saturating_mul(GAS_PER_MERKLE_TX) @@ -1567,6 +1580,11 @@ impl Client { already_stored.append(&mut wave_already_stored); (payment_info, already_stored) } else { + // One prepared batch is one signature and one payment, so + // more than MAX_LEAVES addresses is refused with + // `MerkleBatchTooLarge` before any candidate collection — + // the wallet path's multi-transaction split has no + // external-signing equivalent to fall back on. match self .prepare_merkle_batch_external( &merkle_plan.to_upload, @@ -3451,6 +3469,24 @@ mod tests { assert_eq!(distributed_sample_indices(5, 5), vec![0, 1, 2, 3, 4]); } + /// The estimator bills the padded tree, and the leaf total it bills comes + /// from the batches the payment path really builds — a `[255, 2]` split of + /// 257 chunks, not a `[256, 1]` one that could never be paid. + #[test] + fn estimator_leaf_total_is_the_padded_payment_partition() { + for chunks in [2u64, 64, 65, 100, 129, 255, 256, 257, 300, 512, 513, 769] { + let from_partition: u64 = merkle_batch_sizes(chunks as usize) + .into_iter() + .map(|size| size.next_power_of_two() as u64) + .sum(); + assert_eq!( + merkle_billable_leaves(chunks), + from_partition, + "{chunks} chunks must be billed for the partition the payment path pays" + ); + } + } + #[test] fn distributed_sample_indices_is_in_range_and_increasing() { assert!(distributed_sample_indices(0, 5).is_empty()); diff --git a/ant-core/src/data/client/merkle.rs b/ant-core/src/data/client/merkle.rs index af0e44b..2973c7e 100644 --- a/ant-core/src/data/client/merkle.rs +++ b/ant-core/src/data/client/merkle.rs @@ -37,6 +37,29 @@ use xor_name::XorName; /// Default threshold: use merkle payments when chunk count >= this value. pub const DEFAULT_MERKLE_THRESHOLD: usize = 64; +/// Payment multiplier applied to a quoted price before settlement. +/// +/// Deliberately the **same constant** the single-node path uses rather than a +/// second copy of `3`: the whole defect this fixes was the two paths disagreeing +/// about the multiplier, so they now read it from one place. (The storer's +/// `PAID_QUOTE_PAYMENT_MULTIPLIER` and `ant_protocol`'s single-node builder are +/// still separate literals; folding all of them into one `ant-protocol` +/// constant is a follow-up.) +/// +/// The single-node path pays the median-priced issuer 3× its quoted price, so +/// the network receives the same revenue as paying three members of the close +/// group while costing one transaction's gas. The merkle path never applied it: +/// it submitted the raw quoted price as the on-chain payable amount, so the +/// contract's `median16(amount) × 2^depth` came to **1×** the median per padded +/// leaf — a third of what the same chunk earns on the single-node path, for +/// identical storage and replication. +/// +/// The on-chain field is `CandidateNode.amount`, the sum the vault pays out, +/// not the quote itself; the signed candidate keeps its 1× quoted `price` and +/// the pool hash is unchanged, so every proof still verifies against the +/// quotes the nodes actually signed. +use crate::data::client::payment::SINGLE_NODE_PAYMENT_MULTIPLIER as MERKLE_PAYMENT_MULTIPLIER; + /// ADR-0004 resolve-before-pay gate for a merkle candidate — the merkle-path /// equivalent of the single-node `quote_commitment_binding_is_valid`. Runs the /// FULL binding check (shape, cap, exact price, and for bound candidates the @@ -111,6 +134,34 @@ fn merkle_candidate_binding_is_valid( Ok(()) } +/// Build the on-chain [`PoolCommitment`] for a candidate pool, applying +/// [`MERKLE_PAYMENT_MULTIPLIER`] to every candidate's payable amount. +/// +/// The contract derives what it pays out from the amounts submitted here +/// (`total = median16(amount) × 2^depth`, split evenly across `depth` +/// winners), so multiplying here — and only here — brings the merkle path to +/// the same per-chunk revenue as the single-node path. +/// +/// The pool hash is deliberately left as `pool.hash()`, computed over the +/// candidates' **signed** 1× prices. It is the key the storer resolves the +/// on-chain payment record under, and it commits to the quotes the nodes +/// actually signed; multiplying the payable amount must not disturb it. +fn pool_commitment_with_payment_multiplier( + pool: &MerklePaymentCandidatePool, +) -> Result { + let mut commitment = pool.to_commitment(); + let multiplier = Amount::from(MERKLE_PAYMENT_MULTIPLIER); + for candidate in &mut commitment.candidates { + candidate.price = candidate.price.checked_mul(multiplier).ok_or_else(|| { + Error::Payment(format!( + "Merkle candidate amount overflow applying {MERKLE_PAYMENT_MULTIPLIER}x to price {}", + candidate.price + )) + })?; + } + Ok(commitment) +} + /// Payment mode for uploads. #[derive(Debug, Clone, Copy, Default, PartialEq, Eq, serde::Serialize, serde::Deserialize)] #[serde(rename_all = "snake_case")] @@ -304,6 +355,118 @@ fn preflight_stored_status(result: Result) -> Result { } } +/// Split `total` addresses into the merkle batches an upload is actually paid in. +/// +/// Every batch becomes one `MerkleTree`, and a tree is only valid with +/// `2..=MAX_LEAVES` leaves. The obvious `addresses.chunks(MAX_LEAVES)` split +/// respects the upper bound but not the lower one: 257 addresses come out as +/// `[256, 1]`, the 256-address batch is paid for on-chain, and the singleton +/// remainder cannot build a tree — so the upload fails as a *paid* partial. +/// Every count congruent to 1 modulo `MAX_LEAVES` (257, 513, 769, …) hits it, +/// and the merkle preflight can leave an arbitrary count behind. +/// +/// Borrowing one address from the preceding batch removes the case entirely: +/// 257 splits as `[255, 2]` and 513 as `[256, 255, 2]`. Order is preserved and +/// no address is duplicated or synthesised, so the partition is a plain +/// in-order cover of the input. +/// +/// Returns an empty vector for `total < 2`, which no merkle path may pay for — +/// `pay_for_merkle_batch` rejects those counts up front. +#[must_use] +pub fn merkle_batch_sizes(total: usize) -> Vec { + if total < 2 { + return Vec::new(); + } + + let mut sizes = Vec::with_capacity(total.div_ceil(MAX_LEAVES)); + let mut remaining = total; + while remaining > MAX_LEAVES { + // Taking a full MAX_LEAVES here would strand a single address as the + // final batch; take one fewer so the tail is a payable two-leaf tree. + let take = if remaining - MAX_LEAVES == 1 { + MAX_LEAVES - 1 + } else { + MAX_LEAVES + }; + sizes.push(take); + remaining -= take; + } + sizes.push(remaining); + sizes +} + +/// Split `addresses` into the sub-batches [`merkle_batch_sizes`] describes. +/// +/// The slices borrow `addresses` in order, so the partition cannot introduce a +/// duplicate or a synthetic address. +#[must_use] +pub fn merkle_batch_partitions(addresses: &[[u8; 32]]) -> Vec<&[[u8; 32]]> { + let mut partitions = Vec::new(); + let mut rest = addresses; + for size in merkle_batch_sizes(addresses.len()) { + let (batch, tail) = rest.split_at(size); + partitions.push(batch); + rest = tail; + } + partitions +} + +/// Leaves one merkle batch of `batch_size` addresses is billed for. +/// +/// `MerkleTree` pads its leaf count up to a power of two and the vault charges +/// `median16 × 2^depth`, so a 65-address batch pays for 128 leaves. +fn padded_leaf_count(batch_size: usize) -> u64 { + // Saturate rather than wrap: a batch is at most MAX_LEAVES, so the + // overflow arm is unreachable, and erring high never under-quotes. + let padded = batch_size + .max(2) + .checked_next_power_of_two() + .unwrap_or(usize::MAX); + u64::try_from(padded).unwrap_or(u64::MAX) +} + +/// Leaves a merkle upload of `chunk_count` chunks is actually billed for. +/// +/// Summed over the batches [`merkle_batch_sizes`] will really pay for, so the +/// estimate and the execution path share one batching model rather than two +/// that can drift. Billing the raw chunk count would under-quote a +/// non-power-of-two batch by up to 2×, and an under-quote is the harmful +/// direction: a caller sizing its wallet from the estimate runs dry +/// mid-upload. +#[must_use] +pub fn merkle_billable_leaves(chunk_count: u64) -> u64 { + let total = usize::try_from(chunk_count).unwrap_or(usize::MAX); + let batches = merkle_batch_sizes(total); + if batches.is_empty() { + // No payable partition exists below two chunks. Nothing costs nothing; + // a lone chunk is still quoted as the two-leaf minimum a tree needs. + return if total == 0 { 0 } else { 2 }; + } + + batches + .into_iter() + .map(padded_leaf_count) + .fold(0u64, u64::saturating_add) +} + +/// Reject an address set that cannot be prepared as a single merkle tree. +/// +/// The wallet path splits an oversized upload with [`merkle_batch_sizes`] and +/// pays each batch in its own transaction. The external-signer contract is one +/// prepared batch → one signature → one payment, so it has no way to express +/// that split. Refusing before any candidate collection or on-chain spend is +/// the honest answer; silently switching the caller to a different payment +/// model is not. +fn ensure_single_merkle_tree_batch(address_count: usize) -> Result<()> { + if address_count > MAX_LEAVES { + return Err(Error::MerkleBatchTooLarge { + addresses: address_count, + max_leaves: MAX_LEAVES, + }); + } + Ok(()) +} + /// Determine whether to use merkle payments for a given batch size. /// Free function — no Client needed. #[must_use] @@ -325,7 +488,8 @@ impl Client { /// Pay for a batch of chunks using merkle batch payment. /// /// Builds a merkle tree, collects candidate pools, pays on-chain in one tx, - /// and returns per-chunk proofs. Splits into sub-batches if > `MAX_LEAVES`. + /// and returns per-chunk proofs. Anything longer than `MAX_LEAVES` is split + /// by [`merkle_batch_sizes`] and paid one transaction per sub-batch. /// /// This low-level helper assumes the caller has already selected the /// addresses that need payment. User-facing upload paths first run the @@ -490,12 +654,22 @@ impl Client { /// Builds the merkle tree, collects candidate pools from the network, /// and returns the data needed for the on-chain payment call. /// Requires `EvmNetwork` but NOT a wallet. + /// + /// # Errors + /// + /// Returns [`Error::MerkleBatchTooLarge`] if `addresses` holds more than + /// `MAX_LEAVES` entries. One prepared batch is one signature and one + /// payment, so an oversized set has no valid external-signing form; the + /// wallet path splits it across transactions instead. The check runs + /// before any candidate collection, so nothing is spent. pub async fn prepare_merkle_batch_external( &self, addresses: &[[u8; 32]], data_type: u32, data_size: u64, ) -> Result { + ensure_single_merkle_tree_batch(addresses.len())?; + let chunk_count = addresses.len(); let xornames: Vec = addresses.iter().map(|a| XorName(*a)).collect(); @@ -537,11 +711,14 @@ impl Client { ) .await?; - // 4. Build pool commitments for on-chain payment + // 4. Build pool commitments for on-chain payment. Every candidate's + // payable amount carries MERKLE_PAYMENT_MULTIPLIER so a merkle + // chunk settles for the same amount a single-node chunk does; the + // signed candidate prices and the pool hashes are untouched. let pool_commitments: Vec = candidate_pools .iter() - .map(MerklePaymentCandidatePool::to_commitment) - .collect(); + .map(pool_commitment_with_payment_multiplier) + .collect::>>()?; Ok(PreparedMerkleBatch { depth, @@ -596,7 +773,11 @@ impl Client { data_type: u32, data_size: u64, ) -> Result { - let sub_batches: Vec<&[[u8; 32]]> = addresses.chunks(MAX_LEAVES).collect(); + // Partition with the shared helper, NOT `chunks(MAX_LEAVES)`: the naive + // split leaves a one-address final batch for every count congruent to 1 + // modulo MAX_LEAVES, which cannot build a tree and so turns a paid + // upload into a partial failure. + let sub_batches = merkle_batch_partitions(addresses); let total_sub_batches = sub_batches.len(); let mut all_proofs = HashMap::with_capacity(addresses.len()); let mut total_storage = Amount::ZERO; @@ -1825,8 +2006,9 @@ mod tests { let pool_commitments = candidate_pools .iter() - .map(MerklePaymentCandidatePool::to_commitment) - .collect(); + .map(pool_commitment_with_payment_multiplier) + .collect::>>() + .unwrap(); PreparedMerkleBatch { depth: tree.depth(), @@ -1838,6 +2020,102 @@ mod tests { } } + /// Candidate pool with distinct prices, so the median is a specific + /// candidate rather than an artifact of every price being equal. + fn pool_with_varied_prices(timestamp: u64) -> MerklePaymentCandidatePool { + let addrs = make_test_addresses(4); + let xornames: Vec = addrs.iter().map(|a| XorName(*a)).collect(); + let tree = MerkleTree::from_xornames(xornames).unwrap(); + let midpoint = tree + .reward_candidates(timestamp) + .unwrap() + .into_iter() + .next() + .unwrap(); + + let candidate_nodes = std::array::from_fn(|i| MerklePaymentCandidateNode { + pub_key: vec![i as u8; 32], + // 100, 200, ... 1600 — upper median (index 8 of 16) is 900. + price: Amount::from((i as u64 + 1) * 100), + reward_address: RewardsAddress::new([i as u8; 20]), + merkle_payment_timestamp: timestamp, + signature: vec![i as u8; 64], + committed_key_count: 0, + commitment_pin: None, + }); + + MerklePaymentCandidatePool { + midpoint_proof: midpoint, + candidate_nodes, + } + } + + /// The contract's `median16`: upper median, index 8 of 16 ascending. + fn median16(mut amounts: Vec) -> Amount { + amounts.sort_unstable(); + *amounts.get(amounts.len() / 2).unwrap() + } + + #[test] + fn pool_commitment_applies_payment_multiplier_to_every_candidate() { + let pool = pool_with_varied_prices(1_700_000_000); + let commitment = pool_commitment_with_payment_multiplier(&pool).unwrap(); + + for (candidate, signed) in commitment + .candidates + .iter() + .zip(pool.candidate_nodes.iter()) + { + assert_eq!( + candidate.price, + signed.price * Amount::from(MERKLE_PAYMENT_MULTIPLIER), + "on-chain payable amount must be {MERKLE_PAYMENT_MULTIPLIER}x the quoted price" + ); + } + } + + #[test] + fn pool_commitment_multiplier_leaves_signed_prices_and_pool_hash_untouched() { + let pool = pool_with_varied_prices(1_700_000_000); + let before: Vec = pool.candidate_nodes.iter().map(|c| c.price).collect(); + + let commitment = pool_commitment_with_payment_multiplier(&pool).unwrap(); + + let after: Vec = pool.candidate_nodes.iter().map(|c| c.price).collect(); + assert_eq!(before, after, "signed candidate prices must not change"); + assert_eq!( + commitment.pool_hash, + pool.hash(), + "pool hash is the storer's on-chain lookup key and must be \ + computed over the signed 1x prices" + ); + } + + /// The invariant the fix exists for, stated at the level it actually + /// holds: the median payable amount over the winning pool is + /// `MERKLE_PAYMENT_MULTIPLIER x` the median *quoted* price. The contract + /// spends `median16(amount) * 2^depth` over `2^depth` **padded** leaves, so + /// this is the settlement per padded leaf — the same figure the single-node + /// path pays its median-priced issuer per chunk. It says nothing about cost + /// per *actual* chunk: a batch whose size is not a power of two pays for + /// padding leaves too (that is what `merkle_billable_leaves` bills for). + #[test] + fn merkle_settlement_per_padded_leaf_is_the_multiplied_pool_median() { + let pool = pool_with_varied_prices(1_700_000_000); + let commitment = pool_commitment_with_payment_multiplier(&pool).unwrap(); + + let quoted_median = median16(pool.candidate_nodes.iter().map(|c| c.price).collect()); + let per_chunk = median16(commitment.candidates.iter().map(|c| c.price).collect()); + + assert_eq!(quoted_median, Amount::from(900u64)); + assert_eq!( + per_chunk, + quoted_median * Amount::from(MERKLE_PAYMENT_MULTIPLIER), + "merkle per-chunk settlement must equal the single-node \ + {MERKLE_PAYMENT_MULTIPLIER}x median, not the bare quoted price" + ); + } + #[test] fn test_finalize_merkle_batch_with_valid_winner() { let prepared = make_prepared_merkle_batch(4); @@ -1927,19 +2205,179 @@ mod tests { // Batch splitting edge cases // ========================================================================= + /// Counts spanning every interesting case: the minimum tree, the merkle + /// threshold and just past it, either side of a full batch, and the + /// `1 mod MAX_LEAVES` counts the naive `chunks(MAX_LEAVES)` split turned + /// into an unpayable `[..., 1]` tail. + const PARTITION_CASES: [(usize, &[usize]); 10] = [ + (2, &[2]), + (64, &[64]), + (65, &[65]), + (255, &[255]), + (256, &[256]), + (257, &[255, 2]), + (300, &[256, 44]), + (512, &[256, 256]), + (513, &[256, 255, 2]), + (769, &[256, 256, 255, 2]), + ]; + #[test] - fn test_batch_split_calculation() { - // MAX_LEAVES chunks should fit in 1 batch - let addrs = make_test_addresses(MAX_LEAVES); - assert_eq!(addrs.chunks(MAX_LEAVES).count(), 1); + fn merkle_batch_sizes_rebalance_singleton_remainders() { + for (total, expected) in PARTITION_CASES { + assert_eq!( + merkle_batch_sizes(total), + expected, + "{total} addresses must partition as {expected:?}" + ); + } + } + + /// The defect: `[256, 1]` pays the first batch on-chain and then hands a + /// single address to a tree that needs two, so the upload fails *after* + /// spending. Every count must produce trees that can all be built. + #[test] + fn merkle_batch_sizes_are_always_buildable_trees() { + for total in 2..=(4 * MAX_LEAVES + 3) { + let sizes = merkle_batch_sizes(total); + assert!(!sizes.is_empty(), "{total} addresses must produce batches"); + assert_eq!( + sizes.iter().sum::(), + total, + "{total} addresses: partition must cover every address" + ); + for size in sizes { + assert!( + (2..=MAX_LEAVES).contains(&size), + "{total} addresses produced a batch of {size}, outside 2..={MAX_LEAVES}" + ); + } + } + } + + #[test] + fn merkle_batch_sizes_below_two_have_no_payable_partition() { + assert!(merkle_batch_sizes(0).is_empty()); + assert!(merkle_batch_sizes(1).is_empty()); + } + + #[test] + fn merkle_batch_partitions_preserve_order_and_use_each_address_once() { + for (total, _) in PARTITION_CASES { + let addrs = make_test_addresses(total); + let partitions = merkle_batch_partitions(&addrs); + + let flattened: Vec<[u8; 32]> = partitions.concat(); + assert_eq!( + flattened, addrs, + "{total} addresses: partitions must concatenate back to the input in order" + ); + + let unique: std::collections::HashSet<[u8; 32]> = flattened.iter().copied().collect(); + assert_eq!( + unique.len(), + total, + "{total} addresses: no address may be duplicated or synthesised" + ); + } + } + + /// A merkle upload of 257 chunks — the count the old split could not pay — + /// is the shape preflight routinely leaves behind, since `to_upload` is + /// whatever the network did not already hold. + #[test] + fn post_preflight_plan_of_257_partitions_into_payable_batches() { + let plan = MerkleUploadPlan { + already_stored: make_test_addresses(3), + to_upload: make_test_addresses(257), + to_upload_total_bytes: 257 * 1024, + }; + assert_eq!(plan.to_upload.len(), 257); + + let partitions = merkle_batch_partitions(&plan.to_upload); + let sizes: Vec = partitions.iter().map(|batch| batch.len()).collect(); + assert_eq!(sizes, vec![255, 2]); + for batch in partitions { + let xornames: Vec = batch.iter().map(|a| XorName(*a)).collect(); + assert!( + MerkleTree::from_xornames(xornames).is_ok(), + "every partition of a 257-chunk plan must build a tree" + ); + } + } + + /// No batch may be paid for and then fail to build its tree: the partition + /// is what payment iterates, so proving every batch builds proves no + /// on-chain payment can be followed by a singleton-tree failure. + #[test] + fn no_partition_pays_before_a_singleton_tree_failure() { + for total in [257usize, 513, 769] { + let addrs = make_test_addresses(total); + for batch in merkle_batch_partitions(&addrs) { + let xornames: Vec = batch.iter().map(|a| XorName(*a)).collect(); + assert!( + MerkleTree::from_xornames(xornames).is_ok(), + "{total} addresses: batch of {} is unpayable", + batch.len() + ); + } + } + } + + #[test] + fn merkle_billable_leaves_sum_the_padded_partitions() { + for (total, expected) in PARTITION_CASES { + let padded: u64 = expected + .iter() + .map(|size| size.next_power_of_two() as u64) + .sum(); + assert_eq!( + merkle_billable_leaves(total as u64), + padded, + "{total} chunks must bill for the padded partition {expected:?}" + ); + } + + // Known figures, spelled out: padding is billed, never hidden. + assert_eq!(merkle_billable_leaves(65), 128); + assert_eq!(merkle_billable_leaves(257), 256 + 2); + assert_eq!(merkle_billable_leaves(300), 256 + 64); + // Nothing to upload costs nothing; a lone chunk still quotes the + // two-leaf minimum a tree needs. + assert_eq!(merkle_billable_leaves(0), 0); + assert_eq!(merkle_billable_leaves(1), 2); + } - // MAX_LEAVES + 1 should split into 2 - let addrs = make_test_addresses(MAX_LEAVES + 1); - assert_eq!(addrs.chunks(MAX_LEAVES).count(), 2); + #[test] + fn merkle_billable_leaves_never_under_quote() { + for chunks in 1..2000u64 { + assert!( + merkle_billable_leaves(chunks) >= chunks, + "{chunks} chunks must never be billed as fewer leaves" + ); + } + } - // 3 * MAX_LEAVES should split into 3 - let addrs = make_test_addresses(3 * MAX_LEAVES); - assert_eq!(addrs.chunks(MAX_LEAVES).count(), 3); + /// The external signer prepares one tree, signs once, and pays once, so an + /// oversized set is refused up front rather than being quietly paid under a + /// different model. + #[test] + fn external_preparation_refuses_more_than_one_tree_of_addresses() { + assert!(ensure_single_merkle_tree_batch(2).is_ok()); + assert!(ensure_single_merkle_tree_batch(MAX_LEAVES).is_ok()); + + for oversized in [MAX_LEAVES + 1, 300, 513] { + match ensure_single_merkle_tree_batch(oversized) { + Err(Error::MerkleBatchTooLarge { + addresses, + max_leaves, + }) => { + assert_eq!(addresses, oversized); + assert_eq!(max_leaves, MAX_LEAVES); + } + other => panic!("{oversized} addresses should be refused, got {other:?}"), + } + } } // ========================================================================= diff --git a/ant-core/src/data/client/mod.rs b/ant-core/src/data/client/mod.rs index 224afaf..7a56edf 100644 --- a/ant-core/src/data/client/mod.rs +++ b/ant-core/src/data/client/mod.rs @@ -95,6 +95,10 @@ pub(crate) fn classify_error(err: &Error) -> Outcome { | Error::Cancelled(_) | Error::BadQuoteBinding { .. } | Error::BadQuoteCommitment { .. } + // An external-signer merkle batch larger than one tree can hold — + // a caller-shape refusal raised before any network work, so it says + // nothing about link capacity. + | Error::MerkleBatchTooLarge { .. } // A remote node responded with a structured rejection — the // transport round-trip succeeded, so the node declined at the // application layer (payment/disk/quote/pool). Not a local @@ -702,6 +706,15 @@ mod tests { Error::CloseGroupShortfall("Stored on 3 peers, need 4".to_string()), Outcome::ApplicationError, ), + // Refusing an oversized external-signer merkle batch happens + // before any network work, so it is not a capacity signal. + ( + Error::MerkleBatchTooLarge { + addresses: 257, + max_leaves: 256, + }, + Outcome::ApplicationError, + ), ]; for (err, expected) in &cases { let got = classify_error(err); @@ -785,6 +798,7 @@ mod tests { | Error::PartialUpload { .. } | Error::BadQuoteBinding { .. } | Error::BadQuoteCommitment { .. } + | Error::MerkleBatchTooLarge { .. } | Error::RemotePut { .. } | Error::CloseGroupShortfall(_) => (), }; diff --git a/ant-core/src/data/error.rs b/ant-core/src/data/error.rs index 8039e30..552277f 100644 --- a/ant-core/src/data/error.rs +++ b/ant-core/src/data/error.rs @@ -147,6 +147,25 @@ pub enum Error { #[error("insufficient disk space: {0}")] InsufficientDiskSpace(String), + /// An external-signer merkle preparation was handed more addresses than a + /// single merkle tree can hold. + /// + /// The wallet path splits an oversized upload into several trees and pays + /// each in its own transaction. The external-signer contract is one + /// prepared batch → one signature → one payment, so that split has no + /// representation there. Raised before any candidate collection or + /// on-chain spend, rather than silently paying under a different model. + #[error( + "merkle batch of {addresses} addresses exceeds the {max_leaves}-leaf limit of a single \ + merkle tree; external signing cannot span multiple payment transactions" + )] + MerkleBatchTooLarge { + /// Number of addresses the caller asked to prepare. + addresses: usize, + /// Maximum leaves one merkle tree can hold (`MAX_LEAVES`). + max_leaves: usize, + }, + /// Cost estimation could not reach a representative quote. /// /// Returned by [`crate::data::Client::estimate_upload_cost`] when every @@ -329,6 +348,19 @@ mod tests { ); } + #[test] + fn test_display_merkle_batch_too_large() { + let err = Error::MerkleBatchTooLarge { + addresses: 257, + max_leaves: 256, + }; + assert_eq!( + err.to_string(), + "merkle batch of 257 addresses exceeds the 256-leaf limit of a single merkle tree; \ + external signing cannot span multiple payment transactions" + ); + } + #[test] fn test_display_cost_estimation_inconclusive() { let err = Error::CostEstimationInconclusive( diff --git a/ant-core/tests/e2e_cost_estimate.rs b/ant-core/tests/e2e_cost_estimate.rs index 40cb299..c704e89 100644 --- a/ant-core/tests/e2e_cost_estimate.rs +++ b/ant-core/tests/e2e_cost_estimate.rs @@ -10,7 +10,7 @@ mod support; -use ant_core::data::client::merkle::PaymentMode; +use ant_core::data::client::merkle::{merkle_billable_leaves, PaymentMode}; use ant_core::data::{Client, ClientConfig, CostEstimateConfidence}; use serial_test::serial; use std::io::Write; @@ -341,3 +341,66 @@ async fn test_estimate_all_stored_full_sample_is_verified() { CostEstimateConfidence::VerifiedAllAlreadyStored ); } + +/// Merkle estimate vs actual on a batch whose leaf count is padded. +/// +/// The estimator bills `median x 3` per *padded* leaf, so a 9-chunk merkle +/// batch must quote — and the contract must settle — 16 leaves. Sizing the +/// file at `6 x MAX_CHUNK_SIZE` makes the count exact: self-encryption emits +/// 6 data chunks, and `shrink_data_map` adds 3 more for any file above 3 +/// chunks, giving 9. `PaymentMode::Merkle` forces the merkle path at a chunk +/// count small enough to upload in ~1 minute. +/// +/// The 65- and 257-chunk batch-boundary cases are proved directly against +/// on-chain payment in `e2e_merkle::test_merkle_payment_across_batch_boundary`, +/// which needs no multi-hundred-megabyte file to reach those counts. +#[tokio::test(flavor = "multi_thread")] +#[serial] +async fn test_merkle_estimate_vs_actual_pads_to_the_billed_tree() { + // 20+ nodes so merkle candidate pools (CANDIDATES_PER_POOL = 16) fill. + let testnet = MiniTestnet::start(20).await; + let node = testnet.node(3).expect("Node 3 should exist"); + let client = Client::from_node(Arc::clone(&node), ClientConfig::default()) + .with_wallet(testnet.wallet().clone()); + + let work_dir = TempDir::new().expect("create work dir"); + let file_size = 6 * self_encryption::MAX_CHUNK_SIZE as u64; + let path = create_test_file(work_dir.path(), file_size, "merkle_padded.bin", 0xEE09_0001); + + let (est_atto, act_atto, est_chunks, act_chunks) = + compare_estimate_vs_actual(&client, &path, PaymentMode::Merkle).await; + + eprintln!( + "merkle padded batch: est_chunks={est_chunks}, act_chunks={act_chunks}, \ + billable_leaves={}, est={est_atto} atto, actual={act_atto} atto", + merkle_billable_leaves(est_chunks as u64) + ); + + assert_eq!( + est_chunks, 9, + "6 data chunks + 3 shrunk DataMap chunks should give 9 chunks" + ); + assert_eq!( + est_chunks, act_chunks, + "estimate and upload must agree on the chunk count" + ); + assert_eq!( + merkle_billable_leaves(est_chunks as u64), + 16, + "9 chunks must be billed as a padded 16-leaf tree" + ); + + assert!( + act_atto > 0, + "a merkle upload must settle a non-zero amount" + ); + let ratio = if est_atto > act_atto { + est_atto as f64 / act_atto as f64 + } else { + act_atto as f64 / est_atto as f64 + }; + assert!( + ratio < 1.15, + "merkle estimate too far from actual: est={est_atto}, actual={act_atto}, ratio={ratio:.2}" + ); +} diff --git a/ant-core/tests/e2e_merkle.rs b/ant-core/tests/e2e_merkle.rs index 9c33cc9..eebc947 100644 --- a/ant-core/tests/e2e_merkle.rs +++ b/ant-core/tests/e2e_merkle.rs @@ -12,7 +12,7 @@ mod support; -use ant_core::data::client::merkle::PaymentMode; +use ant_core::data::client::merkle::{merkle_billable_leaves, PaymentMode}; use ant_core::data::{compute_address, Client, ClientConfig}; use serial_test::serial; use std::io::Write; @@ -281,6 +281,130 @@ async fn test_attack_merkle_proof_swap_within_batch() { testnet.teardown().await; } +/// Merkle payment either side of the 256-leaf batch boundary, against a real +/// on-chain settlement. +/// +/// 257 is the count the old `addresses.chunks(MAX_LEAVES)` split partitioned +/// as `[256, 1]`: the 256-address batch was paid for on-chain and the +/// singleton remainder could not build a tree, so the call returned a *paid* +/// partial result carrying 256 proofs. It now splits as `[255, 2]` and every +/// address comes back with a proof. +/// +/// Paying directly is what makes the boundary reachable: getting 257 chunks +/// out of self-encryption needs a ~1 GB file, while `pay_for_merkle_batch` +/// takes the address set straight. +/// +/// Settlement is checked against the same padded-leaf model the estimator +/// bills with — 65 addresses settle 128 leaves, 257 settle 256 + 2 — so the +/// two counts must cost in that ratio. +#[tokio::test(flavor = "multi_thread")] +#[serial] +async fn test_merkle_payment_across_batch_boundary() { + let (client, testnet) = setup_merkle_testnet().await; + + // Distinct, deterministic addresses. Payment binds to addresses only; the + // chunks themselves are never stored by this test. + let addresses = |count: usize, tag: u8| -> Vec<[u8; 32]> { + (0..count) + .map(|i| { + let mut addr = [0u8; 32]; + addr[0] = tag; + addr[1..9].copy_from_slice(&(i as u64).to_be_bytes()); + addr + }) + .collect() + }; + + let mut paid: Vec<(usize, u128)> = Vec::new(); + + for (count, tag) in [(65usize, 0xA1u8), (257usize, 0xB2u8)] { + let addrs = addresses(count, tag); + + // A 65/257-address tree collects far more candidate pools than the + // small-tree tests above, and an in-process 35-node testnet on a loaded + // CI runner can leave one pool a candidate short ("Got 15 merkle + // candidates, need 16"). That shortfall is transient, so retry it. + // The bug this test guards is not: a `[256, 1]` partition fails to + // build its second tree on every attempt, so a short proof set + // survives all three and still fails the assertion below. + let mut result = None; + let mut last_shortfall = String::new(); + for attempt in 1..=3 { + eprintln!("Paying for {count} addresses via merkle batch (attempt {attempt}/3)..."); + match client + .pay_for_merkle_batch(&addrs, 0, TEST_CHUNK_SIZE as u64) + .await + { + Ok(full) if full.proofs.len() == count => { + result = Some(full); + break; + } + Ok(partial) => { + last_shortfall = format!( + "paid but returned {} of {count} proofs", + partial.proofs.len() + ); + } + Err(e) => last_shortfall = e.to_string(), + } + eprintln!(" attempt {attempt}/3 fell short: {last_shortfall}"); + } + let result = result.unwrap_or_else(|| { + panic!( + "merkle payment for {count} addresses never returned a full proof set \ + after 3 attempts: {last_shortfall}" + ) + }); + + assert_eq!( + result.proofs.len(), + count, + "every one of the {count} addresses must come back with a proof, not just the \ + sub-batches that fit a tree" + ); + assert_eq!(result.chunk_count, count); + for addr in &addrs { + assert!( + result.proofs.contains_key(addr), + "missing proof for {}", + hex::encode(addr) + ); + } + + let settled: u128 = result + .storage_cost_atto + .parse() + .expect("settled amount should parse"); + assert!( + settled > 0, + "{count} addresses must settle a non-zero amount" + ); + eprintln!( + " {count} addresses: {} leaves billed, settled {settled} atto", + merkle_billable_leaves(count as u64) + ); + paid.push((count, settled)); + } + + // Prices are uniform across a freshly started local testnet and this test + // stores nothing, so the only thing separating the two settlements is the + // padded leaf count each partition pays for. + let [(small, small_atto), (large, large_atto)] = paid[..] else { + panic!("expected two payments"); + }; + let expected = + merkle_billable_leaves(large as u64) as f64 / merkle_billable_leaves(small as u64) as f64; + let observed = large_atto as f64 / small_atto as f64; + assert!( + (observed - expected).abs() / expected < 0.15, + "settlement should scale with padded leaves: expected ~{expected:.3}x \ + ({small} -> {large} addresses), observed {observed:.3}x" + ); + + drop(client); + testnet.teardown().await; +} + // Single-node coexistence is tested in e2e_file.rs (DEFAULT_NODE_COUNT testnet). // The 35-node testnet's DHT can have sparse XOR regions where single-node // quotes can't find 5 peers for a random chunk address, making that test diff --git a/ant-core/tests/merkle_unit.rs b/ant-core/tests/merkle_unit.rs index dd9138d..335b9e7 100644 --- a/ant-core/tests/merkle_unit.rs +++ b/ant-core/tests/merkle_unit.rs @@ -1,11 +1,18 @@ //! Merkle payment unit tests. //! -//! These tests use the free function `should_use_merkle` — no Client or network needed. -//! The real tests are in `src/client/merkle.rs` (inline test module). +//! These tests exercise the free functions — `should_use_merkle` and the +//! batch partitioning that payment and cost estimation share — so no Client or +//! network is needed. The rest live in `src/data/client/merkle.rs` (inline test +//! module). #![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)] -use ant_core::data::client::merkle::{should_use_merkle, PaymentMode, DEFAULT_MERKLE_THRESHOLD}; +use ant_core::data::client::merkle::{ + merkle_batch_partitions, merkle_batch_sizes, merkle_billable_leaves, should_use_merkle, + PaymentMode, DEFAULT_MERKLE_THRESHOLD, +}; +use ant_protocol::evm::MAX_LEAVES; +use std::collections::HashSet; #[test] fn test_threshold_constant() { @@ -28,3 +35,86 @@ fn test_merkle_mode() { fn test_single_mode() { assert!(!should_use_merkle(1000, PaymentMode::Single)); } + +/// Counts covering the minimum tree, the merkle threshold, either side of a +/// full batch, and the `1 mod MAX_LEAVES` counts a naive `chunks(MAX_LEAVES)` +/// split turned into an unpayable one-address tail. +const PARTITION_CASES: [(usize, &[usize]); 10] = [ + (2, &[2]), + (64, &[64]), + (65, &[65]), + (255, &[255]), + (256, &[256]), + (257, &[255, 2]), + (300, &[256, 44]), + (512, &[256, 256]), + (513, &[256, 255, 2]), + (769, &[256, 256, 255, 2]), +]; + +fn addresses(count: usize) -> Vec<[u8; 32]> { + (0..count) + .map(|i| { + let mut addr = [0u8; 32]; + addr[..8].copy_from_slice(&(i as u64).to_be_bytes()); + addr + }) + .collect() +} + +#[test] +fn batch_sizes_rebalance_singleton_remainders() { + for (total, expected) in PARTITION_CASES { + assert_eq!( + merkle_batch_sizes(total), + expected, + "{total} addresses must partition as {expected:?}, never with a one-address batch" + ); + } +} + +#[test] +fn batch_sizes_are_valid_tree_sizes_and_cover_the_input() { + for total in 2..=(3 * MAX_LEAVES + 5) { + let sizes = merkle_batch_sizes(total); + assert_eq!(sizes.iter().sum::(), total); + for size in sizes { + assert!( + (2..=MAX_LEAVES).contains(&size), + "{total} addresses produced a batch of {size}, outside 2..={MAX_LEAVES}" + ); + } + } +} + +#[test] +fn partitions_preserve_order_and_use_each_address_once() { + for (total, expected) in PARTITION_CASES { + let addrs = addresses(total); + let partitions = merkle_batch_partitions(&addrs); + + let sizes: Vec = partitions.iter().map(|batch| batch.len()).collect(); + assert_eq!(sizes, expected); + + let flattened: Vec<[u8; 32]> = partitions.concat(); + assert_eq!(flattened, addrs, "{total}: order must be preserved"); + + let unique: HashSet<[u8; 32]> = flattened.iter().copied().collect(); + assert_eq!(unique.len(), total, "{total}: no address may repeat"); + } +} + +#[test] +fn billable_leaves_are_the_padded_partitions() { + for (total, expected) in PARTITION_CASES { + let padded: u64 = expected + .iter() + .map(|size| size.next_power_of_two() as u64) + .sum(); + assert_eq!( + merkle_billable_leaves(total as u64), + padded, + "{total} chunks must bill for the partition {expected:?}, padded" + ); + } +}