From 0c1f9208b9c396dfc2c2fc929b806786ceb7c40c Mon Sep 17 00:00:00 2001 From: Ignacio Amigo Date: Sat, 30 May 2026 13:00:44 -0300 Subject: [PATCH] feat: store genesis commitment and export ntxauth --- bin/ntx-builder/src/actor/mod.rs | 1 + bin/ntx-builder/src/clients/rpc.rs | 223 ++++++++++++++++-- bin/ntx-builder/src/db/migrations.rs | 2 +- .../src/db/migrations/001_initial.sql | 3 + bin/ntx-builder/src/db/mod.rs | 16 ++ .../src/db/models/queries/chain_state.rs | 70 +++++- bin/ntx-builder/src/db/schema.rs | 1 + bin/ntx-builder/src/lib.rs | 22 +- 8 files changed, 302 insertions(+), 36 deletions(-) diff --git a/bin/ntx-builder/src/actor/mod.rs b/bin/ntx-builder/src/actor/mod.rs index 8c9078f09..1d92488be 100644 --- a/bin/ntx-builder/src/actor/mod.rs +++ b/bin/ntx-builder/src/actor/mod.rs @@ -153,6 +153,7 @@ impl AccountActorContext { clients: GrpcClients { rpc: RpcClient::new( url.clone(), + miden_protocol::Word::default(), Duration::from_millis(100), Duration::from_secs(30), ), diff --git a/bin/ntx-builder/src/clients/rpc.rs b/bin/ntx-builder/src/clients/rpc.rs index 34491355d..7056b3d7b 100644 --- a/bin/ntx-builder/src/clients/rpc.rs +++ b/bin/ntx-builder/src/clients/rpc.rs @@ -5,12 +5,26 @@ use backon::{ExponentialBuilder, Retryable}; use futures::Stream; use futures::stream::TryStreamExt; use miden_node_proto::clients::{Builder, RpcClient as InnerRpcClient}; +use miden_node_proto::domain::account::{ + AccountDetails, AccountResponse, AccountVaultDetails, StorageMapEntries +}; +use miden_node_proto::errors::ConversionError; +use miden_node_proto::generated::rpc::account_request::account_detail_request::{StorageMapDetailRequest, storage_map_detail_request}; +use miden_node_proto::generated::rpc::account_request::account_detail_request::storage_map_detail_request::MapKeys; use miden_node_proto::generated::rpc::{BlockSubscriptionRequest, BlockSubscriptionResponse}; use miden_node_proto::generated::{self as proto}; use miden_node_utils::ErrorReport; use miden_protocol::Word; -use miden_protocol::account::{AccountId, StorageMapKey, StorageMapWitness, StorageSlotName}; -use miden_protocol::asset::{AssetVaultKey, AssetWitness}; +use miden_protocol::account::{ + AccountCode, + AccountId, + PartialAccount, + PartialStorage, + StorageMapKey, + StorageMapWitness, + StorageSlotName, +}; +use miden_protocol::asset::{Asset, AssetVault, AssetVaultKey, AssetWitness, PartialVault}; use miden_protocol::block::{BlockNumber, SignedBlock}; use miden_protocol::note::NoteScript; use miden_protocol::transaction::{AccountInputs, ProvenTransaction, TransactionInputs}; @@ -41,14 +55,23 @@ impl RpcClient { /// /// `backoff_initial` / `backoff_max` configure the exponential backoff schedule applied to /// `block_subscription` retries (the only operation that retries today). - pub fn new(rpc_url: Url, backoff_initial: Duration, backoff_max: Duration) -> Self { - Self::new_with_auth(rpc_url, None, backoff_initial, backoff_max) + pub fn new( + rpc_url: Url, + genesis_commitment: Word, + backoff_initial: Duration, + backoff_max: Duration, + ) -> Self { + Self::new_with_auth(rpc_url, None, genesis_commitment, backoff_initial, backoff_max) } /// Creates a new client with an optional metadata header for internal RPC authentication. + /// + /// `genesis_commitment` is sent as the `genesis` parameter of the `Accept` header so that the + /// node accepts write RPCs such as `SubmitProvenTx`, which require a matching genesis. pub fn new_with_auth( rpc_url: Url, rpc_auth_header_value: Option, + genesis_commitment: Word, backoff_initial: Duration, backoff_max: Duration, ) -> Self { @@ -58,7 +81,7 @@ impl RpcClient { .without_tls() .without_timeout() .without_metadata_version() - .without_metadata_genesis(); + .with_metadata_genesis(genesis_commitment.to_hex()); let builder = match rpc_auth_header_value { Some(value) => builder.with_auth_header_value(value), None => builder.without_auth_header(), @@ -151,45 +174,193 @@ fn decode_block_subscription_response( // ACTOR-PATH METHODS // ================================================================================================ // -// The actor module still references these methods. PR 1 keeps the actor code in tree as dead -// code (it is not spawned), so the methods exist as stubs to preserve compilation. PR 2 wires -// them through the appropriate RPC gRPC service. - -#[expect(clippy::unused_async)] +// Required endpoint implementations for the NTX `DataStore` implementation impl RpcClient { + /// Fetches the transaction inputs for a specific account. + /// + /// These inputs reference a specific `block_num`, and include a minimal partial account, + /// plus its witness. pub async fn get_account_inputs( &self, - _account_id: AccountId, - _block_num: BlockNumber, + account_id: AccountId, + block_num: BlockNumber, ) -> Result { - unimplemented!("get_account_inputs is rewired in PR 2 of the ntx-builder refactor") + // Only request account code + let request = proto::rpc::AccountRequest { + account_id: Some(proto::account::AccountId { id: account_id.to_bytes() }), + block_num: Some(block_num.into()), + // TODO: should these commitments be cached on the NTX builder? + details: Some(proto::rpc::account_request::AccountDetailRequest { + code_commitment: Some(Word::default().into()), + asset_vault_commitment: None, // + storage_maps: vec![], + }), + }; + + let response = self.get_account(request).await?; + let details = response.details.as_ref().ok_or_else(|| { + RpcError::InvalidResponse("response did not include account details".into()) + })?; + let partial_account = build_minimal_partial_account(details)?; + + Ok(AccountInputs::new(partial_account, response.witness)) } + /// Fetches asset vault witnesses for the given keys at the reference block. pub async fn get_vault_asset_witnesses( &self, - _account_id: AccountId, - _vault_keys: BTreeSet, - _block_num: Option, + account_id: AccountId, + vault_keys: BTreeSet, + block_num: Option, ) -> Result, RpcError> { - unimplemented!("get_vault_asset_witnesses is rewired in PR 2 of the ntx-builder refactor") + if vault_keys.is_empty() { + return Ok(Vec::new()); + } + + let request = proto::rpc::AccountRequest { + account_id: Some(proto::account::AccountId { id: account_id.to_bytes() }), + block_num: block_num.map(Into::into), + details: Some(proto::rpc::account_request::AccountDetailRequest { + code_commitment: None, + asset_vault_commitment: Some(Word::default().into()), + storage_maps: vec![], + }), + }; + + let response = self.get_account(request).await?; + let assets: Vec = match response.details.map(|details| details.vault_details) { + Some(AccountVaultDetails::Assets(assets)) => assets, + Some(AccountVaultDetails::LimitExceeded) => { + // NOTE: in the tx kernel, `get_vault_asset_witnesses` is called either for single + // asset keys, or when pre-loading all the assets related to input notes involved in + // the transaction. This should never exceed the maximum amount of keys you can + // request to RPC, but this needs double-checking. If it able to exceed them, + // batching needs to be implemented as a workaround. + panic!("should never exceed maximum number of requested keys") + }, + None => Vec::new(), + }; + + let vault = + AssetVault::new(&assets).map_err(|err| RpcError::InvalidResponse(err.as_report()))?; + + Ok(vault_keys.into_iter().map(|key| vault.open(key)).collect()) } + /// Fetches a storage map witness for a single key at the reference block. pub async fn get_storage_map_witness( &self, - _account_id: AccountId, - _slot_name: StorageSlotName, - _map_key: StorageMapKey, - _block_num: Option, + account_id: AccountId, + slot_name: StorageSlotName, + map_key: StorageMapKey, + block_num: Option, ) -> Result { - unimplemented!("get_storage_map_witness is rewired in PR 2 of the ntx-builder refactor") + let request = proto::rpc::AccountRequest { + account_id: Some(proto::account::AccountId { id: account_id.to_bytes() }), + block_num: block_num.map(Into::into), + details: Some(proto::rpc::account_request::AccountDetailRequest { + code_commitment: None, + asset_vault_commitment: None, + storage_maps: vec![StorageMapDetailRequest { + slot_name: slot_name.to_string(), + slot_data: Some(storage_map_detail_request::SlotData::MapKeys(MapKeys { + map_keys: vec![map_key.into()], + })), + }], + }), + }; + + let response = self.get_account(request).await?; + let details = response.details.as_ref().ok_or_else(|| { + RpcError::InvalidResponse("response did not include account details".into()) + })?; + + let map_details = details + .storage_details + .map_details + .iter() + .find(|detail| detail.slot_name == slot_name) + .ok_or_else(|| { + RpcError::InvalidResponse(format!( + "response is missing storage map details for slot {slot_name}" + )) + })?; + + let StorageMapEntries::EntriesWithProofs(proofs) = &map_details.entries else { + return Err(RpcError::InvalidResponse( + "response did not include storage map entry proofs".into(), + )); + }; + + let proof = proofs.first().cloned().ok_or_else(|| { + RpcError::InvalidResponse( + "response did not include a proof for the requested key".into(), + ) + })?; + + StorageMapWitness::new(proof, [map_key]) + .map_err(|err| RpcError::InvalidResponse(err.as_report())) } + /// Fetches a note script by its root, returning `None` if the node does not know it. + #[instrument(target = COMPONENT, name = "ntx.rpc.client.get_note_script_by_root", skip_all, err)] pub async fn get_note_script_by_root( &self, - _script_root: Word, + script_root: Word, ) -> Result, RpcError> { - unimplemented!("get_note_script_by_root is rewired in PR 2 of the ntx-builder refactor") + let request = proto::note::NoteScriptRoot { root: Some(script_root.into()) }; + + let script = self + .inner + .clone() + .get_note_script_by_root(request) + .await + .map_err(RpcError::GrpcClientError)? + .into_inner() + .script; + + script.map(NoteScript::try_from).transpose().map_err(RpcError::Conversion) } + + /// Issues a `GetAccount` request and decodes the response into the domain [`AccountResponse`]. + async fn get_account( + &self, + request: proto::rpc::AccountRequest, + ) -> Result { + let response = self + .inner + .clone() + .get_account(request) + .await + .map_err(RpcError::GrpcClientError)? + .into_inner(); + + AccountResponse::try_from(response).map_err(RpcError::Conversion) + } +} + +/// Builds a minimal partial account from account details. +fn build_minimal_partial_account(details: &AccountDetails) -> Result { + let code_bytes = details + .account_code + .as_ref() + .ok_or_else(|| RpcError::InvalidResponse("response did not include account code".into()))?; + let account_code = AccountCode::read_from_bytes(code_bytes).map_err(RpcError::Deserialize)?; + + let partial_storage = PartialStorage::new(details.storage_details.header.clone(), []) + .map_err(|err| RpcError::InvalidResponse(err.as_report()))?; + + let partial_vault = PartialVault::new(details.account_header.vault_root()); + + PartialAccount::new( + details.account_header.id(), + details.account_header.nonce(), + account_code, + partial_storage, + partial_vault, + None, + ) + .map_err(|err| RpcError::InvalidResponse(err.as_report())) } // RPC ERROR @@ -201,4 +372,8 @@ pub enum RpcError { GrpcClientError(#[source] tonic::Status), #[error("failed to deserialize RPC payload")] Deserialize(#[source] miden_protocol::utils::serde::DeserializationError), + #[error("failed to convert RPC response")] + Conversion(#[source] ConversionError), + #[error("invalid RPC response: {0}")] + InvalidResponse(String), } diff --git a/bin/ntx-builder/src/db/migrations.rs b/bin/ntx-builder/src/db/migrations.rs index 8b2d25bb0..2b014889f 100644 --- a/bin/ntx-builder/src/db/migrations.rs +++ b/bin/ntx-builder/src/db/migrations.rs @@ -30,7 +30,7 @@ mod tests { use super::*; const EXPECTED_SCHEMA_HASHES: [SchemaHash; 1] = [SchemaHash::from_hex( - "e7383731af6f594a2f84ea8c3863325f0219899cff13e1396630c4ea8fed8157", + "8f580504230fb5ebc91bdf3e99f316bd919ec7e7312a45cbc8a52682edf8e68c", )]; #[test] diff --git a/bin/ntx-builder/src/db/migrations/001_initial.sql b/bin/ntx-builder/src/db/migrations/001_initial.sql index ad952d3d2..870254ca5 100644 --- a/bin/ntx-builder/src/db/migrations/001_initial.sql +++ b/bin/ntx-builder/src/db/migrations/001_initial.sql @@ -11,6 +11,9 @@ CREATE TABLE chain_state ( block_header BLOB NOT NULL, -- Serialized PartialMmr corresponding to `block_header`. chain_mmr BLOB NOT NULL, + -- Serialized genesis block commitment (Word). Set once at bootstrap and retained across tip + -- updates; used for the `genesis` Accept-header param required by write RPCs. + genesis_commitment BLOB, CONSTRAINT chain_state_block_num_is_u32 CHECK (block_num BETWEEN 0 AND 0xFFFFFFFF) ); diff --git a/bin/ntx-builder/src/db/mod.rs b/bin/ntx-builder/src/db/mod.rs index 67f39cac6..2dcca26b0 100644 --- a/bin/ntx-builder/src/db/mod.rs +++ b/bin/ntx-builder/src/db/mod.rs @@ -98,14 +98,30 @@ impl Db { "ntx-builder database is already bootstrapped", ); + let genesis_commitment = genesis.header().commitment(); + let effects = CommittedBlockEffects::from_signed_block(genesis); db.apply_committed_block(effects, PartialMmr::default()) .await .context("failed to insert genesis block")?; + db.inner + .transact("set_genesis_commitment", move |conn| { + queries::set_genesis_commitment(conn, &genesis_commitment) + }) + .await + .context("failed to persist genesis commitment")?; + Ok(()) } + /// Reads the genesis block commitment persisted at bootstrap. + pub async fn get_genesis_commitment(&self) -> Result { + self.inner + .query("get_genesis_commitment", queries::select_genesis_commitment) + .await + } + // BLOCK APPLICATION // ============================================================================================ diff --git a/bin/ntx-builder/src/db/models/queries/chain_state.rs b/bin/ntx-builder/src/db/models/queries/chain_state.rs index 1b8f27363..86cfc5838 100644 --- a/bin/ntx-builder/src/db/models/queries/chain_state.rs +++ b/bin/ntx-builder/src/db/models/queries/chain_state.rs @@ -1,7 +1,9 @@ //! Chain state queries and models. use diesel::prelude::*; +use diesel::upsert::excluded; use miden_node_db::DatabaseError; +use miden_protocol::Word; use miden_protocol::block::{BlockHeader, BlockNumber}; use miden_protocol::crypto::merkle::mmr::PartialMmr; use miden_protocol::utils::serde::{Deserializable, Serializable}; @@ -35,14 +37,19 @@ struct ChainStateRow { // QUERIES // ================================================================================================ -/// Inserts or replaces the singleton chain state row, persisting the chain tip header and the -/// associated partial chain MMR. +/// Upserts the singleton chain state row, persisting the chain tip header and the associated +/// partial chain MMR. On conflict only the tip columns are updated, so the `genesis_commitment` +/// set at bootstrap is retained. /// /// # Raw SQL /// /// ```sql -/// INSERT OR REPLACE INTO chain_state (id, block_num, block_header, chain_mmr) +/// INSERT INTO chain_state (id, block_num, block_header, chain_mmr) /// VALUES (0, ?1, ?2, ?3) +/// ON CONFLICT(id) DO UPDATE SET +/// block_num = excluded.block_num, +/// block_header = excluded.block_header, +/// chain_mmr = excluded.chain_mmr /// ``` pub fn upsert_chain_state( conn: &mut SqliteConnection, @@ -50,16 +57,71 @@ pub fn upsert_chain_state( block_header: &BlockHeader, chain_mmr: &PartialMmr, ) -> Result<(), DatabaseError> { + use schema::chain_state::columns; + let row = ChainStateInsert { id: 0, block_num: conversions::block_num_to_i64(block_num), block_header: conversions::block_header_to_bytes(block_header), chain_mmr: chain_mmr.to_bytes(), }; - diesel::replace_into(schema::chain_state::table).values(&row).execute(conn)?; + diesel::insert_into(schema::chain_state::table) + .values(&row) + .on_conflict(columns::id) + .do_update() + .set(( + columns::block_num.eq(excluded(columns::block_num)), + columns::block_header.eq(excluded(columns::block_header)), + columns::chain_mmr.eq(excluded(columns::chain_mmr)), + )) + .execute(conn)?; + Ok(()) +} + +/// Persists the genesis block commitment into the singleton chain state row. Called once at +/// bootstrap, after the genesis chain state has been inserted. +/// +/// # Raw SQL +/// +/// ```sql +/// UPDATE chain_state SET genesis_commitment = ?1 WHERE id = 0 +/// ``` +pub fn set_genesis_commitment( + conn: &mut SqliteConnection, + genesis_commitment: &Word, +) -> Result<(), DatabaseError> { + diesel::update(schema::chain_state::table.find(0i32)) + .set( + schema::chain_state::genesis_commitment + .eq(conversions::word_to_bytes(genesis_commitment)), + ) + .execute(conn)?; Ok(()) } +/// Reads the genesis block commitment from the singleton chain state row. +/// +/// # Raw SQL +/// +/// ```sql +/// SELECT genesis_commitment FROM chain_state WHERE id = 0 +/// ``` +/// +/// # Errors +/// +/// - If the genesis commitment had not been set +pub fn select_genesis_commitment(conn: &mut SqliteConnection) -> Result { + let commitment: Option> = schema::chain_state::table + .find(0i32) + .select(schema::chain_state::genesis_commitment) + .first(conn)?; + + let commitment = commitment.ok_or(diesel::result::Error::NotFound)?; + + Word::read_from_bytes(&commitment) + .map_err(|e| DatabaseError::deserialization("genesis commitment", e)) +} + /// Reads the singleton chain state row, returning the persisted block number, header, and chain /// MMR if any block has been applied locally. /// diff --git a/bin/ntx-builder/src/db/schema.rs b/bin/ntx-builder/src/db/schema.rs index 6c5151fdd..ab2d6db4f 100644 --- a/bin/ntx-builder/src/db/schema.rs +++ b/bin/ntx-builder/src/db/schema.rs @@ -14,6 +14,7 @@ diesel::table! { block_num -> BigInt, block_header -> Binary, chain_mmr -> Binary, + genesis_commitment -> Nullable, } } diff --git a/bin/ntx-builder/src/lib.rs b/bin/ntx-builder/src/lib.rs index c30f5e53f..c0111a887 100644 --- a/bin/ntx-builder/src/lib.rs +++ b/bin/ntx-builder/src/lib.rs @@ -374,27 +374,35 @@ impl NtxBuilderConfig { "sqlite connection pool size must be at least 2 (the event loop pins one connection)", ); + // Set up the database (bootstrap + connection pool). + let db = Db::setup_with_pool_size( + self.database_filepath.clone(), + self.sqlite_connection_pool_size, + ) + .await?; + + // Get the genesis commitment to send in the accept header + let genesis_commitment = db.get_genesis_commitment().await.context( + "failed to read genesis commitment; \ + run `miden-ntx-builder bootstrap` first", + )?; + let rpc = match self.rpc_auth_header.clone() { Some(rpc_auth_header_value) => RpcClient::new_with_auth( self.rpc_url.clone(), Some(rpc_auth_header_value), + genesis_commitment, self.request_backoff_initial, self.request_backoff_max, ), None => RpcClient::new( self.rpc_url.clone(), + genesis_commitment, self.request_backoff_initial, self.request_backoff_max, ), }; - // Set up the database (bootstrap + connection pool). - let db = Db::setup_with_pool_size( - self.database_filepath.clone(), - self.sqlite_connection_pool_size, - ) - .await?; - // The database is bootstrapped with the genesis block before startup (see // `miden-ntx-builder bootstrap`), so a persisted chain state is always present. Load it and // resume the subscription from the block after the last applied one.