diff --git a/Cargo.lock b/Cargo.lock index bd4d89ae6ac..e327ed2940d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -119,6 +119,32 @@ dependencies = [ "strum", ] +[[package]] +name = "alloy-consensus" +version = "1.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f16daaf7e1f95f62c6c3bf8a3fc3d78b08ae9777810c0bb5e94966c7cd57ef0" +dependencies = [ + "alloy-eips 1.8.3", + "alloy-primitives", + "alloy-rlp", + "alloy-serde 1.8.3", + "alloy-trie", + "alloy-tx-macros 1.8.3", + "auto_impl", + "borsh", + "c-kzg", + "derive_more", + "either", + "k256", + "once_cell", + "secp256k1 0.30.0", + "serde", + "serde_json", + "serde_with", + "thiserror 2.0.18", +] + [[package]] name = "alloy-consensus" version = "2.0.0" @@ -130,7 +156,7 @@ dependencies = [ "alloy-rlp", "alloy-serde 2.0.0", "alloy-trie", - "alloy-tx-macros", + "alloy-tx-macros 2.0.0", "arbitrary", "auto_impl", "borsh", @@ -153,7 +179,7 @@ version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "88fc7bbfb98cf5605a35aadf0ba43a7d9f1608d6f220d05e4fbd5144d3b0b625" dependencies = [ - "alloy-consensus", + "alloy-consensus 2.0.0", "alloy-eips 2.0.0", "alloy-primitives", "alloy-rlp", @@ -168,7 +194,7 @@ version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f4c16fa30b623e40a5b216da00f3b61870f5cbe863b59816ac1ecc2489515a40" dependencies = [ - "alloy-consensus", + "alloy-consensus 2.0.0", "alloy-dyn-abi", "alloy-json-abi", "alloy-network", @@ -316,7 +342,7 @@ version = "0.33.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bd3e12b99b0c8f7298ffd3604c58310cba310203c5f6cd5e024f3b2bd9c3b09c" dependencies = [ - "alloy-consensus", + "alloy-consensus 2.0.0", "alloy-eips 2.0.0", "alloy-hardforks 0.4.7", "alloy-primitives", @@ -330,6 +356,21 @@ dependencies = [ "tracing", ] +[[package]] +name = "alloy-genesis" +version = "1.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbf9480307b09d22876efb67d30cadd9013134c21f3a17ec9f93fd7536d38024" +dependencies = [ + "alloy-eips 1.8.3", + "alloy-primitives", + "alloy-serde 1.8.3", + "alloy-trie", + "borsh", + "serde", + "serde_with", +] + [[package]] name = "alloy-genesis" version = "2.0.0" @@ -405,7 +446,7 @@ version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f0a3f5a7f3678b71d33fcc45b714fab8928dbc647d5aff2145e72032d5c849bb" dependencies = [ - "alloy-consensus", + "alloy-consensus 2.0.0", "alloy-consensus-any", "alloy-eips 2.0.0", "alloy-json-rpc", @@ -431,7 +472,7 @@ version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fb50dc1fb0e0b2c8748d5bee1aa7acdd18f9e036311bc93a71d97be624030317" dependencies = [ - "alloy-consensus", + "alloy-consensus 2.0.0", "alloy-eips 2.0.0", "alloy-primitives", "alloy-serde 2.0.0", @@ -444,7 +485,7 @@ version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "85195890fcee519312718dc8418035935ad0d57f57943ca82689732432a702c9" dependencies = [ - "alloy-genesis", + "alloy-genesis 2.0.0", "alloy-hardforks 0.2.13", "alloy-network", "alloy-primitives", @@ -498,7 +539,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f2ba5468f78c8893be2d68a7f2fda61753336e5653f006af19781001b5f99e6c" dependencies = [ "alloy-chains", - "alloy-consensus", + "alloy-consensus 2.0.0", "alloy-eips 2.0.0", "alloy-json-rpc", "alloy-network", @@ -625,7 +666,7 @@ version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5f1d057dcbacf8be8f689a7737e0d697fd40a2dc5b664c9035f182ff016649ea" dependencies = [ - "alloy-genesis", + "alloy-genesis 2.0.0", "alloy-primitives", "serde", "serde_json", @@ -697,7 +738,7 @@ version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e59bc947935732cae5b072753e5e034c0b70a8b031c2839f45e2659ba07df9ae" dependencies = [ - "alloy-consensus", + "alloy-consensus 2.0.0", "alloy-eips 2.0.0", "alloy-primitives", "alloy-rlp", @@ -718,7 +759,7 @@ version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cc280a41931bd419af86e9e859dd9726b73313aaa2e479b33c0e344f4b892ddb" dependencies = [ - "alloy-consensus", + "alloy-consensus 2.0.0", "alloy-consensus-any", "alloy-eips 2.0.0", "alloy-network-primitives", @@ -740,7 +781,7 @@ version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "286c40ce0d715217a5bfa9fb452779b11e6769e56680afa0de691ae8f3a848ac" dependencies = [ - "alloy-consensus", + "alloy-consensus 2.0.0", "alloy-eips 2.0.0", "alloy-primitives", "alloy-rpc-types-eth", @@ -819,7 +860,7 @@ version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3c67d2372aada343130d41e249b59a3cef29b1678dcd3fd80f1c2c4d6b5318f2" dependencies = [ - "alloy-consensus", + "alloy-consensus 2.0.0", "alloy-network", "alloy-primitives", "alloy-signer", @@ -1000,6 +1041,18 @@ dependencies = [ "tracing", ] +[[package]] +name = "alloy-tx-macros" +version = "1.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d69722eddcdf1ce096c3ab66cf8116999363f734eb36fe94a148f4f71c85da84" +dependencies = [ + "darling 0.23.0", + "proc-macro2", + "quote", + "syn 2.0.117", +] + [[package]] name = "alloy-tx-macros" version = "2.0.0" @@ -1063,7 +1116,7 @@ version = "1.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc" dependencies = [ - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] @@ -1074,7 +1127,7 @@ checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d" dependencies = [ "anstyle", "once_cell_polyfill", - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] @@ -2831,9 +2884,9 @@ dependencies = [ name = "custom-hardforks" version = "0.1.0" dependencies = [ - "alloy-consensus", + "alloy-consensus 2.0.0", "alloy-eips 2.0.0", - "alloy-genesis", + "alloy-genesis 2.0.0", "alloy-primitives", "reth-chainspec", "reth-network-peers", @@ -2949,7 +3002,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7ab67060fc6b8ef687992d439ca0fa36e7ed17e9a0b16b25b601e8757df720de" dependencies = [ "data-encoding", - "syn 1.0.109", + "syn 2.0.117", ] [[package]] @@ -3278,9 +3331,9 @@ dependencies = [ name = "ef-tests" version = "2.1.0" dependencies = [ - "alloy-consensus", + "alloy-consensus 2.0.0", "alloy-eips 2.0.0", - "alloy-genesis", + "alloy-genesis 2.0.0", "alloy-primitives", "alloy-rlp", "rayon", @@ -3438,7 +3491,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] @@ -3496,7 +3549,7 @@ dependencies = [ name = "example-beacon-api-sidecar-fetcher" version = "0.1.0" dependencies = [ - "alloy-consensus", + "alloy-consensus 2.0.0", "alloy-eips 2.0.0", "alloy-primitives", "alloy-rpc-types-beacon", @@ -3587,7 +3640,7 @@ dependencies = [ name = "example-custom-dev-node" version = "0.0.0" dependencies = [ - "alloy-genesis", + "alloy-genesis 2.0.0", "alloy-primitives", "eyre", "futures-util", @@ -3600,7 +3653,7 @@ dependencies = [ name = "example-custom-engine-types" version = "0.0.0" dependencies = [ - "alloy-genesis", + "alloy-genesis 2.0.0", "alloy-primitives", "alloy-rpc-types", "eyre", @@ -3619,7 +3672,7 @@ name = "example-custom-evm" version = "0.0.0" dependencies = [ "alloy-evm", - "alloy-genesis", + "alloy-genesis 2.0.0", "alloy-primitives", "eyre", "reth-ethereum", @@ -3721,7 +3774,7 @@ dependencies = [ name = "example-manual-p2p" version = "0.0.0" dependencies = [ - "alloy-consensus", + "alloy-consensus 2.0.0", "eyre", "futures", "reth-discv4", @@ -3791,7 +3844,7 @@ dependencies = [ name = "example-polygon-p2p" version = "0.0.0" dependencies = [ - "alloy-genesis", + "alloy-genesis 2.0.0", "reth-discv4", "reth-ethereum", "reth-tracing", @@ -3806,7 +3859,7 @@ name = "example-precompile-cache" version = "0.0.0" dependencies = [ "alloy-evm", - "alloy-genesis", + "alloy-genesis 2.0.0", "alloy-primitives", "eyre", "parking_lot", @@ -3943,6 +3996,29 @@ version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" +[[package]] +name = "firehose-tracer" +version = "5.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "794633ac82fd0a4073b672a6a100c6065cd225d7895fb2f3271c743011aa9207" +dependencies = [ + "alloy-consensus 1.8.3", + "alloy-eips 1.8.3", + "alloy-genesis 1.8.3", + "alloy-primitives", + "alloy-rlp", + "eyre", + "hex", + "k256", + "num_enum", + "prost", + "prost-types", + "rbase64", + "serde", + "serde_json", + "tracing", +] + [[package]] name = "fixed-cache" version = "0.1.8" @@ -4202,7 +4278,7 @@ dependencies = [ "libc", "log", "rustversion", - "windows-link 0.1.3", + "windows-link 0.2.1", "windows-result 0.4.1", ] @@ -5091,7 +5167,7 @@ checksum = "3640c1c38b8e4e43584d8df18be5fc6b0aa314ce6ebf51b53313d4306cca8e46" dependencies = [ "hermit-abi", "libc", - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] @@ -6067,7 +6143,7 @@ version = "0.50.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" dependencies = [ - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -6878,6 +6954,15 @@ dependencies = [ "syn 2.0.117", ] +[[package]] +name = "prost-types" +version = "0.14.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8991c4cbdb8bc5b11f0b074ffe286c30e523de90fee5ba8132f1399f23cb3dd7" +dependencies = [ + "prost", +] + [[package]] name = "quanta" version = "0.12.6" @@ -7173,6 +7258,10 @@ dependencies = [ "crossbeam-utils", ] +[[package]] +name = "rbase64" +version = "2.0.3" + [[package]] name = "recvmsg" version = "1.0.0" @@ -7376,6 +7465,7 @@ dependencies = [ "reth-ethereum-cli", "reth-ethereum-payload-builder", "reth-ethereum-primitives", + "reth-firehose", "reth-network", "reth-network-api", "reth-node-api", @@ -7407,7 +7497,7 @@ dependencies = [ name = "reth-basic-payload-builder" version = "2.1.0" dependencies = [ - "alloy-consensus", + "alloy-consensus 2.0.0", "alloy-eips 2.0.0", "alloy-primitives", "futures-core", @@ -7433,7 +7523,7 @@ dependencies = [ name = "reth-bb" version = "2.1.0" dependencies = [ - "alloy-consensus", + "alloy-consensus 2.0.0", "alloy-eips 2.0.0", "alloy-evm", "alloy-hardforks 0.4.7", @@ -7481,7 +7571,7 @@ dependencies = [ name = "reth-bench" version = "2.1.0" dependencies = [ - "alloy-consensus", + "alloy-consensus 2.0.0", "alloy-eip7928", "alloy-eips 2.0.0", "alloy-json-rpc", @@ -7529,7 +7619,7 @@ dependencies = [ name = "reth-chain-state" version = "2.1.0" dependencies = [ - "alloy-consensus", + "alloy-consensus 2.0.0", "alloy-eips 2.0.0", "alloy-primitives", "alloy-signer", @@ -7562,10 +7652,10 @@ name = "reth-chainspec" version = "2.1.0" dependencies = [ "alloy-chains", - "alloy-consensus", + "alloy-consensus 2.0.0", "alloy-eips 2.0.0", "alloy-evm", - "alloy-genesis", + "alloy-genesis 2.0.0", "alloy-primitives", "alloy-rlp", "alloy-trie", @@ -7581,7 +7671,7 @@ dependencies = [ name = "reth-cli" version = "2.1.0" dependencies = [ - "alloy-genesis", + "alloy-genesis 2.0.0", "clap", "eyre", "reth-cli-runner", @@ -7594,7 +7684,7 @@ name = "reth-cli-commands" version = "2.1.0" dependencies = [ "alloy-chains", - "alloy-consensus", + "alloy-consensus 2.0.0", "alloy-eips 2.0.0", "alloy-primitives", "alloy-rlp", @@ -7715,9 +7805,9 @@ version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a79b3247ae4fbb1d4d35ce83a11fc596428a4c6ea836c98a75a55340192578a4" dependencies = [ - "alloy-consensus", + "alloy-consensus 2.0.0", "alloy-eips 2.0.0", - "alloy-genesis", + "alloy-genesis 2.0.0", "alloy-primitives", "alloy-trie", "arbitrary", @@ -7763,7 +7853,7 @@ dependencies = [ name = "reth-consensus" version = "2.1.0" dependencies = [ - "alloy-consensus", + "alloy-consensus 2.0.0", "alloy-primitives", "auto_impl", "reth-execution-types", @@ -7775,7 +7865,7 @@ dependencies = [ name = "reth-consensus-common" version = "2.1.0" dependencies = [ - "alloy-consensus", + "alloy-consensus 2.0.0", "alloy-eips 2.0.0", "alloy-primitives", "rand 0.9.4", @@ -7789,7 +7879,7 @@ dependencies = [ name = "reth-consensus-debug-client" version = "2.1.0" dependencies = [ - "alloy-consensus", + "alloy-consensus 2.0.0", "alloy-eips 2.0.0", "alloy-json-rpc", "alloy-primitives", @@ -7814,7 +7904,7 @@ dependencies = [ name = "reth-db" version = "2.1.0" dependencies = [ - "alloy-consensus", + "alloy-consensus 2.0.0", "alloy-primitives", "arbitrary", "assert_matches", @@ -7849,7 +7939,7 @@ dependencies = [ name = "reth-db-api" version = "2.1.0" dependencies = [ - "alloy-consensus", + "alloy-consensus 2.0.0", "alloy-primitives", "arbitrary", "arrayvec", @@ -7877,8 +7967,8 @@ dependencies = [ name = "reth-db-common" version = "2.1.0" dependencies = [ - "alloy-consensus", - "alloy-genesis", + "alloy-consensus 2.0.0", + "alloy-genesis 2.0.0", "alloy-primitives", "boyer-moore-magiclen", "eyre", @@ -8003,7 +8093,7 @@ dependencies = [ name = "reth-downloaders" version = "2.1.0" dependencies = [ - "alloy-consensus", + "alloy-consensus 2.0.0", "alloy-eips 2.0.0", "alloy-primitives", "alloy-rlp", @@ -8041,7 +8131,7 @@ dependencies = [ name = "reth-e2e-test-utils" version = "2.1.0" dependencies = [ - "alloy-consensus", + "alloy-consensus 2.0.0", "alloy-eips 2.0.0", "alloy-network", "alloy-primitives", @@ -8124,7 +8214,7 @@ dependencies = [ name = "reth-engine-local" version = "2.1.0" dependencies = [ - "alloy-consensus", + "alloy-consensus 2.0.0", "alloy-primitives", "alloy-rpc-types-engine", "eyre", @@ -8146,7 +8236,7 @@ dependencies = [ name = "reth-engine-primitives" version = "2.1.0" dependencies = [ - "alloy-consensus", + "alloy-consensus 2.0.0", "alloy-eips 2.0.0", "alloy-primitives", "alloy-rpc-types-engine", @@ -8170,7 +8260,7 @@ dependencies = [ name = "reth-engine-tree" version = "2.1.0" dependencies = [ - "alloy-consensus", + "alloy-consensus 2.0.0", "alloy-eip7928", "alloy-eips 2.0.0", "alloy-evm", @@ -8242,7 +8332,7 @@ dependencies = [ name = "reth-engine-util" version = "2.1.0" dependencies = [ - "alloy-consensus", + "alloy-consensus 2.0.0", "alloy-rpc-types-engine", "eyre", "futures", @@ -8269,7 +8359,7 @@ dependencies = [ name = "reth-era" version = "2.1.0" dependencies = [ - "alloy-consensus", + "alloy-consensus 2.0.0", "alloy-eips 2.0.0", "alloy-primitives", "alloy-rlp", @@ -8309,7 +8399,7 @@ dependencies = [ name = "reth-era-utils" version = "2.1.0" dependencies = [ - "alloy-consensus", + "alloy-consensus 2.0.0", "alloy-primitives", "bytes", "eyre", @@ -8346,7 +8436,7 @@ name = "reth-eth-wire" version = "2.1.0" dependencies = [ "alloy-chains", - "alloy-consensus", + "alloy-consensus 2.0.0", "alloy-eips 2.0.0", "alloy-primitives", "alloy-rlp", @@ -8384,10 +8474,10 @@ name = "reth-eth-wire-types" version = "2.1.0" dependencies = [ "alloy-chains", - "alloy-consensus", + "alloy-consensus 2.0.0", "alloy-eip7928", "alloy-eips 2.0.0", - "alloy-genesis", + "alloy-genesis 2.0.0", "alloy-hardforks 0.4.7", "alloy-primitives", "alloy-rlp", @@ -8472,7 +8562,7 @@ dependencies = [ name = "reth-ethereum-consensus" version = "2.1.0" dependencies = [ - "alloy-consensus", + "alloy-consensus 2.0.0", "alloy-eips 2.0.0", "alloy-primitives", "reth-chainspec", @@ -8517,7 +8607,7 @@ dependencies = [ name = "reth-ethereum-payload-builder" version = "2.1.0" dependencies = [ - "alloy-consensus", + "alloy-consensus 2.0.0", "alloy-eips 2.0.0", "alloy-primitives", "alloy-rlp", @@ -8546,7 +8636,7 @@ dependencies = [ name = "reth-ethereum-primitives" version = "2.1.0" dependencies = [ - "alloy-consensus", + "alloy-consensus 2.0.0", "alloy-eips 2.0.0", "alloy-primitives", "alloy-rlp", @@ -8577,7 +8667,7 @@ dependencies = [ name = "reth-evm" version = "2.1.0" dependencies = [ - "alloy-consensus", + "alloy-consensus 2.0.0", "alloy-eips 2.0.0", "alloy-evm", "alloy-primitives", @@ -8601,10 +8691,10 @@ dependencies = [ name = "reth-evm-ethereum" version = "2.1.0" dependencies = [ - "alloy-consensus", + "alloy-consensus 2.0.0", "alloy-eips 2.0.0", "alloy-evm", - "alloy-genesis", + "alloy-genesis 2.0.0", "alloy-primitives", "alloy-rpc-types-engine", "reth-chainspec", @@ -8653,7 +8743,7 @@ dependencies = [ name = "reth-execution-types" version = "2.1.0" dependencies = [ - "alloy-consensus", + "alloy-consensus 2.0.0", "alloy-eips 2.0.0", "alloy-evm", "alloy-primitives", @@ -8674,9 +8764,9 @@ dependencies = [ name = "reth-exex" version = "2.1.0" dependencies = [ - "alloy-consensus", + "alloy-consensus 2.0.0", "alloy-eips 2.0.0", - "alloy-genesis", + "alloy-genesis 2.0.0", "alloy-primitives", "eyre", "futures", @@ -8763,6 +8853,15 @@ dependencies = [ "serde_with", ] +[[package]] +name = "reth-firehose" +version = "2.1.0" +dependencies = [ + "clap", + "firehose-tracer", + "tracing", +] + [[package]] name = "reth-fs-util" version = "2.1.0" @@ -8776,7 +8875,7 @@ dependencies = [ name = "reth-invalid-block-hooks" version = "2.1.0" dependencies = [ - "alloy-consensus", + "alloy-consensus 2.0.0", "alloy-eips 2.0.0", "alloy-primitives", "alloy-rlp", @@ -8889,9 +8988,9 @@ dependencies = [ name = "reth-network" version = "2.1.0" dependencies = [ - "alloy-consensus", + "alloy-consensus 2.0.0", "alloy-eips 2.0.0", - "alloy-genesis", + "alloy-genesis 2.0.0", "alloy-primitives", "alloy-rlp", "aquamarine", @@ -8950,7 +9049,7 @@ dependencies = [ name = "reth-network-api" version = "2.1.0" dependencies = [ - "alloy-consensus", + "alloy-consensus 2.0.0", "alloy-primitives", "alloy-rpc-types-admin", "alloy-rpc-types-eth", @@ -8974,7 +9073,7 @@ dependencies = [ name = "reth-network-p2p" version = "2.1.0" dependencies = [ - "alloy-consensus", + "alloy-consensus 2.0.0", "alloy-eips 2.0.0", "alloy-primitives", "auto_impl", @@ -9067,7 +9166,7 @@ dependencies = [ name = "reth-node-builder" version = "2.1.0" dependencies = [ - "alloy-consensus", + "alloy-consensus 2.0.0", "alloy-eips 2.0.0", "alloy-primitives", "alloy-provider", @@ -9138,7 +9237,7 @@ dependencies = [ name = "reth-node-core" version = "2.1.0" dependencies = [ - "alloy-consensus", + "alloy-consensus 2.0.0", "alloy-eips 2.0.0", "alloy-primitives", "alloy-rpc-types-engine", @@ -9194,10 +9293,10 @@ dependencies = [ name = "reth-node-ethereum" version = "2.1.0" dependencies = [ - "alloy-consensus", + "alloy-consensus 2.0.0", "alloy-contract", "alloy-eips 2.0.0", - "alloy-genesis", + "alloy-genesis 2.0.0", "alloy-network", "alloy-primitives", "alloy-provider", @@ -9254,7 +9353,7 @@ dependencies = [ name = "reth-node-ethstats" version = "2.1.0" dependencies = [ - "alloy-consensus", + "alloy-consensus 2.0.0", "alloy-primitives", "chrono", "futures-util", @@ -9277,7 +9376,7 @@ dependencies = [ name = "reth-node-events" version = "2.1.0" dependencies = [ - "alloy-consensus", + "alloy-consensus 2.0.0", "alloy-eips 2.0.0", "alloy-primitives", "alloy-rpc-types-engine", @@ -9340,7 +9439,7 @@ dependencies = [ name = "reth-payload-builder" version = "2.1.0" dependencies = [ - "alloy-consensus", + "alloy-consensus 2.0.0", "alloy-primitives", "alloy-rpc-types", "derive_more", @@ -9374,7 +9473,7 @@ dependencies = [ name = "reth-payload-primitives" version = "2.1.0" dependencies = [ - "alloy-consensus", + "alloy-consensus 2.0.0", "alloy-eips 2.0.0", "alloy-primitives", "alloy-rlp", @@ -9399,7 +9498,7 @@ dependencies = [ name = "reth-payload-util" version = "2.1.0" dependencies = [ - "alloy-consensus", + "alloy-consensus 2.0.0", "alloy-primitives", "reth-transaction-pool", ] @@ -9408,7 +9507,7 @@ dependencies = [ name = "reth-payload-validator" version = "2.1.0" dependencies = [ - "alloy-consensus", + "alloy-consensus 2.0.0", "alloy-rpc-types-engine", "reth-primitives-traits", ] @@ -9419,9 +9518,9 @@ version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cc759fd87c3f65440e5d3bfa3107fe8a13a61a6807cd485c62c49d63c7bf6717" dependencies = [ - "alloy-consensus", + "alloy-consensus 2.0.0", "alloy-eips 2.0.0", - "alloy-genesis", + "alloy-genesis 2.0.0", "alloy-primitives", "alloy-rlp", "alloy-rpc-types-eth", @@ -9450,9 +9549,9 @@ dependencies = [ name = "reth-provider" version = "2.1.0" dependencies = [ - "alloy-consensus", + "alloy-consensus 2.0.0", "alloy-eips 2.0.0", - "alloy-genesis", + "alloy-genesis 2.0.0", "alloy-primitives", "alloy-rpc-types-engine", "assert_matches", @@ -9501,7 +9600,7 @@ dependencies = [ name = "reth-prune" version = "2.1.0" dependencies = [ - "alloy-consensus", + "alloy-consensus 2.0.0", "alloy-eips 2.0.0", "alloy-primitives", "assert_matches", @@ -9558,7 +9657,7 @@ dependencies = [ name = "reth-revm" version = "2.1.0" dependencies = [ - "alloy-consensus", + "alloy-consensus 2.0.0", "alloy-primitives", "alloy-rlp", "alloy-rpc-types-debug", @@ -9574,11 +9673,11 @@ dependencies = [ name = "reth-rpc" version = "2.1.0" dependencies = [ - "alloy-consensus", + "alloy-consensus 2.0.0", "alloy-dyn-abi", "alloy-eips 2.0.0", "alloy-evm", - "alloy-genesis", + "alloy-genesis 2.0.0", "alloy-network", "alloy-primitives", "alloy-rlp", @@ -9655,7 +9754,7 @@ name = "reth-rpc-api" version = "2.1.0" dependencies = [ "alloy-eips 2.0.0", - "alloy-genesis", + "alloy-genesis 2.0.0", "alloy-json-rpc", "alloy-primitives", "alloy-rpc-types", @@ -9761,7 +9860,7 @@ dependencies = [ name = "reth-rpc-convert" version = "2.1.0" dependencies = [ - "alloy-consensus", + "alloy-consensus 2.0.0", "alloy-evm", "alloy-json-rpc", "alloy-network", @@ -9781,7 +9880,7 @@ dependencies = [ name = "reth-rpc-e2e-tests" version = "2.1.0" dependencies = [ - "alloy-genesis", + "alloy-genesis 2.0.0", "alloy-rpc-types-engine", "eyre", "futures-util", @@ -9837,7 +9936,7 @@ dependencies = [ name = "reth-rpc-eth-api" version = "2.1.0" dependencies = [ - "alloy-consensus", + "alloy-consensus 2.0.0", "alloy-dyn-abi", "alloy-eip7928", "alloy-eips 2.0.0", @@ -9882,7 +9981,7 @@ dependencies = [ name = "reth-rpc-eth-types" version = "2.1.0" dependencies = [ - "alloy-consensus", + "alloy-consensus 2.0.0", "alloy-eips 2.0.0", "alloy-evm", "alloy-network", @@ -9964,7 +10063,7 @@ version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b766da61ec7c46596386b4bc88d9b57d1939d3da2bc9e927567a8a23650e5ce9" dependencies = [ - "alloy-consensus", + "alloy-consensus 2.0.0", "alloy-network", "alloy-primitives", "alloy-rpc-types-eth", @@ -9977,9 +10076,9 @@ dependencies = [ name = "reth-stages" version = "2.1.0" dependencies = [ - "alloy-consensus", + "alloy-consensus 2.0.0", "alloy-eips 2.0.0", - "alloy-genesis", + "alloy-genesis 2.0.0", "alloy-primitives", "alloy-rlp", "assert_matches", @@ -10130,7 +10229,7 @@ dependencies = [ name = "reth-storage-api" version = "2.1.0" dependencies = [ - "alloy-consensus", + "alloy-consensus 2.0.0", "alloy-eips 2.0.0", "alloy-primitives", "alloy-rpc-types-engine", @@ -10170,7 +10269,7 @@ dependencies = [ name = "reth-storage-rpc-provider" version = "2.1.0" dependencies = [ - "alloy-consensus", + "alloy-consensus 2.0.0", "alloy-eips 2.0.0", "alloy-network", "alloy-primitives", @@ -10219,9 +10318,9 @@ dependencies = [ name = "reth-testing-utils" version = "2.1.0" dependencies = [ - "alloy-consensus", + "alloy-consensus 2.0.0", "alloy-eips 2.0.0", - "alloy-genesis", + "alloy-genesis 2.0.0", "alloy-primitives", "alloy-rlp", "rand 0.8.6", @@ -10279,7 +10378,7 @@ dependencies = [ name = "reth-transaction-pool" version = "2.1.0" dependencies = [ - "alloy-consensus", + "alloy-consensus 2.0.0", "alloy-eips 2.0.0", "alloy-primitives", "alloy-rlp", @@ -10330,7 +10429,7 @@ dependencies = [ name = "reth-trie" version = "2.1.0" dependencies = [ - "alloy-consensus", + "alloy-consensus 2.0.0", "alloy-eips 2.0.0", "alloy-primitives", "alloy-rlp", @@ -10363,8 +10462,8 @@ dependencies = [ name = "reth-trie-common" version = "2.1.0" dependencies = [ - "alloy-consensus", - "alloy-genesis", + "alloy-consensus 2.0.0", + "alloy-genesis 2.0.0", "alloy-primitives", "alloy-rlp", "alloy-rpc-types-eth", @@ -10395,7 +10494,7 @@ dependencies = [ name = "reth-trie-db" version = "2.1.0" dependencies = [ - "alloy-consensus", + "alloy-consensus 2.0.0", "alloy-primitives", "alloy-rlp", "metrics", @@ -10913,7 +11012,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys", - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] @@ -10993,7 +11092,7 @@ dependencies = [ "security-framework", "security-framework-sys", "webpki-root-certs 1.0.7", - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] @@ -11582,7 +11681,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3a766e1110788c36f4fa1c2b71b387a7815aa65f88ce0229841826633d93723e" dependencies = [ "libc", - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] @@ -11782,7 +11881,7 @@ dependencies = [ "getrandom 0.4.2", "once_cell", "rustix", - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] @@ -13051,7 +13150,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index e3c153ce556..0c9dd1cb9b4 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -151,6 +151,7 @@ members = [ "testing/testing-utils", "testing/runner", "crates/tracing-otlp", + "crates/firehose", ] default-members = ["bin/reth"] exclude = ["docs/cli"] @@ -368,6 +369,7 @@ reth-execution-types = { path = "crates/evm/execution-types", default-features = reth-exex = { path = "crates/exex/exex" } reth-exex-test-utils = { path = "crates/exex/test-utils" } reth-exex-types = { path = "crates/exex/types" } +reth-firehose = { path = "crates/firehose" } reth-fs-util = { path = "crates/fs-util" } reth-invalid-block-hooks = { path = "crates/engine/invalid-block-hooks" } reth-ipc = { path = "crates/rpc/ipc" } @@ -569,6 +571,7 @@ tokio-util = { version = "0.7.4", features = ["codec"] } async-compression = { version = "0.4", default-features = false } async-stream = "0.3" async-trait = "0.1.68" +firehose-tracer = "5.1.0" futures = "0.3" futures-core = "0.3" futures-util = { version = "0.3", default-features = false } @@ -700,3 +703,8 @@ vergen-git2 = "9.1.0" # networking ipnet = "2.11" + +[patch.crates-io] +# rbase64 unconditionally sets #[global_allocator] (MiMalloc), conflicting with reth's jemalloc. +# This vendor patch removes that declaration. +rbase64 = { path = "vendor/rbase64" } diff --git a/bin/reth/Cargo.toml b/bin/reth/Cargo.toml index 424f93a5d23..9e764e2dff8 100644 --- a/bin/reth/Cargo.toml +++ b/bin/reth/Cargo.toml @@ -57,6 +57,7 @@ reth-node-ethereum.workspace = true reth-node-builder.workspace = true reth-node-metrics.workspace = true reth-consensus.workspace = true +reth-firehose.workspace = true # alloy alloy-primitives.workspace = true diff --git a/bin/reth/src/lib.rs b/bin/reth/src/lib.rs index f8a4be0424a..845a11dc5b5 100644 --- a/bin/reth/src/lib.rs +++ b/bin/reth/src/lib.rs @@ -219,4 +219,5 @@ use aquamarine as _; // used in main use clap as _; use reth_cli_util as _; +use reth_firehose as _; use tracing as _; diff --git a/bin/reth/src/main.rs b/bin/reth/src/main.rs index 838f4e1dc66..172d2ad2cb3 100644 --- a/bin/reth/src/main.rs +++ b/bin/reth/src/main.rs @@ -14,6 +14,7 @@ static MALLOC_CONF: &[u8] = b"prof:true,prof_active:true,lg_prof_sample:19\0"; use clap::Parser; use reth::cli::Cli; use reth_ethereum_cli::chainspec::EthereumChainSpecParser; +use reth_firehose::FirehoseArgs; use reth_node_ethereum::EthereumNode; use tracing::info; @@ -25,12 +26,37 @@ fn main() { unsafe { std::env::set_var("RUST_BACKTRACE", "1") }; } - if let Err(err) = Cli::::parse().run(async move |builder, _| { - info!(target: "reth::cli", "Launching node"); - let handle = builder.node(EthereumNode::default()).launch_with_debug_capabilities().await?; - - handle.wait_for_node_exit().await - }) { + if let Err(err) = + Cli::::parse().run(async move |builder, args| { + info!(target: "reth::cli", "Launching node"); + + // Resolve the node data directory for the Firehose cursor file default path. + let data_dir = builder.config().datadir().data_dir().to_path_buf(); + + // Build the tracer config from CLI args and init the global tracer. + let cfg = args.to_tracer_config(&data_dir); + let shutdown_handle = reth_firehose::init_tracer(cfg); + + let handle = builder + .node(EthereumNode::default()) + .on_component_initialized(move |node| { + // Wire the background writer's drain into the node shutdown lifecycle. + if let Some(handle) = shutdown_handle { + node.task_executor.spawn_with_graceful_shutdown_signal( + |shutdown| async move { + let _guard = shutdown.await; + handle.drain(); + }, + ); + } + Ok(()) + }) + .launch_with_debug_capabilities() + .await?; + + handle.wait_for_node_exit().await + }) + { eprintln!("Error: {err:?}"); std::process::exit(1); } diff --git a/crates/firehose/Cargo.toml b/crates/firehose/Cargo.toml new file mode 100644 index 00000000000..207820bac5e --- /dev/null +++ b/crates/firehose/Cargo.toml @@ -0,0 +1,20 @@ +[package] +name = "reth-firehose" +version.workspace = true +edition.workspace = true +rust-version.workspace = true +license.workspace = true +homepage.workspace = true +repository.workspace = true +description = "Firehose integration for Reth: async emission, shutdown wiring, and cursor writing" + +[lints] +workspace = true + +[dependencies] +# firehose +firehose-tracer.workspace = true + +# misc +clap = { workspace = true, features = ["derive", "env"] } +tracing.workspace = true diff --git a/crates/firehose/src/args.rs b/crates/firehose/src/args.rs new file mode 100644 index 00000000000..4a625b4f6a5 --- /dev/null +++ b/crates/firehose/src/args.rs @@ -0,0 +1,112 @@ +//! CLI arguments for the Firehose integration. +//! +//! These arguments control how the Firehose tracer emits blocks to stdout, +//! where to write the cursor file, and how async emission is configured. + +use clap::Args; +use firehose_tracer::EmissionMode; +use std::{path::PathBuf, time::Duration}; + +/// Firehose emission mode, mirroring [`EmissionMode`] for CLI parsing. +#[derive(Debug, Clone, Default, clap::ValueEnum)] +pub enum EmissionModeArg { + /// Encode and write blocks inline on the calling thread (legacy behaviour). + Blocking, + /// Encode and write blocks in a dedicated background thread with backpressure. + Async, + /// Switch automatically based on block age (catch-up → async, live → blocking). + #[default] + Auto, +} + +/// CLI arguments for the Firehose tracer integration. +/// +/// Add `#[command(flatten)]` to include these in a `NodeCommand` extension struct. +#[derive(Debug, Clone, Args)] +pub struct FirehoseArgs { + /// Controls when and how encoded blocks are written to stdout. + /// + /// - `blocking`: encode → base64 → write, all inline on the calling thread (legacy). + /// - `async`: encode and write in a background thread; backpressure via channel. + /// - `auto`: use async for blocks older than `--firehose.live-threshold`; use blocking for + /// blocks within the live window (default). + #[arg( + id = "firehose.emission-mode", + long = "firehose.emission-mode", + value_name = "MODE", + default_value = "auto", + verbatim_doc_comment + )] + pub emission_mode: EmissionModeArg, + + /// Channel capacity for the async emission path. + /// + /// The background writer thread will block producers once this many encoded + /// blocks are waiting, providing backpressure. Only relevant for `async` and + /// `auto` modes. + #[arg( + id = "firehose.channel-capacity", + long = "firehose.channel-capacity", + value_name = "N", + default_value_t = 32 + )] + pub channel_capacity: usize, + + /// Age threshold in seconds used by `auto` emission mode. + /// + /// Blocks with a timestamp more than this many seconds behind wall-clock time + /// are considered historical (catch-up) and will use the async path. + /// Blocks within this window are considered live and will use the blocking path. + #[arg( + id = "firehose.live-threshold", + long = "firehose.live-threshold", + value_name = "SECS", + default_value_t = 60 + )] + pub live_threshold_secs: u64, + + /// Path to the cursor file that tracks the last block successfully emitted to stdout. + /// + /// After each block is written the cursor file is updated atomically so that the + /// node can detect gaps after an unclean shutdown. Defaults to `/firehose.cursor` + /// when not set. + #[arg(id = "firehose.cursor-path", long = "firehose.cursor-path", value_name = "PATH")] + pub cursor_path: Option, +} + +impl Default for FirehoseArgs { + fn default() -> Self { + Self { + emission_mode: EmissionModeArg::Auto, + channel_capacity: 32, + live_threshold_secs: 60, + cursor_path: None, + } + } +} + +impl FirehoseArgs { + /// Convert the parsed CLI args into a [`firehose_tracer::config::Config`]. + /// + /// `data_dir` is used to derive the default cursor file path when + /// `--firehose.cursor-path` is not specified. + pub fn to_tracer_config(&self, data_dir: &std::path::Path) -> firehose_tracer::config::Config { + let cursor_path = + self.cursor_path.clone().unwrap_or_else(|| data_dir.join("firehose.cursor")); + + let emission_mode = match self.emission_mode { + EmissionModeArg::Blocking => EmissionMode::Blocking, + EmissionModeArg::Async => { + EmissionMode::Async { channel_capacity: self.channel_capacity } + } + EmissionModeArg::Auto => EmissionMode::Auto { + channel_capacity: self.channel_capacity, + live_threshold: Duration::from_secs(self.live_threshold_secs), + }, + }; + + firehose_tracer::config::Config::new() + .with_emission_mode(emission_mode) + .with_cursor_path(cursor_path) + } +} diff --git a/crates/firehose/src/lib.rs b/crates/firehose/src/lib.rs new file mode 100644 index 00000000000..6c6bde06b7f --- /dev/null +++ b/crates/firehose/src/lib.rs @@ -0,0 +1,57 @@ +//! Reth ↔ Firehose integration. +//! +//! This crate provides: +//! +//! * **`FirehoseArgs`** — clap argument group that exposes `--firehose.*` CLI flags (emission mode, +//! channel capacity, live threshold, cursor path). +//! * **`init_tracer`** — one-shot initialisation that stores the tracer in a process-wide +//! `OnceLock` and returns an optional [`ShutdownHandle`](firehose_tracer::ShutdownHandle) for the +//! async emission background thread. + +#![cfg_attr(not(test), warn(unused_crate_dependencies))] + +mod args; +pub use args::FirehoseArgs; + +use std::sync::{Mutex, OnceLock}; +use tracing::{debug, info}; + +/// Process-wide Firehose tracer, initialised at most once per process. +static GLOBAL_TRACER: OnceLock> = OnceLock::new(); + +/// Initialise the global Firehose tracer with the given configuration. +/// +/// Returns a [`ShutdownHandle`](firehose_tracer::ShutdownHandle) when the emission mode has an +/// async background thread. The caller is responsible for calling +/// [`ShutdownHandle::drain`](firehose_tracer::ShutdownHandle::drain) before the process exits so +/// that the background thread can flush all buffered blocks. +/// +/// # Panics +/// +/// Panics if called more than once in the same process. +pub fn init_tracer(config: firehose_tracer::config::Config) -> Option { + info!( + target: "reth::firehose", + cursor_path = ?config.cursor_path, + "Initialising Firehose tracer" + ); + + let mut tracer = firehose_tracer::Tracer::new(config); + let shutdown_handle = tracer.shutdown_handle(); + + debug!( + target: "reth::firehose", + has_async_thread = shutdown_handle.is_some(), + "Firehose tracer created" + ); + + GLOBAL_TRACER.set(Mutex::new(tracer)).expect("init_tracer called more than once"); + + shutdown_handle +} + +/// Returns a reference to the global [`Mutex`], or `None` if the +/// tracer has not been initialised via [`init_tracer`]. +pub fn get_tracer() -> Option<&'static Mutex> { + GLOBAL_TRACER.get() +} diff --git a/vendor/rbase64/Cargo.toml b/vendor/rbase64/Cargo.toml new file mode 100644 index 00000000000..60a7970bb03 --- /dev/null +++ b/vendor/rbase64/Cargo.toml @@ -0,0 +1,16 @@ +# Patched rbase64: removed mimalloc global allocator that conflicts with reth's jemalloc. +# See the workspace [patch.crates-io] section. + +[package] +edition = "2021" +name = "rbase64" +version = "2.0.3" +authors = ["Marcel Riera "] +description = "A fast multi-threaded base64 encoding library and CLI tool (patched: no global allocator)" +license = "MIT OR Apache-2.0" +repository = "https://github.com/uhmarcel/rbase64" + +[dependencies] + +[dev-dependencies] +rand = { version = "0.8.5", features = ["small_rng"] } diff --git a/vendor/rbase64/src/common.rs b/vendor/rbase64/src/common.rs new file mode 100644 index 00000000000..16db1c7e137 --- /dev/null +++ b/vendor/rbase64/src/common.rs @@ -0,0 +1,42 @@ +pub const ENCODE_MAP: &[u8; 64] = + b"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; +pub const DECODE_MAP: &[u8; 256] = &construct_decode_map(); + +pub const ENC_CHUNK_SIZE: usize = 5; +pub const DEC_CHUNK_SIZE: usize = 2; + +pub const SIX_BIT_MASK: u128 = 0x3f; +pub const BYTE_MASK: u64 = 0xff; +pub const INVALID_BYTE: u8 = 0x40; + +#[cfg(feature = "parallel")] +pub const PARALLEL_THRESHOLD_BYTES: usize = 2 << 16; // 128 KiB +#[cfg(feature = "parallel")] +pub const PARALLEL_BATCH_SIZE: usize = 256; + +const fn construct_decode_map() -> [u8; 256] { + let mut map = [INVALID_BYTE; 256]; + let mut index = 0; + + while index < 64 { + map[ENCODE_MAP[index] as usize] = index as u8; + index += 1; + } + map +} + +#[cfg(test)] +mod tests { + use crate::common::construct_decode_map; + use crate::ENCODE_MAP; + + #[test] + fn should_construct_matching_encode_decode_tables() { + for byte in 0..64 { + assert_eq!( + construct_decode_map()[ENCODE_MAP[byte] as usize], + byte as u8 + ); + } + } +} diff --git a/vendor/rbase64/src/decode.rs b/vendor/rbase64/src/decode.rs new file mode 100644 index 00000000000..6963e24f261 --- /dev/null +++ b/vendor/rbase64/src/decode.rs @@ -0,0 +1,100 @@ +use crate::common::*; +use core::fmt; +use std::error; + +#[derive(Clone, PartialEq, Eq, Debug)] +pub enum DecodeError { + InvalidByte(u8), +} + +impl error::Error for DecodeError {} + +impl fmt::Display for DecodeError { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + match *self { + Self::InvalidByte(byte) => write!(f, "Invalid non-base64 byte '{}'", byte as char), + } + } +} + +#[inline(always)] +#[cfg(not(feature = "parallel"))] +pub(crate) fn decode_u64_chunks(input: &[u8], buffer: &mut [u8]) -> Result<(), DecodeError> { + decode_u64_chunks_sync(input, buffer) +} + +#[inline(always)] +#[cfg(feature = "parallel")] +pub(crate) fn decode_u64_chunks(input: &[u8], buffer: &mut [u8]) -> Result<(), DecodeError> { + if input.len() < PARALLEL_THRESHOLD_BYTES { + decode_u64_chunks_sync(input, buffer) + } else { + decode_u64_chunks_parallel(input, buffer) + } +} + +#[inline(always)] +fn decode_u64_chunks_sync(input: &[u8], buffer: &mut [u8]) -> Result<(), DecodeError> { + let in_chunks = input.chunks_exact(DEC_CHUNK_SIZE * 4); + let out_chunks = buffer.chunks_exact_mut(DEC_CHUNK_SIZE * 3); + + in_chunks + .zip(out_chunks) + .try_for_each(|(in_chunk, out_chunk)| decode_u64(in_chunk, out_chunk)) +} + +#[inline(always)] +#[cfg(feature = "parallel")] +fn decode_u64_chunks_parallel(input: &[u8], buffer: &mut [u8]) -> Result<(), DecodeError> { + use rayon::prelude::*; + + let batch_size = PARALLEL_BATCH_SIZE * DEC_CHUNK_SIZE; + let in_batch = input.par_chunks(batch_size * 4); + let out_batch = buffer.par_chunks_mut(batch_size * 3); + + in_batch + .zip(out_batch) + .try_for_each(|(in_chunk, out_chunk)| decode_u64_chunks_sync(in_chunk, out_chunk)) +} + +#[inline(always)] +pub(crate) fn decode_u64_remainder(input: &[u8], buffer: &mut [u8]) -> Result { + let mut in_u64 = 0u64; + let mut in_bits = 0; + + for in_byte in input.iter() { + if *in_byte == b'=' { + break; + } + in_u64 = (in_u64 << 6) | decode_byte(*in_byte)? as u64; + in_bits += 6; + } + + let byte_count = in_bits / 8; + + for (i, out_byte) in buffer[..byte_count].iter_mut().enumerate() { + *out_byte = ((in_u64 >> (in_bits - 8 * (i + 1))) & BYTE_MASK) as u8; + } + Ok(byte_count) +} + +#[inline(always)] +fn decode_u64(input: &[u8], buffer: &mut [u8]) -> Result<(), DecodeError> { + let mut out_u64 = 0u64; + + for (i, in_byte) in input.iter().enumerate() { + out_u64 |= (decode_byte(*in_byte)? as u64) << (64 - ((i + 1) * 6)); + } + + let out_bytes: [u8; 8] = u64::to_be_bytes(out_u64); + buffer.copy_from_slice(&out_bytes[..6]); + Ok(()) +} + +#[inline(always)] +fn decode_byte(byte: u8) -> Result { + match DECODE_MAP[byte as usize] { + INVALID_BYTE => Err(DecodeError::InvalidByte(byte)), + decoded => Ok(decoded), + } +} diff --git a/vendor/rbase64/src/encode.rs b/vendor/rbase64/src/encode.rs new file mode 100644 index 00000000000..3727a19f53a --- /dev/null +++ b/vendor/rbase64/src/encode.rs @@ -0,0 +1,93 @@ +use crate::common::*; +use std::cmp::min; + +#[inline(always)] +#[cfg(not(feature = "parallel"))] +pub(crate) fn encode_u128_chunks(input: &[u8], buffer: &mut [u8]) { + encode_u128_chunks_sync(input, buffer); +} + +#[inline(always)] +#[cfg(feature = "parallel")] +pub(crate) fn encode_u128_chunks(input: &[u8], buffer: &mut [u8]) { + if input.len() < PARALLEL_THRESHOLD_BYTES { + encode_u128_chunks_sync(input, buffer); + } else { + encode_u128_chunks_parallel(input, buffer); + }; +} + +#[inline(always)] +fn encode_u128_chunks_sync(input: &[u8], buffer: &mut [u8]) { + let in_chunks = input.chunks_exact(ENC_CHUNK_SIZE * 3); + let out_chunks = buffer.chunks_exact_mut(ENC_CHUNK_SIZE * 4); + + in_chunks.zip(out_chunks).for_each(|(in_chunk, out_chunk)| { + encode_u128(in_chunk, out_chunk); + }); +} + +#[inline(always)] +#[cfg(feature = "parallel")] +fn encode_u128_chunks_parallel(input: &[u8], buffer: &mut [u8]) { + use rayon::prelude::*; + + let batch_size = PARALLEL_BATCH_SIZE * ENC_CHUNK_SIZE; + let in_batches = input.par_chunks(batch_size * 3); + let out_batches = buffer.par_chunks_mut(batch_size * 4); + + in_batches + .zip(out_batches) + .for_each(|(in_batch, out_batch)| { + encode_u128_chunks_sync(in_batch, out_batch); + }); +} + +#[inline(always)] +pub(crate) fn encode_u128_remainder(input: &[u8], buffer: &mut [u8]) -> usize { + let in_u128 = read_u128_partial(input); + let mut in_bits = 8 * input.len(); + let mut out_index = 0; + + while in_bits >= 6 { + in_bits -= 6; + buffer[out_index] = encode_byte(((in_u128 >> in_bits) & SIX_BIT_MASK) as u8); + out_index += 1; + } + + if in_bits > 0 { + buffer[out_index] = encode_byte(((in_u128 << (6 - in_bits)) & SIX_BIT_MASK) as u8); + out_index += 1; + } + + while out_index % 4 > 0 { + buffer[out_index] = b'='; + out_index += 1; + } + out_index +} + +#[inline(always)] +fn encode_u128(input: &[u8], buffer: &mut [u8]) { + let in_u128 = read_u128_partial(input); + let offset = (ENC_CHUNK_SIZE * 3 - 1) * 8; + + buffer.iter_mut().enumerate().for_each(|(i, out_b)| { + *out_b = encode_byte(((in_u128 >> (2 + offset - (i * 6))) & SIX_BIT_MASK) as u8); + }); +} + +#[inline(always)] +fn read_u128_partial(bytes: &[u8]) -> u128 { + let size = min(bytes.len(), 16); + let mut buffer = [0u8; 16]; + + buffer[16 - size..].copy_from_slice(&bytes[..size]); + + u128::from_be_bytes(buffer) +} + +#[inline(always)] +fn encode_byte(byte: u8) -> u8 { + ENCODE_MAP[byte as usize] +} diff --git a/vendor/rbase64/src/lib.rs b/vendor/rbase64/src/lib.rs new file mode 100644 index 00000000000..8d5602462dd --- /dev/null +++ b/vendor/rbase64/src/lib.rs @@ -0,0 +1,45 @@ +use crate::common::*; +use crate::decode::DecodeError; + +mod common; +mod decode; +mod encode; + +// NOTE: The original rbase64 crate set MiMalloc as the global allocator here, +// which conflicts with binaries that define their own global allocator (e.g. +// jemalloc in reth). This patched version removes that declaration so that +// the crate can be used as a library without hijacking the allocator. + +pub fn encode(input: &[u8]) -> String { + let mut buffer = vec![0; ((input.len() / 3) + 1) * 4]; + let total_chunks = input.len() / (ENC_CHUNK_SIZE * 3); + + encode::encode_u128_chunks(input, &mut buffer); + + let bytes_rem = encode::encode_u128_remainder( + &input[ENC_CHUNK_SIZE * total_chunks * 3..], + &mut buffer[ENC_CHUNK_SIZE * total_chunks * 4..], + ); + + buffer.truncate(ENC_CHUNK_SIZE * total_chunks * 4 + bytes_rem); + + // SAFETY: The buffer only contains bytes from the base64 alphabet (A-Z, a-z, 0-9, +, /, =), + // which are all valid single-byte UTF-8 characters. + unsafe { String::from_utf8_unchecked(buffer) } +} + +pub fn decode(encoded: &str) -> Result, DecodeError> { + let input = encoded.as_bytes(); + let mut buffer = vec![0; ((input.len() + 3) / 4) * 3]; + + let total_chunks = input.len().saturating_sub(2) / (DEC_CHUNK_SIZE * 4); + let in_limit = total_chunks * DEC_CHUNK_SIZE * 4; + let out_limit = total_chunks * DEC_CHUNK_SIZE * 3; + + decode::decode_u64_chunks(&input[..in_limit], &mut buffer)?; + + let bytes_rem = decode::decode_u64_remainder(&input[in_limit..], &mut buffer[out_limit..])?; + + buffer.truncate(out_limit + bytes_rem); + Ok(buffer) +}