From 7dfb0c6ce2977ed8db5c5b475ee6bd2f7349d3fd Mon Sep 17 00:00:00 2001 From: Gustavo Valverde Date: Fri, 26 Sep 2025 17:02:11 +0100 Subject: [PATCH 001/431] fix(ci): resolve multiple CI workflow issues and required status checks (#9928) * fix(ci): add clippy component to Rust toolchain setup in test-crates workflow The test-crates workflow was failing with error "'cargo-clippy' is not installed for the toolchain 'stable-x86_64-unknown-linux-gnu'" because the actions-rust-lang/setup-rust-toolchain action wasn't configured to install the clippy component. Added 'components: clippy' to both Rust setup steps in the workflow to ensure clippy is available when running cargo clippy commands. * fix(ci): fromJSON errors in workflows Fixed "Error from function 'fromJSON': empty input" by checking if get-available-disks job succeeded before parsing its outputs. When the job is skipped (e.g., for external PRs), its outputs are undefined and fromJSON fails * fix(ci): add mutually exclusive status checks for path-filtered workflows Added status-checks.patch.yml to provide status checks when main workflows are skipped due to path filters. Uses paths-ignore to ensure mutual exclusion - this workflow ONLY runs when the main workflows DON'T run, preventing race conditions and status check conflicts. Security consideration: The paths-ignore list must be the exact inverse of paths in the main workflows to prevent both from running simultaneously, which could allow malicious code to pass checks. --- .github/workflows/status-checks.patch.yml | 43 +++++++++++++++++++ .github/workflows/test-crates.yml | 6 ++- .github/workflows/test-docker.yml | 4 +- .../zfnd-ci-integration-tests-gcp.yml | 29 +++++++------ 4 files changed, 64 insertions(+), 18 deletions(-) create mode 100644 .github/workflows/status-checks.patch.yml diff --git a/.github/workflows/status-checks.patch.yml b/.github/workflows/status-checks.patch.yml new file mode 100644 index 00000000000..d237c6414d5 --- /dev/null +++ b/.github/workflows/status-checks.patch.yml @@ -0,0 +1,43 @@ +# This workflow ensures required status checks pass when main workflows are skipped. +# !CRITICAL: Uses paths-ignore to ensure mutual exclusion with main workflows. +# Only runs when NO Rust/config files are modified. +# See: https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/collaborating-on-repositories-with-code-quality-features/troubleshooting-required-status-checks#handling-skipped-but-required-checks + +name: Status Check Patch + +on: + pull_request: + branches: [main] + paths-ignore: + # This MUST be an exact inverse of paths in lint.yml, tests-unit.yml, and test-crates.yml + # to ensure mutual exclusion - only one set of workflows runs + - '**/*.rs' + - '**/Cargo.toml' + - '**/Cargo.lock' + - .cargo/config.toml + - '**/clippy.toml' + - .github/workflows/lint.yml + - .github/workflows/tests-unit.yml + - .github/workflows/test-crates.yml + +permissions: + contents: read + +jobs: + lint-success: + name: lint success + runs-on: ubuntu-latest + steps: + - run: echo "No lint needed - no Rust files modified" + + test-success: + name: test success + runs-on: ubuntu-latest + steps: + - run: echo "No tests needed - no Rust files modified" + + test-crate-build-success: + name: test crate build success + runs-on: ubuntu-latest + steps: + - run: echo "No crate build needed - no Rust files modified" \ No newline at end of file diff --git a/.github/workflows/test-crates.yml b/.github/workflows/test-crates.yml index b769478718d..c74a80c864d 100644 --- a/.github/workflows/test-crates.yml +++ b/.github/workflows/test-crates.yml @@ -9,7 +9,7 @@ on: - "**/Cargo.lock" - .cargo/config.toml - "**/clippy.toml" - - .github/workflows/test-crate-build.yml + - .github/workflows/test-crates.yml push: branches: [main] @@ -19,7 +19,7 @@ on: - "**/Cargo.lock" - .cargo/config.toml - "**/clippy.toml" - - .github/workflows/test-crate-build.yml + - .github/workflows/test-crates.yml workflow_dispatch: @@ -58,6 +58,7 @@ jobs: - uses: actions-rust-lang/setup-rust-toolchain@ab6845274e2ff01cd4462007e1a9d9df1ab49f42 #v1.14.0 with: toolchain: stable + components: clippy cache-on-failure: true # This step dynamically creates a JSON containing the values of each crate @@ -108,6 +109,7 @@ jobs: - uses: actions-rust-lang/setup-rust-toolchain@ab6845274e2ff01cd4462007e1a9d9df1ab49f42 #v1.14.0 with: toolchain: stable + components: clippy cache-key: crate-build-${{ matrix.crate }} cache-on-failure: true diff --git a/.github/workflows/test-docker.yml b/.github/workflows/test-docker.yml index 1367d214905..ccc79406058 100644 --- a/.github/workflows/test-docker.yml +++ b/.github/workflows/test-docker.yml @@ -11,7 +11,7 @@ on: - docker/entrypoint.sh - docker/**/*.toml - zebrad/tests/common/configs/** - - .github/workflows/test-docker-config.yml + - .github/workflows/test-docker.yml push: branches: [main] @@ -23,7 +23,7 @@ on: - docker/entrypoint.sh - docker/**/*.toml - zebrad/tests/common/configs/** - - .github/workflows/test-docker-config.yml + - .github/workflows/test-docker.yml # Ensures that only one workflow task will run at a time. Previous builds, if # already in process, will get cancelled. Only the latest commit will be allowed diff --git a/.github/workflows/zfnd-ci-integration-tests-gcp.yml b/.github/workflows/zfnd-ci-integration-tests-gcp.yml index 577ae7b1449..cb54b0baa96 100644 --- a/.github/workflows/zfnd-ci-integration-tests-gcp.yml +++ b/.github/workflows/zfnd-ci-integration-tests-gcp.yml @@ -193,7 +193,7 @@ jobs: contents: read id-token: write uses: ./.github/workflows/zfnd-deploy-integration-tests-gcp.yml - if: ${{ !fromJSON(needs.get-available-disks.outputs.zebra_checkpoint_disk) || github.event.inputs.regenerate-disks == 'true' }} + if: ${{ (needs.get-available-disks.result == 'success' && !fromJSON(needs.get-available-disks.outputs.zebra_checkpoint_disk)) || github.event.inputs.regenerate-disks == 'true' }} concurrency: group: ${{ github.event_name == 'workflow_dispatch' && format('manual-{0}-sync-to-mandatory-checkpoint', github.run_id) || 'sync-to-mandatory-checkpoint' }} cancel-in-progress: false @@ -224,7 +224,7 @@ jobs: contents: read id-token: write uses: ./.github/workflows/zfnd-deploy-integration-tests-gcp.yml - if: ${{ !cancelled() && !failure() && (fromJSON(needs.get-available-disks.outputs.zebra_checkpoint_disk) || needs.sync-to-mandatory-checkpoint.result == 'success') && github.event.inputs.regenerate-disks != 'true' && github.event.inputs.run-full-sync != 'true' && github.event.inputs.run-lwd-sync != 'true' }} + if: ${{ !cancelled() && !failure() && ((needs.get-available-disks.result == 'success' && fromJSON(needs.get-available-disks.outputs.zebra_checkpoint_disk)) || needs.sync-to-mandatory-checkpoint.result == 'success') && github.event.inputs.regenerate-disks != 'true' && github.event.inputs.run-full-sync != 'true' && github.event.inputs.run-lwd-sync != 'true' }} secrets: GCP_SSH_PRIVATE_KEY: ${{ secrets.GCP_SSH_PRIVATE_KEY }} with: @@ -256,7 +256,7 @@ jobs: contents: read id-token: write uses: ./.github/workflows/zfnd-deploy-integration-tests-gcp.yml - if: ${{ github.event_name == 'schedule' || !fromJSON(needs.get-available-disks.outputs.zebra_tip_disk) || (github.event.inputs.run-full-sync == 'true' && (inputs.network || vars.ZCASH_NETWORK) == 'Mainnet') }} + if: ${{ github.event_name == 'schedule' || (needs.get-available-disks.result == 'success' && !fromJSON(needs.get-available-disks.outputs.zebra_tip_disk)) || (github.event.inputs.run-full-sync == 'true' && (inputs.network || vars.ZCASH_NETWORK) == 'Mainnet') }} concurrency: group: ${{ github.event_name == 'workflow_dispatch' && format('manual-{0}-sync-full-mainnet', github.run_id) || 'sync-full-mainnet' }} cancel-in-progress: false @@ -291,7 +291,7 @@ jobs: contents: read id-token: write uses: ./.github/workflows/zfnd-deploy-integration-tests-gcp.yml - if: ${{ !cancelled() && !failure() && (fromJSON(needs.get-available-disks.outputs.zebra_tip_disk) || needs.sync-full-mainnet.result == 'success') && github.event.inputs.regenerate-disks != 'true' && github.event.inputs.run-full-sync != 'true' && github.event.inputs.run-lwd-sync != 'true' }} + if: ${{ !cancelled() && !failure() && ((needs.get-available-disks.result == 'success' && fromJSON(needs.get-available-disks.outputs.zebra_tip_disk)) || needs.sync-full-mainnet.result == 'success') && github.event.inputs.regenerate-disks != 'true' && github.event.inputs.run-full-sync != 'true' && github.event.inputs.run-lwd-sync != 'true' }} secrets: GCP_SSH_PRIVATE_KEY: ${{ secrets.GCP_SSH_PRIVATE_KEY }} with: @@ -325,7 +325,7 @@ jobs: contents: read id-token: write uses: ./.github/workflows/zfnd-deploy-integration-tests-gcp.yml - if: ${{ !cancelled() && !failure() && (fromJSON(needs.get-available-disks.outputs.zebra_tip_disk) || needs.sync-full-mainnet.result == 'success') && github.event.inputs.regenerate-disks != 'true' && github.event.inputs.run-full-sync != 'true' && github.event.inputs.run-lwd-sync != 'true' }} + if: ${{ !cancelled() && !failure() && ((needs.get-available-disks.result == 'success' && fromJSON(needs.get-available-disks.outputs.zebra_tip_disk)) || needs.sync-full-mainnet.result == 'success') && github.event.inputs.regenerate-disks != 'true' && github.event.inputs.run-full-sync != 'true' && github.event.inputs.run-lwd-sync != 'true' }} secrets: GCP_SSH_PRIVATE_KEY: ${{ secrets.GCP_SSH_PRIVATE_KEY }} with: @@ -361,7 +361,7 @@ jobs: contents: read id-token: write uses: ./.github/workflows/zfnd-deploy-integration-tests-gcp.yml - if: ${{ !fromJSON(needs.get-available-disks-testnet.outputs.zebra_tip_disk) || (github.event.inputs.run-full-sync == 'true' && (inputs.network || vars.ZCASH_NETWORK) == 'Testnet') }} + if: ${{ (needs.get-available-disks-testnet.result == 'success' && !fromJSON(needs.get-available-disks-testnet.outputs.zebra_tip_disk)) || (github.event.inputs.run-full-sync == 'true' && (inputs.network || vars.ZCASH_NETWORK) == 'Testnet') }} concurrency: group: ${{ github.event_name == 'workflow_dispatch' && format('manual-{0}-sync-full-testnet', github.run_id) || 'sync-full-testnet' }} cancel-in-progress: false @@ -399,7 +399,7 @@ jobs: contents: read id-token: write uses: ./.github/workflows/zfnd-deploy-integration-tests-gcp.yml - if: ${{ !cancelled() && !failure() && (fromJSON(needs.get-available-disks-testnet.outputs.zebra_tip_disk) || needs.sync-full-testnet.result == 'success') && github.event.inputs.regenerate-disks != 'true' && github.event.inputs.run-full-sync != 'true' && github.event.inputs.run-lwd-sync != 'true' }} + if: ${{ !cancelled() && !failure() && ((needs.get-available-disks-testnet.result == 'success' && fromJSON(needs.get-available-disks-testnet.outputs.zebra_tip_disk)) || needs.sync-full-testnet.result == 'success') && github.event.inputs.regenerate-disks != 'true' && github.event.inputs.run-full-sync != 'true' && github.event.inputs.run-lwd-sync != 'true' }} secrets: GCP_SSH_PRIVATE_KEY: ${{ secrets.GCP_SSH_PRIVATE_KEY }} with: @@ -434,7 +434,7 @@ jobs: id-token: write uses: ./.github/workflows/zfnd-deploy-integration-tests-gcp.yml # Currently the lightwalletd tests only work on Mainnet - if: ${{ !cancelled() && !failure() && (inputs.network || vars.ZCASH_NETWORK) == 'Mainnet' && (fromJSON(needs.get-available-disks.outputs.zebra_tip_disk) || needs.sync-full-mainnet.result == 'success') && (github.event_name == 'schedule' || !fromJSON(needs.get-available-disks.outputs.lwd_tip_disk) || github.event.inputs.run-lwd-sync == 'true' ) }} + if: ${{ !cancelled() && !failure() && (inputs.network || vars.ZCASH_NETWORK) == 'Mainnet' && ((needs.get-available-disks.result == 'success' && fromJSON(needs.get-available-disks.outputs.zebra_tip_disk)) || needs.sync-full-mainnet.result == 'success') && (github.event_name == 'schedule' || (needs.get-available-disks.result == 'success' && !fromJSON(needs.get-available-disks.outputs.lwd_tip_disk)) || github.event.inputs.run-lwd-sync == 'true' ) }} concurrency: group: ${{ github.event_name == 'workflow_dispatch' && format('manual-{0}-lwd-sync-full', github.run_id) || 'lwd-sync-full' }} cancel-in-progress: false @@ -468,7 +468,7 @@ jobs: contents: read id-token: write uses: ./.github/workflows/zfnd-deploy-integration-tests-gcp.yml - if: ${{ !cancelled() && !failure() && (inputs.network || vars.ZCASH_NETWORK) == 'Mainnet' && (fromJSON(needs.get-available-disks.outputs.lwd_tip_disk) || needs.lwd-sync-full.result == 'success') && github.event.inputs.regenerate-disks != 'true' && github.event.inputs.run-full-sync != 'true' && github.event.inputs.run-lwd-sync != 'true' }} + if: ${{ !cancelled() && !failure() && (inputs.network || vars.ZCASH_NETWORK) == 'Mainnet' && ((needs.get-available-disks.result == 'success' && fromJSON(needs.get-available-disks.outputs.lwd_tip_disk)) || needs.lwd-sync-full.result == 'success') && github.event.inputs.regenerate-disks != 'true' && github.event.inputs.run-full-sync != 'true' && github.event.inputs.run-lwd-sync != 'true' }} secrets: GCP_SSH_PRIVATE_KEY: ${{ secrets.GCP_SSH_PRIVATE_KEY }} with: @@ -500,7 +500,7 @@ jobs: contents: read id-token: write uses: ./.github/workflows/zfnd-deploy-integration-tests-gcp.yml - if: ${{ !cancelled() && !failure() && (inputs.network || vars.ZCASH_NETWORK) == 'Mainnet' && (fromJSON(needs.get-available-disks.outputs.zebra_tip_disk) || needs.sync-full-mainnet.result == 'success') && github.event.inputs.regenerate-disks != 'true' && github.event.inputs.run-full-sync != 'true' && github.event.inputs.run-lwd-sync != 'true' }} + if: ${{ !cancelled() && !failure() && (inputs.network || vars.ZCASH_NETWORK) == 'Mainnet' && ((needs.get-available-disks.result == 'success' && fromJSON(needs.get-available-disks.outputs.zebra_tip_disk)) || needs.sync-full-mainnet.result == 'success') && github.event.inputs.regenerate-disks != 'true' && github.event.inputs.run-full-sync != 'true' && github.event.inputs.run-lwd-sync != 'true' }} secrets: GCP_SSH_PRIVATE_KEY: ${{ secrets.GCP_SSH_PRIVATE_KEY }} with: @@ -526,7 +526,7 @@ jobs: contents: read id-token: write uses: ./.github/workflows/zfnd-deploy-integration-tests-gcp.yml - if: ${{ !cancelled() && !failure() && (inputs.network || vars.ZCASH_NETWORK) == 'Mainnet' && (fromJSON(needs.get-available-disks.outputs.lwd_tip_disk) || needs.lwd-sync-full.result == 'success') && github.event.inputs.regenerate-disks != 'true' && github.event.inputs.run-full-sync != 'true' && github.event.inputs.run-lwd-sync != 'true' }} + if: ${{ !cancelled() && !failure() && (inputs.network || vars.ZCASH_NETWORK) == 'Mainnet' && ((needs.get-available-disks.result == 'success' && fromJSON(needs.get-available-disks.outputs.lwd_tip_disk)) || needs.lwd-sync-full.result == 'success') && github.event.inputs.regenerate-disks != 'true' && github.event.inputs.run-full-sync != 'true' && github.event.inputs.run-lwd-sync != 'true' }} secrets: GCP_SSH_PRIVATE_KEY: ${{ secrets.GCP_SSH_PRIVATE_KEY }} with: @@ -553,7 +553,7 @@ jobs: contents: read id-token: write uses: ./.github/workflows/zfnd-deploy-integration-tests-gcp.yml - if: ${{ !cancelled() && !failure() && (inputs.network || vars.ZCASH_NETWORK) == 'Mainnet' && (fromJSON(needs.get-available-disks.outputs.lwd_tip_disk) || needs.lwd-sync-full.result == 'success') && github.event.inputs.regenerate-disks != 'true' && github.event.inputs.run-full-sync != 'true' && github.event.inputs.run-lwd-sync != 'true' }} + if: ${{ !cancelled() && !failure() && (inputs.network || vars.ZCASH_NETWORK) == 'Mainnet' && ((needs.get-available-disks.result == 'success' && fromJSON(needs.get-available-disks.outputs.lwd_tip_disk)) || needs.lwd-sync-full.result == 'success') && github.event.inputs.regenerate-disks != 'true' && github.event.inputs.run-full-sync != 'true' && github.event.inputs.run-lwd-sync != 'true' }} secrets: GCP_SSH_PRIVATE_KEY: ${{ secrets.GCP_SSH_PRIVATE_KEY }} with: @@ -584,7 +584,7 @@ jobs: contents: read id-token: write uses: ./.github/workflows/zfnd-deploy-integration-tests-gcp.yml - if: ${{ !cancelled() && !failure() && (fromJSON(needs.get-available-disks.outputs.zebra_tip_disk) || needs.sync-full-mainnet.result == 'success') && github.event.inputs.regenerate-disks != 'true' && github.event.inputs.run-full-sync != 'true' && github.event.inputs.run-lwd-sync != 'true' }} + if: ${{ !cancelled() && !failure() && ((needs.get-available-disks.result == 'success' && fromJSON(needs.get-available-disks.outputs.zebra_tip_disk)) || needs.sync-full-mainnet.result == 'success') && github.event.inputs.regenerate-disks != 'true' && github.event.inputs.run-full-sync != 'true' && github.event.inputs.run-lwd-sync != 'true' }} secrets: GCP_SSH_PRIVATE_KEY: ${{ secrets.GCP_SSH_PRIVATE_KEY }} with: @@ -611,7 +611,7 @@ jobs: contents: read id-token: write uses: ./.github/workflows/zfnd-deploy-integration-tests-gcp.yml - if: ${{ !cancelled() && !failure() && (fromJSON(needs.get-available-disks.outputs.zebra_tip_disk) || needs.sync-full-mainnet.result == 'success') && github.event.inputs.regenerate-disks != 'true' && github.event.inputs.run-full-sync != 'true' && github.event.inputs.run-lwd-sync != 'true' }} + if: ${{ !cancelled() && !failure() && ((needs.get-available-disks.result == 'success' && fromJSON(needs.get-available-disks.outputs.zebra_tip_disk)) || needs.sync-full-mainnet.result == 'success') && github.event.inputs.regenerate-disks != 'true' && github.event.inputs.run-full-sync != 'true' && github.event.inputs.run-lwd-sync != 'true' }} secrets: GCP_SSH_PRIVATE_KEY: ${{ secrets.GCP_SSH_PRIVATE_KEY }} with: @@ -688,4 +688,5 @@ jobs: - name: Decide whether the needed jobs succeeded or failed uses: re-actors/alls-green@05ac9388f0aebcb5727afa17fcccfecd6f8ec5fe #v1.2.2 with: + allowed-skips: sync-to-mandatory-checkpoint,sync-full-mainnet,lwd-sync-full,sync-past-mandatory-checkpoint,sync-update-mainnet,generate-checkpoints-mainnet,sync-full-testnet,generate-checkpoints-testnet,lwd-sync-update,lwd-rpc-test,lwd-rpc-send-tx,lwd-grpc-wallet,rpc-get-block-template,rpc-submit-block jobs: ${{ toJSON(needs) }} From c65672cd80fbecc4da72728adafff65956a65986 Mon Sep 17 00:00:00 2001 From: Conrado Date: Fri, 26 Sep 2025 15:46:14 -0300 Subject: [PATCH 002/431] zebra-rpc: bump to 2.0.1 (#9930) Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> --- Cargo.lock | 2 +- zebra-rpc/Cargo.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 8f2265ef84e..63b1f1432f5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6903,7 +6903,7 @@ dependencies = [ [[package]] name = "zebra-rpc" -version = "2.0.0" +version = "2.0.1" dependencies = [ "base64 0.22.1", "chrono", diff --git a/zebra-rpc/Cargo.toml b/zebra-rpc/Cargo.toml index 9fce61aea69..be50c085a02 100644 --- a/zebra-rpc/Cargo.toml +++ b/zebra-rpc/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "zebra-rpc" -version = "2.0.0" +version = "2.0.1" authors = ["Zcash Foundation "] description = "A Zebra JSON Remote Procedure Call (JSON-RPC) interface" license = "MIT OR Apache-2.0" From 5b52e630acb715c1194471d74d85075b25d8e1cd Mon Sep 17 00:00:00 2001 From: Gustavo Valverde Date: Mon, 29 Sep 2025 19:16:53 +0100 Subject: [PATCH 003/431] feat(docker): add `curl` for health-check validation (#9886) * feat(docker): add `curl` for health-check validation This would allow to validate that Zebra nodes can receive traffic by creating a healt-check similar to this one: ``` curl -s -f --user "" --data-binary '{"jsonrpc": "2.0", "id":"healthcheck", "method": "getinfo", "params": [] }' -H 'Content-Type: application/json' http://127.0.0.1:${ZEBRA_RPC_PORT}/ | grep -q '"result":' || exit 1 ``` * chore: update docker/Dockerfile Co-authored-by: Arya --------- Co-authored-by: Arya Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> --- docker/Dockerfile | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/docker/Dockerfile b/docker/Dockerfile index c43a49fa736..343d6d8dec9 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -195,6 +195,12 @@ ENV USER=${USER} ARG HOME ENV HOME=${HOME} +# Install curl for healtchecks +RUN apt-get -q update && \ + apt-get -q install -y --no-install-recommends \ + curl \ + && rm -rf /var/lib/apt/lists/* /tmp/* + RUN addgroup --quiet --gid ${GID} ${USER} && \ adduser --quiet --gid ${GID} --uid ${UID} --home ${HOME} ${USER} --disabled-password --gecos "" From 0a8fbc91eaffd76b3f141ff70b941918774c88be Mon Sep 17 00:00:00 2001 From: Conrado Date: Tue, 30 Sep 2025 07:44:41 -0300 Subject: [PATCH 004/431] rpc: use specific error code for addnode; reuse message in response filter (#9931) * rpc: use specific error code for addnode; reuse message in response filter * remove not-needed one side call to `connect_nodes` (#9934) --------- Co-authored-by: Alfredo Garcia --- zebra-rpc/qa/rpc-tests/test_framework/util.py | 5 +---- zebra-rpc/src/methods.rs | 2 +- zebra-rpc/src/server/rpc_call_compatibility.rs | 9 ++++++++- 3 files changed, 10 insertions(+), 6 deletions(-) diff --git a/zebra-rpc/qa/rpc-tests/test_framework/util.py b/zebra-rpc/qa/rpc-tests/test_framework/util.py index 6db7ea7e9bb..b956e840ca9 100644 --- a/zebra-rpc/qa/rpc-tests/test_framework/util.py +++ b/zebra-rpc/qa/rpc-tests/test_framework/util.py @@ -261,9 +261,7 @@ def wait_for_bitcoind_start(process, url, i): Wait for bitcoind to start. This means that RPC is accessible and fully initialized. Raise an exception if bitcoind exits during initialization. ''' - # Zebra can do migration and other stuff at startup, even in regtest mode, - # giving 10 seconds for it to complete. - time.sleep(10) + time.sleep(1) # give the node a moment to start while True: if process.poll() is not None: raise Exception('%s node %d exited with status %i during initialization' % (zcashd_binary(), i, process.returncode)) @@ -639,7 +637,6 @@ def connect_nodes(from_connection, node_num): def connect_nodes_bi(nodes, a, b): connect_nodes(nodes[a], b) - connect_nodes(nodes[b], a) def find_output(node, txid, amount): """ diff --git a/zebra-rpc/src/methods.rs b/zebra-rpc/src/methods.rs index 634fc18e0a2..fa8164a28dd 100644 --- a/zebra-rpc/src/methods.rs +++ b/zebra-rpc/src/methods.rs @@ -2983,7 +2983,7 @@ where Ok(()) } else { return Err(ErrorObject::owned( - ErrorCode::InvalidParams.code(), + server::error::LegacyCode::ClientNodeAlreadyAdded.into(), format!("peer address was already present in the address book: {addr}"), None::<()>, )); diff --git a/zebra-rpc/src/server/rpc_call_compatibility.rs b/zebra-rpc/src/server/rpc_call_compatibility.rs index 654fe710122..9fc2cc00e35 100644 --- a/zebra-rpc/src/server/rpc_call_compatibility.rs +++ b/zebra-rpc/src/server/rpc_call_compatibility.rs @@ -63,7 +63,14 @@ impl<'a> RpcServiceT<'a> for FixRpcResponseMiddleware { return MethodResponse::error( id, - ErrorObject::borrowed(new_error_code, "Invalid params", None), + ErrorObject::borrowed( + new_error_code, + json.get("error") + .and_then(|v| v.get("message")) + .and_then(|m| m.as_str()) + .unwrap_or("Invalid params"), + None, + ), ); } } From 9213dd49d49532591c4ec634c155bb4895664053 Mon Sep 17 00:00:00 2001 From: Conrado Date: Tue, 30 Sep 2025 11:29:37 -0300 Subject: [PATCH 005/431] feat(rpc): support side chains in getrawtransaction (#9884) * rpc: support side chains in getrawtransaction * add test * fix existing test * Update zebra-rpc/src/methods/types/transaction.rs Co-authored-by: Marek * Apply suggestions from code review Co-authored-by: Alfredo Garcia * fix RPC return issue; and multiple other issues - Change AnyTx to include the block of tx - Use random coinbase data in generate() to make txs unique - Propagate submit_block() errors in generate() - Allow comitting blocks even if there was an invalidated block with the same height * Apply suggestions from code review Co-authored-by: Marek * address comments * remove unused import --------- Co-authored-by: Marek Co-authored-by: Alfredo Garcia Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> --- zebra-rpc/qa/pull-tester/rpc-tests.py | 3 +- .../rpc-tests/getrawtransaction_sidechain.py | 72 ++++++++++ zebra-rpc/src/methods.rs | 130 +++++++++++++----- zebra-rpc/src/methods/tests/prop.rs | 4 +- zebra-rpc/src/methods/tests/vectors.rs | 2 +- .../src/methods/types/get_block_template.rs | 24 +++- zebra-rpc/src/methods/types/transaction.rs | 32 +++-- zebra-state/src/lib.rs | 2 +- zebra-state/src/request.rs | 38 +++++ zebra-state/src/response.rs | 31 +++++ zebra-state/src/service.rs | 55 ++++++++ .../src/service/non_finalized_state.rs | 6 +- zebra-state/src/service/read.rs | 4 +- zebra-state/src/service/read/block.rs | 74 +++++++++- zebrad/src/components/tracing/component.rs | 3 +- zebrad/tests/acceptance.rs | 6 +- 16 files changed, 424 insertions(+), 62 deletions(-) create mode 100755 zebra-rpc/qa/rpc-tests/getrawtransaction_sidechain.py diff --git a/zebra-rpc/qa/pull-tester/rpc-tests.py b/zebra-rpc/qa/pull-tester/rpc-tests.py index d2351ceaf98..89addbedebf 100755 --- a/zebra-rpc/qa/pull-tester/rpc-tests.py +++ b/zebra-rpc/qa/pull-tester/rpc-tests.py @@ -44,7 +44,8 @@ 'wallet.py', 'feature_nu6.py', 'feature_nu6_1.py', - 'feature_backup_non_finalized_state.py'] + 'feature_backup_non_finalized_state.py', + 'getrawtransaction_sidechain.py'] ZMQ_SCRIPTS = [ # ZMQ test can only be run if bitcoin was built with zmq-enabled. diff --git a/zebra-rpc/qa/rpc-tests/getrawtransaction_sidechain.py b/zebra-rpc/qa/rpc-tests/getrawtransaction_sidechain.py new file mode 100755 index 00000000000..1c697eecb89 --- /dev/null +++ b/zebra-rpc/qa/rpc-tests/getrawtransaction_sidechain.py @@ -0,0 +1,72 @@ +#!/usr/bin/env python3 +# Copyright (c) 2025 The Zcash developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or https://www.opensource.org/licenses/mit-license.php . + +# +# Test getrawtransaction on side chains +# + +from test_framework.test_framework import BitcoinTestFramework +from test_framework.util import start_nodes, assert_equal +from test_framework.proxy import JSONRPCException + + +class GetRawTransactionSideChainTest(BitcoinTestFramework): + def __init__(self): + super().__init__() + self.num_nodes = 1 + self.cache_behavior = 'clean' + + def setup_nodes(self): + return start_nodes(self.num_nodes, self.options.tmpdir) + + def setup_network(self, split=False, do_mempool_sync=True): + self.nodes = self.setup_nodes() + + def run_test(self): + n = self.nodes[0] + + # Generate two blocks + hashes = n.generate(2) + + # Get a transaction from block at height 2 + block_hash_a = n.getbestblockhash() + txid_a = n.getblock("2", 1)['tx'][0] + tx_a = n.getrawtransaction(txid_a, 1) + assert_equal(tx_a['height'], 2) + assert_equal(tx_a['blockhash'], block_hash_a) + + # Invalidate last block (height 2) + n.invalidateblock(block_hash_a) + try: + tx_a = n.getrawtransaction(txid_a, 1) + assert False, "getrawtransaction should have failed" + except JSONRPCException: + pass + + # Regenerate a height 2 block + hashes = n.generate(1) + txid_b = n.getblock(hashes[0], 1)['tx'][0] + + # Get a transaction from the new block at height 2 + tx_b = n.getrawtransaction(txid_b, 1) + assert_equal(tx_b['height'], 2) + + # Reconsider the invalidated block + n.reconsiderblock(block_hash_a) + + # We now have two chains. Try to get transactions from both. + tx_a = n.getrawtransaction(txid_a, 1) + tx_b = n.getrawtransaction(txid_b, 1) + assert(tx_a['blockhash'] != tx_b['blockhash']) + # Exactly one of the transactions should be in a side chain + assert((tx_a['confirmations'] == 0) ^ (tx_b['confirmations'] == 0)) + assert((tx_a['height'] == -1) ^ (tx_b['height'] == -1)) + # Exactly one of the transactions should be in the main chain + assert((tx_a['height'] == 2) ^ (tx_b['height'] == 2)) + assert((tx_a['confirmations'] == 1) ^ (tx_b['confirmations'] == 1)) + + +if __name__ == '__main__': + GetRawTransactionSideChainTest().main() diff --git a/zebra-rpc/src/methods.rs b/zebra-rpc/src/methods.rs index fa8164a28dd..7063c20f878 100644 --- a/zebra-rpc/src/methods.rs +++ b/zebra-rpc/src/methods.rs @@ -49,6 +49,7 @@ use indexmap::IndexMap; use jsonrpsee::core::{async_trait, RpcResult as Result}; use jsonrpsee_proc_macros::rpc; use jsonrpsee_types::{ErrorCode, ErrorObject}; +use rand::{rngs::OsRng, RngCore}; use tokio::{ sync::{broadcast, watch}, task::JoinHandle, @@ -84,7 +85,9 @@ use zebra_chain::{ use zebra_consensus::{funding_stream_address, RouterError}; use zebra_network::{address_book_peers::AddressBookPeers, types::PeerServices, PeerSocketAddr}; use zebra_node_services::mempool; -use zebra_state::{HashOrHeight, OutputLocation, ReadRequest, ReadResponse, TransactionLocation}; +use zebra_state::{ + AnyTx, HashOrHeight, OutputLocation, ReadRequest, ReadResponse, TransactionLocation, +}; use crate::{ client::Treestate, @@ -646,7 +649,7 @@ pub trait Rpc { /// - `block_hash`: (hex-encoded block hash, required) The block hash to invalidate. // TODO: Invalidate block hashes even if they're not present in the non-finalized state (#9553). #[method(name = "invalidateblock")] - async fn invalidate_block(&self, block_hash: block::Hash) -> Result<()>; + async fn invalidate_block(&self, block_hash: String) -> Result<()>; /// Reconsiders a previously invalidated block if it exists in the cache of previously invalidated blocks. /// @@ -654,7 +657,7 @@ pub trait Rpc { /// /// - `block_hash`: (hex-encoded block hash, required) The block hash to reconsider. #[method(name = "reconsiderblock")] - async fn reconsider_block(&self, block_hash: block::Hash) -> Result>; + async fn reconsider_block(&self, block_hash: String) -> Result>; #[method(name = "generate")] /// Mine blocks immediately. Returns the block hashes of the generated blocks. @@ -1778,31 +1781,33 @@ where }; } - // TODO: this should work for blocks in side chains let txid = if let Some(block_hash) = block_hash { let block_hash = block::Hash::from_hex(block_hash) .map_error(server::error::LegacyCode::InvalidAddressOrKey)?; match self .read_state .clone() - .oneshot(zebra_state::ReadRequest::TransactionIdsForBlock( + .oneshot(zebra_state::ReadRequest::AnyChainTransactionIdsForBlock( block_hash.into(), )) .await .map_misc_error()? { - zebra_state::ReadResponse::TransactionIdsForBlock(tx_ids) => *tx_ids + zebra_state::ReadResponse::AnyChainTransactionIdsForBlock(tx_ids) => *tx_ids .ok_or_error( server::error::LegacyCode::InvalidAddressOrKey, "block not found", )? + .0 .iter() .find(|id| **id == txid) .ok_or_error( server::error::LegacyCode::InvalidAddressOrKey, "txid not found", )?, - _ => unreachable!("unmatched response to a `TransactionsByMinedId` request"), + _ => { + unreachable!("unmatched response to a `AnyChainTransactionIdsForBlock` request") + } } } else { txid @@ -1812,40 +1817,61 @@ where match self .read_state .clone() - .oneshot(zebra_state::ReadRequest::Transaction(txid)) + .oneshot(zebra_state::ReadRequest::AnyChainTransaction(txid)) .await .map_misc_error()? { - zebra_state::ReadResponse::Transaction(Some(tx)) => Ok(if verbose { - let block_hash = match self - .read_state - .clone() - .oneshot(zebra_state::ReadRequest::BestChainBlockHash(tx.height)) - .await - .map_misc_error()? - { - zebra_state::ReadResponse::BlockHash(block_hash) => block_hash, - _ => unreachable!("unmatched response to a `TransactionsByMinedId` request"), - }; - - GetRawTransactionResponse::Object(Box::new(TransactionObject::from_transaction( - tx.tx.clone(), - Some(tx.height), - Some(tx.confirmations), - &self.network, - // TODO: Performance gain: - // https://github.com/ZcashFoundation/zebra/pull/9458#discussion_r2059352752 - Some(tx.block_time), - block_hash, - Some(true), - txid, - ))) + zebra_state::ReadResponse::AnyChainTransaction(Some(tx)) => Ok(if verbose { + match tx { + AnyTx::Mined(tx) => { + let block_hash = match self + .read_state + .clone() + .oneshot(zebra_state::ReadRequest::BestChainBlockHash(tx.height)) + .await + .map_misc_error()? + { + zebra_state::ReadResponse::BlockHash(block_hash) => block_hash, + _ => { + unreachable!("unmatched response to a `BestChainBlockHash` request") + } + }; + + GetRawTransactionResponse::Object(Box::new( + TransactionObject::from_transaction( + tx.tx.clone(), + Some(tx.height), + Some(tx.confirmations), + &self.network, + // TODO: Performance gain: + // https://github.com/ZcashFoundation/zebra/pull/9458#discussion_r2059352752 + Some(tx.block_time), + block_hash, + Some(true), + txid, + ), + )) + } + AnyTx::Side((tx, block_hash)) => GetRawTransactionResponse::Object(Box::new( + TransactionObject::from_transaction( + tx.clone(), + None, + None, + &self.network, + None, + Some(block_hash), + Some(false), + txid, + ), + )), + } } else { - let hex = tx.tx.into(); + let tx: Arc = tx.into(); + let hex = tx.into(); GetRawTransactionResponse::Raw(hex) }), - zebra_state::ReadResponse::Transaction(None) => { + zebra_state::ReadResponse::AnyChainTransaction(None) => { Err("No such mempool or main chain transaction") .map_error(server::error::LegacyCode::InvalidAddressOrKey) } @@ -2899,7 +2925,11 @@ where )) } - async fn invalidate_block(&self, block_hash: block::Hash) -> Result<()> { + async fn invalidate_block(&self, block_hash: String) -> Result<()> { + let block_hash = block_hash + .parse() + .map_error(server::error::LegacyCode::InvalidParameter)?; + self.state .clone() .oneshot(zebra_state::Request::InvalidateBlock(block_hash)) @@ -2908,7 +2938,11 @@ where .map_misc_error() } - async fn reconsider_block(&self, block_hash: block::Hash) -> Result> { + async fn reconsider_block(&self, block_hash: String) -> Result> { + let block_hash = block_hash + .parse() + .map_error(server::error::LegacyCode::InvalidParameter)?; + self.state .clone() .oneshot(zebra_state::Request::ReconsiderBlock(block_hash)) @@ -2921,7 +2955,7 @@ where } async fn generate(&self, num_blocks: u32) -> Result> { - let rpc = self.clone(); + let mut rpc = self.clone(); let network = self.network.clone(); if !network.disable_pow() { @@ -2934,6 +2968,15 @@ where let mut block_hashes = Vec::new(); for _ in 0..num_blocks { + // Use random coinbase data in order to ensure the coinbase + // transaction is unique. This is useful for tests that exercise + // forks, since otherwise the coinbase txs of blocks with the same + // height across different forks would be identical. + let mut extra_coinbase_data = [0u8; 32]; + OsRng.fill_bytes(&mut extra_coinbase_data); + rpc.gbt + .set_extra_coinbase_data(extra_coinbase_data.to_vec()); + let block_template = rpc .get_block_template(None) .await @@ -2960,9 +3003,20 @@ where .map_error(server::error::LegacyCode::default())?, ); - rpc.submit_block(hex_proposal_block, None) + let r = rpc + .submit_block(hex_proposal_block, None) .await .map_error(server::error::LegacyCode::default())?; + match r { + SubmitBlockResponse::Accepted => { /* pass */ } + SubmitBlockResponse::ErrorResponse(response) => { + return Err(ErrorObject::owned( + server::error::LegacyCode::Misc.into(), + format!("block was rejected: {:?}", response), + None::<()>, + )); + } + } block_hashes.push(GetBlockHashResponse(proposal_block.hash())); } diff --git a/zebra-rpc/src/methods/tests/prop.rs b/zebra-rpc/src/methods/tests/prop.rs index 318662a2901..4f1b28dfd4d 100644 --- a/zebra-rpc/src/methods/tests/prop.rs +++ b/zebra-rpc/src/methods/tests/prop.rs @@ -345,8 +345,8 @@ proptest! { .map_ok(|r| r.respond(mempool::Response::Transactions(vec![]))); let state_query = state - .expect_request(zebra_state::ReadRequest::Transaction(unknown_txid)) - .map_ok(|r| r.respond(zebra_state::ReadResponse::Transaction(None))); + .expect_request(zebra_state::ReadRequest::AnyChainTransaction(unknown_txid)) + .map_ok(|r| r.respond(zebra_state::ReadResponse::AnyChainTransaction(None))); let rpc_query = rpc.get_raw_transaction(unknown_txid.encode_hex(), Some(1), None); diff --git a/zebra-rpc/src/methods/tests/vectors.rs b/zebra-rpc/src/methods/tests/vectors.rs index 417be8fafe9..0d5cc66595a 100644 --- a/zebra-rpc/src/methods/tests/vectors.rs +++ b/zebra-rpc/src/methods/tests/vectors.rs @@ -1127,7 +1127,7 @@ async fn rpc_getrawtransaction() { let confirmations = confirmations.expect("state requests should have confirmations"); assert_eq!(hex.as_ref(), tx.zcash_serialize_to_vec().unwrap()); - assert_eq!(height, block_idx as u32); + assert_eq!(height, block_idx as i32); let depth_response = read_state .oneshot(zebra_state::ReadRequest::Depth(block.hash())) diff --git a/zebra-rpc/src/methods/types/get_block_template.rs b/zebra-rpc/src/methods/types/get_block_template.rs index fe5a8a11d3b..ce563f3e04f 100644 --- a/zebra-rpc/src/methods/types/get_block_template.rs +++ b/zebra-rpc/src/methods/types/get_block_template.rs @@ -446,6 +446,10 @@ where mined_block_sender: watch::Sender<(block::Hash, block::Height)>, } +// A limit on the configured extra coinbase data, regardless of the current block height. +// This is different from the consensus rule, which limits the total height + data. +const EXTRA_COINBASE_DATA_LIMIT: usize = MAX_COINBASE_DATA_LEN - MAX_COINBASE_HEIGHT_DATA_LEN; + impl GetBlockTemplateHandler where BlockVerifierRouter: Service @@ -479,11 +483,6 @@ where } }); - // A limit on the configured extra coinbase data, regardless of the current block height. - // This is different from the consensus rule, which limits the total height + data. - const EXTRA_COINBASE_DATA_LIMIT: usize = - MAX_COINBASE_DATA_LEN - MAX_COINBASE_HEIGHT_DATA_LEN; - // Hex-decode to bytes if possible, otherwise UTF-8 encode to bytes. let extra_coinbase_data = conf .extra_coinbase_data @@ -519,6 +518,21 @@ where self.extra_coinbase_data.clone() } + /// Changes the extra coinbase data. + /// + /// # Panics + /// + /// If `extra_coinbase_data` exceeds [`EXTRA_COINBASE_DATA_LIMIT`]. + pub fn set_extra_coinbase_data(&mut self, extra_coinbase_data: Vec) { + assert!( + extra_coinbase_data.len() <= EXTRA_COINBASE_DATA_LIMIT, + "extra coinbase data is {} bytes, but Zebra's limit is {}.", + extra_coinbase_data.len(), + EXTRA_COINBASE_DATA_LIMIT, + ); + self.extra_coinbase_data = extra_coinbase_data; + } + /// Returns the sync status. pub fn sync_status(&self) -> SyncStatus { self.sync_status.clone() diff --git a/zebra-rpc/src/methods/types/transaction.rs b/zebra-rpc/src/methods/types/transaction.rs index 1c75c03c6f5..1cf4df84b0b 100644 --- a/zebra-rpc/src/methods/types/transaction.rs +++ b/zebra-rpc/src/methods/types/transaction.rs @@ -8,7 +8,6 @@ use derive_getters::Getters; use derive_new::new; use hex::ToHex; -use serde_with::serde_as; use zebra_chain::{ amount::{self, Amount, NegativeOrZero, NonNegative}, block::{self, merkle::AUTH_DIGEST_PLACEHOLDER, Height}, @@ -158,13 +157,14 @@ pub struct TransactionObject { /// The raw transaction, encoded as hex bytes. #[serde(with = "hex")] pub(crate) hex: SerializedTransaction, - /// The height of the block in the best chain that contains the tx or `None` if the tx is in - /// the mempool. + /// The height of the block in the best chain that contains the tx, -1 if + /// it's in a side chain block, or `None` if the tx is in the mempool. #[serde(skip_serializing_if = "Option::is_none")] #[getter(copy)] - pub(crate) height: Option, - /// The height diff between the block containing the tx and the best chain tip + 1 or `None` - /// if the tx is in the mempool. + pub(crate) height: Option, + /// The height diff between the block containing the tx and the best chain + /// tip + 1, 0 if it's in a side chain, or `None` if the tx is in the + /// mempool. #[serde(skip_serializing_if = "Option::is_none")] #[getter(copy)] pub(crate) confirmations: Option, @@ -625,8 +625,24 @@ impl TransactionObject { let block_time = block_time.map(|bt| bt.timestamp()); Self { hex: tx.clone().into(), - height: height.map(|height| height.0), - confirmations, + height: if in_active_chain.unwrap_or_default() { + height.map(|height| height.0 as i32) + } else if block_hash.is_some() { + // Side chain + Some(-1) + } else { + // Mempool + None + }, + confirmations: if in_active_chain.unwrap_or_default() { + confirmations + } else if block_hash.is_some() { + // Side chain + Some(0) + } else { + // Mempool + None + }, inputs: tx .inputs() .iter() diff --git a/zebra-state/src/lib.rs b/zebra-state/src/lib.rs index a1624880f63..5d156e7bb92 100644 --- a/zebra-state/src/lib.rs +++ b/zebra-state/src/lib.rs @@ -48,7 +48,7 @@ pub use request::{ #[cfg(feature = "indexer")] pub use request::Spend; -pub use response::{GetBlockTemplateChainInfo, KnownBlock, MinedTx, ReadResponse, Response}; +pub use response::{AnyTx, GetBlockTemplateChainInfo, KnownBlock, MinedTx, ReadResponse, Response}; pub use service::{ chain_tip::{ChainTipBlock, ChainTipChange, ChainTipSender, LatestChainTip, TipAction}, check, diff --git a/zebra-state/src/request.rs b/zebra-state/src/request.rs index 954081edfb9..d77a32b3f60 100644 --- a/zebra-state/src/request.rs +++ b/zebra-state/src/request.rs @@ -733,6 +733,16 @@ pub enum Request { /// * [`Response::Transaction(None)`](Response::Transaction) otherwise. Transaction(transaction::Hash), + /// Looks up a transaction by hash in any chain. + /// + /// Returns + /// + /// * [`Response::AnyChainTransaction(Some(AnyTx))`](Response::AnyChainTransaction) + /// if the transaction is in any chain; + /// * [`Response::AnyChainTransaction(None)`](Response::AnyChainTransaction) + /// otherwise. + AnyChainTransaction(transaction::Hash), + /// Looks up a UTXO identified by the given [`OutPoint`](transparent::OutPoint), /// returning `None` immediately if it is unknown. /// @@ -884,6 +894,7 @@ impl Request { Request::Tip => "tip", Request::BlockLocator => "block_locator", Request::Transaction(_) => "transaction", + Request::AnyChainTransaction(_) => "any_chain_transaction", Request::UnspentBestChainUtxo { .. } => "unspent_best_chain_utxo", Request::Block(_) => "block", Request::BlockAndSize(_) => "block_and_size", @@ -980,6 +991,16 @@ pub enum ReadRequest { /// * [`ReadResponse::Transaction(None)`](ReadResponse::Transaction) otherwise. Transaction(transaction::Hash), + /// Looks up a transaction by hash in any chain. + /// + /// Returns + /// + /// * [`ReadResponse::AnyChainTransaction(Some(AnyTx))`](ReadResponse::AnyChainTransaction) + /// if the transaction is in any chain; + /// * [`ReadResponse::AnyChainTransaction(None)`](ReadResponse::AnyChainTransaction) + /// otherwise. + AnyChainTransaction(transaction::Hash), + /// Looks up the transaction IDs for a block, using a block hash or height. /// /// Returns @@ -992,6 +1013,20 @@ pub enum ReadRequest { /// Returned txids are in the order they appear in the block. TransactionIdsForBlock(HashOrHeight), + /// Looks up the transaction IDs for a block, using a block hash or height, + /// for any chain. + /// + /// Returns + /// + /// * An ordered list of transaction hashes and a flag indicating whether + /// the block is in the best chain, or + /// * `None` if the block was not found. + /// + /// Note: Each block has at least one transaction: the coinbase transaction. + /// + /// Returned txids are in the order they appear in the block. + AnyChainTransactionIdsForBlock(HashOrHeight), + /// Looks up a UTXO identified by the given [`OutPoint`](transparent::OutPoint), /// returning `None` immediately if it is unknown. /// @@ -1213,7 +1248,9 @@ impl ReadRequest { ReadRequest::BlockAndSize(_) => "block_and_size", ReadRequest::BlockHeader(_) => "block_header", ReadRequest::Transaction(_) => "transaction", + ReadRequest::AnyChainTransaction(_) => "any_chain_transaction", ReadRequest::TransactionIdsForBlock(_) => "transaction_ids_for_block", + ReadRequest::AnyChainTransactionIdsForBlock(_) => "any_chain_transaction_ids_for_block", ReadRequest::UnspentBestChainUtxo { .. } => "unspent_best_chain_utxo", ReadRequest::AnyChainUtxo { .. } => "any_chain_utxo", ReadRequest::BlockLocator => "block_locator", @@ -1269,6 +1306,7 @@ impl TryFrom for ReadRequest { Request::BlockAndSize(hash_or_height) => Ok(ReadRequest::BlockAndSize(hash_or_height)), Request::BlockHeader(hash_or_height) => Ok(ReadRequest::BlockHeader(hash_or_height)), Request::Transaction(tx_hash) => Ok(ReadRequest::Transaction(tx_hash)), + Request::AnyChainTransaction(tx_hash) => Ok(ReadRequest::AnyChainTransaction(tx_hash)), Request::UnspentBestChainUtxo(outpoint) => { Ok(ReadRequest::UnspentBestChainUtxo(outpoint)) } diff --git a/zebra-state/src/response.rs b/zebra-state/src/response.rs index f1d73cbd8dd..3a33820e9e5 100644 --- a/zebra-state/src/response.rs +++ b/zebra-state/src/response.rs @@ -58,6 +58,9 @@ pub enum Response { /// Response to [`Request::Transaction`] with the specified transaction. Transaction(Option>), + /// Response to [`Request::AnyChainTransaction`] with the specified transaction. + AnyChainTransaction(Option), + /// Response to [`Request::UnspentBestChainUtxo`] with the UTXO UnspentBestChainUtxo(Option), @@ -121,6 +124,24 @@ pub enum KnownBlock { Queue, } +/// Information about a transaction in any chain. +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum AnyTx { + /// A transaction in the best chain. + Mined(MinedTx), + /// A transaction in a side chain, and the hash of the block it is in. + Side((Arc, block::Hash)), +} + +impl From for Arc { + fn from(any_tx: AnyTx) -> Self { + match any_tx { + AnyTx::Mined(mined_tx) => mined_tx.tx, + AnyTx::Side((tx, _)) => tx, + } + } +} + /// Information about a transaction in the best chain #[derive(Clone, Debug, PartialEq, Eq)] pub struct MinedTx { @@ -311,11 +332,19 @@ pub enum ReadResponse { /// Response to [`ReadRequest::Transaction`] with the specified transaction. Transaction(Option), + /// Response to [`Request::Transaction`] with the specified transaction. + AnyChainTransaction(Option), + /// Response to [`ReadRequest::TransactionIdsForBlock`], /// with an list of transaction hashes in block order, /// or `None` if the block was not found. TransactionIdsForBlock(Option>), + /// Response to [`ReadRequest::AnyChainTransactionIdsForBlock`], with an list of + /// transaction hashes in block order and a flag indicating if the block is + /// in the best chain, or `None` if the block was not found. + AnyChainTransactionIdsForBlock(Option<(Arc<[transaction::Hash]>, bool)>), + /// Response to [`ReadRequest::SpendingTransactionId`], /// with an list of transaction hashes in block order, /// or `None` if the block was not found. @@ -471,6 +500,7 @@ impl TryFrom for Response { ReadResponse::Transaction(tx_info) => { Ok(Response::Transaction(tx_info.map(|tx_info| tx_info.tx))) } + ReadResponse::AnyChainTransaction(tx) => Ok(Response::AnyChainTransaction(tx)), ReadResponse::UnspentBestChainUtxo(utxo) => Ok(Response::UnspentBestChainUtxo(utxo)), @@ -487,6 +517,7 @@ impl TryFrom for Response { | ReadResponse::TipPoolValues { .. } | ReadResponse::BlockInfo(_) | ReadResponse::TransactionIdsForBlock(_) + | ReadResponse::AnyChainTransactionIdsForBlock(_) | ReadResponse::SaplingTree(_) | ReadResponse::OrchardTree(_) | ReadResponse::SaplingSubtrees(_) diff --git a/zebra-state/src/service.rs b/zebra-state/src/service.rs index 6fd2db3f98c..4da4b5bcc69 100644 --- a/zebra-state/src/service.rs +++ b/zebra-state/src/service.rs @@ -1219,6 +1219,7 @@ impl Service for StateService { | Request::BestChainBlockHash(_) | Request::BlockLocator | Request::Transaction(_) + | Request::AnyChainTransaction(_) | Request::UnspentBestChainUtxo(_) | Request::Block(_) | Request::BlockAndSize(_) @@ -1550,6 +1551,30 @@ impl Service for ReadStateService { .wait_for_panics() } + ReadRequest::AnyChainTransaction(hash) => { + let state = self.clone(); + + tokio::task::spawn_blocking(move || { + span.in_scope(move || { + let tx = state.non_finalized_state_receiver.with_watch_data( + |non_finalized_state| { + read::any_transaction( + non_finalized_state.chain_iter(), + &state.db, + hash, + ) + }, + ); + + // The work is done in the future. + timer.finish(module_path!(), line!(), "ReadRequest::AnyChainTransaction"); + + Ok(ReadResponse::AnyChainTransaction(tx)) + }) + }) + .wait_for_panics() + } + // Used by the getblock (verbose) RPC. ReadRequest::TransactionIdsForBlock(hash_or_height) => { let state = self.clone(); @@ -1579,6 +1604,36 @@ impl Service for ReadStateService { .wait_for_panics() } + ReadRequest::AnyChainTransactionIdsForBlock(hash_or_height) => { + let state = self.clone(); + + tokio::task::spawn_blocking(move || { + span.in_scope(move || { + let transaction_ids = state.non_finalized_state_receiver.with_watch_data( + |non_finalized_state| { + read::transaction_hashes_for_any_block( + non_finalized_state.chain_iter(), + &state.db, + hash_or_height, + ) + }, + ); + + // The work is done in the future. + timer.finish( + module_path!(), + line!(), + "ReadRequest::AnyChainTransactionIdsForBlock", + ); + + Ok(ReadResponse::AnyChainTransactionIdsForBlock( + transaction_ids, + )) + }) + }) + .wait_for_panics() + } + #[cfg(feature = "indexer")] ReadRequest::SpendingTransactionId(spend) => { let state = self.clone(); diff --git a/zebra-state/src/service/non_finalized_state.rs b/zebra-state/src/service/non_finalized_state.rs index 3444fa584e1..d73e65e9ada 100644 --- a/zebra-state/src/service/non_finalized_state.rs +++ b/zebra-state/src/service/non_finalized_state.rs @@ -528,7 +528,11 @@ impl NonFinalizedState { prepared: SemanticallyVerifiedBlock, finalized_state: &ZebraDb, ) -> Result, ValidateContextError> { - if self.invalidated_blocks.contains_key(&prepared.height) { + if self + .invalidated_blocks + .values() + .any(|blocks| blocks.iter().any(|block| block.hash == prepared.hash)) + { return Err(ValidateContextError::BlockPreviouslyInvalidated { block_hash: prepared.hash, }); diff --git a/zebra-state/src/service/read.rs b/zebra-state/src/service/read.rs index c0f8c4f26b7..6f75e11081d 100644 --- a/zebra-state/src/service/read.rs +++ b/zebra-state/src/service/read.rs @@ -29,8 +29,8 @@ pub use address::{ utxo::{address_utxos, AddressUtxos}, }; pub use block::{ - any_utxo, block, block_and_size, block_header, block_info, mined_transaction, - transaction_hashes_for_block, unspent_utxo, + any_transaction, any_utxo, block, block_and_size, block_header, block_info, mined_transaction, + transaction_hashes_for_any_block, transaction_hashes_for_block, unspent_utxo, }; #[cfg(feature = "indexer")] diff --git a/zebra-state/src/service/read/block.rs b/zebra-state/src/service/read/block.rs index df0a9673c58..cc713c3d503 100644 --- a/zebra-state/src/service/read/block.rs +++ b/zebra-state/src/service/read/block.rs @@ -25,7 +25,7 @@ use zebra_chain::{ }; use crate::{ - response::MinedTx, + response::{AnyTx, MinedTx}, service::{ finalized_state::ZebraDb, non_finalized_state::{Chain, NonFinalizedState}, @@ -149,6 +149,47 @@ where Some(MinedTx::new(tx, height, confirmations, time)) } +/// Returns a [`AnyTx`] for a [`Transaction`] with [`transaction::Hash`], +/// if one exists in any chain in `chains` or finalized `db`. +/// The first chain in `chains` must be the best chain. +pub fn any_transaction<'a>( + chains: impl Iterator>, + db: &ZebraDb, + hash: transaction::Hash, +) -> Option { + // # Correctness + // + // It is ok to do this lookup in multiple different calls. Finalized state updates + // can only add overlapping blocks, and hashes are unique. + let mut best_chain = None; + let (tx, height, time, in_best_chain, containing_chain) = chains + .enumerate() + .find_map(|(i, chain)| { + chain.as_ref().transaction(hash).map(|(tx, height, time)| { + if i == 0 { + best_chain = Some(chain); + } + (tx.clone(), height, time, i == 0, Some(chain)) + }) + }) + .or_else(|| { + db.transaction(hash) + .map(|(tx, height, time)| (tx.clone(), height, time, true, None)) + })?; + + if in_best_chain { + let confirmations = 1 + tip_height(best_chain, db)?.0 - height.0; + Some(AnyTx::Mined(MinedTx::new(tx, height, confirmations, time))) + } else { + let block_hash = containing_chain + .expect("if not in best chain, then it must be in a side chain") + .block(height.into()) + .expect("must exist since tx is in chain") + .hash; + Some(AnyTx::Side((tx, block_hash))) + } +} + /// Returns the [`transaction::Hash`]es for the block with `hash_or_height`, /// if it exists in the non-finalized `chain` or finalized `db`. /// @@ -174,6 +215,37 @@ where .or_else(|| db.transaction_hashes_for_block(hash_or_height)) } +/// Returns the [`transaction::Hash`]es for the block with `hash_or_height`, +/// if it exists in any chain in `chains` or finalized `db`. +/// The first chain in `chains` must be the best chain. +/// +/// The returned hashes are in block order. +/// +/// Returns `None` if the block is not found. +pub fn transaction_hashes_for_any_block<'a>( + chains: impl Iterator>, + db: &ZebraDb, + hash_or_height: HashOrHeight, +) -> Option<(Arc<[transaction::Hash]>, bool)> { + // # Correctness + // + // Since blocks are the same in the finalized and non-finalized state, we + // check the most efficient alternative first. (`chain` is always in memory, + // but `db` stores blocks on disk, with a memory cache.) + chains + .enumerate() + .find_map(|(i, chain)| { + chain + .as_ref() + .transaction_hashes_for_block(hash_or_height) + .map(|hashes| (hashes.clone(), i == 0)) + }) + .or_else(|| { + db.transaction_hashes_for_block(hash_or_height) + .map(|hashes| (hashes, true)) + }) +} + /// Returns the [`Utxo`] for [`transparent::OutPoint`], if it exists in the /// non-finalized `chain` or finalized `db`. /// diff --git a/zebrad/src/components/tracing/component.rs b/zebrad/src/components/tracing/component.rs index a2a21319407..721a38ee636 100644 --- a/zebrad/src/components/tracing/component.rs +++ b/zebrad/src/components/tracing/component.rs @@ -97,7 +97,8 @@ impl Tracing { let flame_root = &config.flamegraph; // Only show the intro for user-focused node server commands like `start` - if uses_intro { + // Also skip the intro for regtest, since it pollutes the QA test logs + if uses_intro && !network.is_regtest() { // If it's a terminal and color escaping is enabled: clear screen and // print Zebra logo (here `use_color` is being interpreted as // "use escape codes") diff --git a/zebrad/tests/acceptance.rs b/zebrad/tests/acceptance.rs index 764e43ef3a4..4d096f8fddf 100644 --- a/zebrad/tests/acceptance.rs +++ b/zebrad/tests/acceptance.rs @@ -3833,7 +3833,11 @@ async fn invalidate_and_reconsider_block() -> Result<()> { tracing::info!("invalidating blocks"); // Note: This is the block at height 7, it's the 6th generated block. - let block_6_hash = blocks.get(5).expect("should have 50 blocks").hash(); + let block_6_hash = blocks + .get(5) + .expect("should have 50 blocks") + .hash() + .to_string(); let params = serde_json::to_string(&vec![block_6_hash]).expect("should serialize successfully"); let _: () = rpc_client From b906ecd7492bbaf357f8b43fc6f66567cb4e5507 Mon Sep 17 00:00:00 2001 From: Gustavo Valverde Date: Tue, 30 Sep 2025 19:26:48 +0100 Subject: [PATCH 006/431] test(lwd): ensure lwd full sync populates the cache disk (#9936) * test(lightwalletd): clarify missing cached state failure - panic with actionable guidance when the expected lightwalletd db is absent - keep cached-state logging for existing directories unchanged * test(lightwalletd): reuse shared cache during full sync - return the default LWD cache dir when a test builds state so lwd-sync-full populates the shared disk image - log zebrad and lightwalletd tip heights while waiting for sync to aid future CI debugging --------- Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> --- zebrad/tests/common/lightwalletd.rs | 18 +++++++++++++++++- zebrad/tests/common/lightwalletd/sync.rs | 16 ++++++++++++++++ zebrad/tests/common/test_type.rs | 12 ++++++++++++ 3 files changed, 45 insertions(+), 1 deletion(-) diff --git a/zebrad/tests/common/lightwalletd.rs b/zebrad/tests/common/lightwalletd.rs index 6ed8719f944..6e40014d6f7 100644 --- a/zebrad/tests/common/lightwalletd.rs +++ b/zebrad/tests/common/lightwalletd.rs @@ -218,7 +218,23 @@ where if !matches!(test_type, TestType::FullSyncFromGenesis { .. }) { let lwd_cache_dir_path = lightwalletd_state_path.join("db/main"); let lwd_cache_entries: Vec<_> = std::fs::read_dir(&lwd_cache_dir_path) - .expect("unexpected failure reading lightwalletd cache dir") + .unwrap_or_else(|error| { + if error.kind() == std::io::ErrorKind::NotFound { + panic!( + "missing cached lightwalletd state at {path:?}.\n\ + Populate the directory (for example by running the lwd-sync-full \n\ + nextest profile) or set {env_var} to a populated cache.", + path = lwd_cache_dir_path, + env_var = LWD_CACHE_DIR, + ); + } + + panic!( + "unexpected failure opening lightwalletd cache dir {path:?}: {error:?}", + path = lwd_cache_dir_path, + error = error, + ); + }) .collect(); let lwd_cache_dir_size = lwd_cache_entries.iter().fold(0, |acc, entry_result| { diff --git a/zebrad/tests/common/lightwalletd/sync.rs b/zebrad/tests/common/lightwalletd/sync.rs index f5e1534db8c..e152ec9694f 100644 --- a/zebrad/tests/common/lightwalletd/sync.rs +++ b/zebrad/tests/common/lightwalletd/sync.rs @@ -214,6 +214,22 @@ pub fn are_zebrad_and_lightwalletd_tips_synced( .as_u64() .expect("unexpected block height: doesn't fit in u64"); + if lightwalletd_tip_height != zebrad_tip_height { + tracing::info!( + lightwalletd_tip_height, + zebrad_tip_height, + zebra_rpc_address = ?zebra_rpc_address, + "lightwalletd tip is behind Zebra tip, waiting for sync", + ); + } else { + tracing::debug!( + lightwalletd_tip_height, + zebrad_tip_height, + zebra_rpc_address = ?zebra_rpc_address, + "lightwalletd tip matches Zebra tip", + ); + } + Ok(lightwalletd_tip_height == zebrad_tip_height) }) } diff --git a/zebrad/tests/common/test_type.rs b/zebrad/tests/common/test_type.rs index e809e2b6895..0975ecf5234 100644 --- a/zebrad/tests/common/test_type.rs +++ b/zebrad/tests/common/test_type.rs @@ -313,6 +313,18 @@ impl TestType { ); None } + } else if self.can_create_lightwalletd_cached_state() { + // Ensure the directory exists so FullSyncFromGenesis can populate it. + if let Err(error) = std::fs::create_dir_all(&default_path) { + tracing::warn!( + ?default_path, + ?error, + "failed to create default lightwalletd cache directory; using an ephemeral temp dir instead", + ); + None + } else { + Some(default_path) + } } else { None } From 9db326dacfd19ef0258cfb90424b234b013c128d Mon Sep 17 00:00:00 2001 From: Za Wil Date: Tue, 30 Sep 2025 16:07:25 -0700 Subject: [PATCH 007/431] add useful standard trait implementations to ease downstream use (#9926) * add Copy, Debug, and PartialEq to ConfiguredActivationHeights * fix clippy lint --- zebra-chain/src/parameters/network/testnet.rs | 2 +- zebra-network/src/config.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/zebra-chain/src/parameters/network/testnet.rs b/zebra-chain/src/parameters/network/testnet.rs index 74c5c7585b0..ca6b9ccc967 100644 --- a/zebra-chain/src/parameters/network/testnet.rs +++ b/zebra-chain/src/parameters/network/testnet.rs @@ -301,7 +301,7 @@ fn check_funding_stream_address_period(funding_streams: &FundingStreams, network } /// Configurable activation heights for Regtest and configured Testnets. -#[derive(Serialize, Deserialize, Default, Debug, Clone)] +#[derive(Serialize, Deserialize, Default, Clone, Copy, Debug, PartialEq)] #[serde(rename_all = "PascalCase", deny_unknown_fields)] pub struct ConfiguredActivationHeights { /// Activation height for `BeforeOverwinter` network upgrade. diff --git a/zebra-network/src/config.rs b/zebra-network/src/config.rs index 0ba2ec4bfa1..fe119b980d9 100644 --- a/zebra-network/src/config.rs +++ b/zebra-network/src/config.rs @@ -830,7 +830,7 @@ impl<'de> Deserialize<'de> for Config { } // Retain default Testnet activation heights unless there's an empty [testnet_parameters.activation_heights] section. - if let Some(activation_heights) = activation_heights.clone() { + if let Some(activation_heights) = activation_heights { params_builder = params_builder.with_activation_heights(activation_heights) } From 01037b939db9af8e9f1603ad3289e782df08a05c Mon Sep 17 00:00:00 2001 From: Block Wizard Date: Wed, 1 Oct 2025 14:28:50 +0300 Subject: [PATCH 008/431] chore: fix typos in docs. (#9943) --- book/src/dev/state-db-upgrades.md | 2 +- zebra-chain/src/history_tree/tests/vectors.rs | 2 +- zebra-chain/src/primitives/zcash_history/tests/vectors.rs | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/book/src/dev/state-db-upgrades.md b/book/src/dev/state-db-upgrades.md index dd9e5bcd587..60d51b889f5 100644 --- a/book/src/dev/state-db-upgrades.md +++ b/book/src/dev/state-db-upgrades.md @@ -281,7 +281,7 @@ So it is better to test with a full sync, and an older cached state. [current]: #current rocksdb provides a persistent, thread-safe `BTreeMap<&[u8], &[u8]>`. Each map is -a distinct "tree". Keys are sorted using lexographic order (`[u8].sorted()`) on byte strings, so +a distinct "tree". Keys are sorted using lexicographic order (`[u8].sorted()`) on byte strings, so integer values should be stored using big-endian encoding (so that the lex order on byte strings is the numeric ordering). diff --git a/zebra-chain/src/history_tree/tests/vectors.rs b/zebra-chain/src/history_tree/tests/vectors.rs index e8aeeeb447d..02a65bc9b5b 100644 --- a/zebra-chain/src/history_tree/tests/vectors.rs +++ b/zebra-chain/src/history_tree/tests/vectors.rs @@ -94,7 +94,7 @@ fn push_and_prune_for_network_upgrade( tree.push(second_block, &second_sapling_root, &Default::default()) .unwrap(); - // Adding a second block will produce a 3-node tree (one parent and two leafs). + // Adding a second block will produce a 3-node tree (one parent and two leaves). assert_eq!(tree.size(), 3); // The tree must have been pruned, resulting in a single peak (the parent). assert_eq!(tree.peaks().len(), 1); diff --git a/zebra-chain/src/primitives/zcash_history/tests/vectors.rs b/zebra-chain/src/primitives/zcash_history/tests/vectors.rs index d23cb078d51..643923fbad6 100644 --- a/zebra-chain/src/primitives/zcash_history/tests/vectors.rs +++ b/zebra-chain/src/primitives/zcash_history/tests/vectors.rs @@ -73,7 +73,7 @@ fn tree_for_network_upgrade(network: &Network, network_upgrade: NetworkUpgrade) .append_leaf(block1, &sapling_root1, &Default::default()) .unwrap(); - // Tree how has 3 nodes: two leafs for each block, and one parent node + // Tree how has 3 nodes: two leaves for each block, and one parent node // which is the new root assert_eq!(tree.inner.len(), 3); // Two nodes were appended: the new leaf and the parent node From 19bca3f1159f9cb9344c9944f7e1cb8d6a82a07f Mon Sep 17 00:00:00 2001 From: Gustavo Valverde Date: Wed, 1 Oct 2025 17:45:34 +0100 Subject: [PATCH 009/431] feat(health): add HTTP `/healthy` and `/ready` endpoints (#9895) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat(health): add HTTP `/healthy` and `/ready` endpoints Add a minimal HTTP/1.1 health server to `zebrad` with two endpoints: - GET /healthy: 200 when process is up and there are at least `min_connected_peers` recently live peers (default: 1); otherwise 503. - GET /ready: 200 when the node is near the tip and the estimated lag is ≤ `ready_max_blocks_behind` (default: 2); otherwise 503. The server is disabled by default and enabled via a new `[health]` config section with: - `listen_addr` - `min_connected_peers` - `ready_max_blocks_behind` - `enforce_on_test_networks` Closes #8830 --------- Co-authored-by: Arya --- CHANGELOG.md | 12 + Cargo.lock | 1 + README.md | 3 + book/src/SUMMARY.md | 1 + book/src/user/docker.md | 34 +- book/src/user/health.md | 63 ++++ book/src/user/run.md | 2 + docker/default-zebra-config.toml | 10 + docker/docker-compose.yml | 1 + .../src/peer_set/initialize/tests/vectors.rs | 6 +- zebrad/Cargo.toml | 14 +- zebrad/src/commands/start.rs | 18 + zebrad/src/components.rs | 1 + zebrad/src/components/health.rs | 355 ++++++++++++++++++ zebrad/src/components/health/config.rs | 42 +++ zebrad/src/components/health/tests.rs | 223 +++++++++++ zebrad/src/components/sync/progress.rs | 40 +- zebrad/src/config.rs | 6 + zebrad/tests/common/configs/v2.6.0.toml | 6 + 19 files changed, 816 insertions(+), 22 deletions(-) create mode 100644 book/src/user/health.md create mode 100644 zebrad/src/components/health.rs create mode 100644 zebrad/src/components/health/config.rs create mode 100644 zebrad/src/components/health/tests.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index 36e18bcc209..3c3f98928a9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3773,3 +3773,15 @@ The goals of this release are to: Currently, Zebra does not validate all the Zcash consensus rules. It may be unreliable on Testnet, and under less-than-perfect network conditions. +### Added + +- zebrad: Optional HTTP health endpoints for cloud-native readiness and liveness checks. + When configured, zebrad serves two simple HTTP/1.1 endpoints on a dedicated listener: + - GET /healthy: returns 200 when the process is up and has at least the configured number of recently live peers; otherwise 503. + - GET /ready: returns 200 when the node is near the chain tip and the estimated block lag is within the configured threshold; otherwise 503. + Configure via the new [health] section in zebrad.toml: + - health.listen_addr (optional, enables the server when set) + - health.min_connected_peers (default 1) + - health.ready_max_blocks_behind (default 2) + - health.enforce_on_test_networks (default false) + See the Zebra Book for examples and Kubernetes probes: https://zebra.zfnd.org/user/health.html diff --git a/Cargo.lock b/Cargo.lock index 63b1f1432f5..92ce4c0cbe5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -7079,6 +7079,7 @@ dependencies = [ "color-eyre", "config", "console-subscriber", + "derive-new", "dirs", "futures", "hex", diff --git a/README.md b/README.md index 152ab2b38c6..faa9f14fffc 100644 --- a/README.md +++ b/README.md @@ -131,6 +131,9 @@ The Zcash Foundation maintains the following resources documenting Zebra: - [User Documentation](https://zebra.zfnd.org/user.html), - [Developer Documentation](https://zebra.zfnd.org/dev.html). + - User guides of note: + - [Zebra Health Endpoints](https://zebra.zfnd.org/user/health.html) — liveness/readiness checks for Kubernetes and load balancers + - The [documentation of the public APIs](https://docs.rs/zebrad/latest/zebrad/#zebra-crates) for the latest releases of the individual Zebra crates. diff --git a/book/src/SUMMARY.md b/book/src/SUMMARY.md index 6a128310ffb..2df57261a79 100644 --- a/book/src/SUMMARY.md +++ b/book/src/SUMMARY.md @@ -11,6 +11,7 @@ - [Zebra with Docker](user/docker.md) - [Tracing Zebra](user/tracing.md) - [Zebra Metrics](user/metrics.md) + - [Zebra Health Endpoints](user/health.md) - [Lightwalletd](user/lightwalletd.md) - [zk-SNARK Parameters](user/parameters.md) - [Mining](user/mining.md) diff --git a/book/src/user/docker.md b/book/src/user/docker.md index 35cd1c1a976..ae3be6c3ad5 100644 --- a/book/src/user/docker.md +++ b/book/src/user/docker.md @@ -58,7 +58,6 @@ docker build \ See [Building Zebra](https://github.com/ZcashFoundation/zebra#manual-build) for more information. - ### Building with Custom Features Zebra supports various features that can be enabled during build time using the `FEATURES` build argument: @@ -120,6 +119,39 @@ By default, Zebra uses cookie-based authentication for RPC requests (`enable_coo Remember that Zebra only generates the cookie file if the RPC server is enabled *and* `enable_cookie_auth` is set to `true` (or omitted, as `true` is the default). +Environment variable examples for health endpoints: + +* `ZEBRA_HEALTH__LISTEN_ADDR=0.0.0.0:8080` +* `ZEBRA_HEALTH__MIN_CONNECTED_PEERS=1` +* `ZEBRA_HEALTH__READY_MAX_BLOCKS_BEHIND=2` +* `ZEBRA_HEALTH__ENFORCE_ON_TEST_NETWORKS=false` + +### Health Endpoints + +Zebra can expose two lightweight HTTP endpoints for liveness and readiness: + +* `GET /healthy`: returns `200 OK` when the process is up and has at least the configured number of recently live peers; otherwise `503`. +* `GET /ready`: returns `200 OK` when the node is near the tip and within the configured lag threshold; otherwise `503`. + +Enable the endpoints by adding a `[health]` section to your config (see the default Docker config at `docker/default-zebra-config.toml`): + +```toml +[health] +listen_addr = "0.0.0.0:8080" +min_connected_peers = 1 +ready_max_blocks_behind = 2 +enforce_on_test_networks = false +``` + +If you want to expose the endpoints to the host, add a port mapping to your compose file: + +```yaml +ports: + - "8080:8080" # Health endpoints (/healthy, /ready) +``` + +For Kubernetes, configure liveness and readiness probes against `/healthy` and `/ready` respectively. See the [Health Endpoints](./health.md) page for details. + ## Examples To make the initial setup of Zebra with other services easier, we provide some diff --git a/book/src/user/health.md b/book/src/user/health.md new file mode 100644 index 00000000000..f355fc6d0f5 --- /dev/null +++ b/book/src/user/health.md @@ -0,0 +1,63 @@ +# Zebra Health Endpoints + +`zebrad` can serve two lightweight HTTP endpoints for liveness and readiness checks. +These endpoints are intended for Kubernetes probes and load balancers. They are +disabled by default and can be enabled via configuration. + +## Endpoints + +- `GET /healthy` + - `200 OK`: process is up and has at least the configured number of recently + live peers (default: 1) + - `503 Service Unavailable`: not enough peers + +- `GET /ready` + - `200 OK`: node is near the network tip and the estimated block lag is within + the configured threshold (default: 2 blocks) + - `503 Service Unavailable`: still syncing or lag exceeds threshold + +## Configuration + +Add a `health` section to your `zebrad.toml`: + +```toml +[health] +listen_addr = "0.0.0.0:8080" # enable server; omit to disable +min_connected_peers = 1 # /healthy threshold +ready_max_blocks_behind = 2 # /ready threshold +enforce_on_test_networks = false # if false, /ready is always 200 on regtest/testnet +``` + +Config struct reference: [`components::health::Config`][health_config]. + +## Kubernetes Probes + +Example Deployment probes: + +```yaml +livenessProbe: + httpGet: + path: /healthy + port: 8080 + initialDelaySeconds: 10 + periodSeconds: 10 +readinessProbe: + httpGet: + path: /ready + port: 8080 + initialDelaySeconds: 10 + periodSeconds: 5 +``` + +## Security + +- Endpoints are unauthenticated and return minimal plain text. +- Bind to an internal address and restrict exposure with network policies, + firewall rules, or Service selectors. + +## Notes + +- Readiness combines a moving‑average “near tip” signal with a hard block‑lag cap. +- Adjust thresholds based on your SLA and desired routing behavior. + +[health_config]: https://docs.rs/zebrad/latest/zebrad/components/health/struct.Config.html diff --git a/book/src/user/run.md b/book/src/user/run.md index f308b8020f1..85f42b94209 100644 --- a/book/src/user/run.md +++ b/book/src/user/run.md @@ -17,6 +17,8 @@ You can run Zebra as a: - [mining backend](https://zebra.zfnd.org/user/mining.html), or - experimental [Sapling shielded transaction scanner](https://zebra.zfnd.org/user/shielded-scan.html). +For Kubernetes and load balancer integrations, Zebra provides simple HTTP health endpoints. See [Zebra Health Endpoints](./health.md). + ## Supported versions Always run a supported version of Zebra, and upgrade it regularly, so it doesn't become unsupported and halt. [More information](../dev/release-process.md#supported-releases). diff --git a/docker/default-zebra-config.toml b/docker/default-zebra-config.toml index 5b5e5b6ebd1..9d309a85481 100644 --- a/docker/default-zebra-config.toml +++ b/docker/default-zebra-config.toml @@ -58,6 +58,16 @@ use_color = true # endpoint_addr = "0.0.0.0:9999" # Prometheus +[health] +# Health endpoints are disabled by default. To enable them, uncomment the +# listen_addr below. These endpoints are intended for Kubernetes probes and +# load balancers. + +# listen_addr = "0.0.0.0:8080" # /healthy and /ready +# min_connected_peers = 1 +# ready_max_blocks_behind = 2 +# enforce_on_test_networks = false + [mining] # If you are going to use Zebra as a backend for a mining pool, set your mining # address. diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index b561312fe27..d22063a0ee3 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -31,6 +31,7 @@ services: # - "18233:18233" # peer connections on Testnet # - "9999:9999" # Metrics # - "3000:3000" # Tracing + # - "8080:8080" # Health endpoints (/healthy, /ready) configs: zebra-config: diff --git a/zebra-network/src/peer_set/initialize/tests/vectors.rs b/zebra-network/src/peer_set/initialize/tests/vectors.rs index 5c338521196..db982af96b7 100644 --- a/zebra-network/src/peer_set/initialize/tests/vectors.rs +++ b/zebra-network/src/peer_set/initialize/tests/vectors.rs @@ -1382,7 +1382,7 @@ async fn add_initial_peers_deadlock() { // still some extra peers left. const PEER_COUNT: usize = 200; const PEERSET_INITIAL_TARGET_SIZE: usize = 2; - const TIME_LIMIT: Duration = Duration::from_secs(10); + const TIME_LIMIT: Duration = Duration::from_secs(20); let _init_guard = zebra_test::init(); @@ -1424,7 +1424,9 @@ async fn add_initial_peers_deadlock() { "Test user agent".to_string(), ); - assert!(tokio::time::timeout(TIME_LIMIT, init_future).await.is_ok()); + tokio::time::timeout(TIME_LIMIT, init_future) + .await + .expect("should not timeout"); } /// Open a local listener on `listen_addr` for `network`. diff --git a/zebrad/Cargo.toml b/zebrad/Cargo.toml index fdd93ebc184..c6de4a807d9 100644 --- a/zebrad/Cargo.toml +++ b/zebrad/Cargo.toml @@ -83,7 +83,7 @@ elasticsearch = [ # Tracing and monitoring sentry = ["dep:sentry"] journald = ["tracing-journald"] -filter-reload = ["hyper", "http-body-util", "hyper-util", "bytes"] +filter-reload = [] progress-bar = [ "howudoin", @@ -198,6 +198,11 @@ metrics = { workspace = true } dirs = { workspace = true } atty = { workspace = true } +# Health check server dependencies +hyper = { workspace = true, features = ["server", "http1"] } +http-body-util = { workspace = true } +hyper-util = { workspace = true, features = ["service", "server", "tokio"] } + # Configuration management config = { workspace = true } @@ -218,10 +223,7 @@ inferno = { workspace = true, optional = true } tracing-journald = { workspace = true, optional = true } # prod feature filter-reload -hyper = { workspace = true, features = ["http1", "http2", "server"], optional = true } -http-body-util = { workspace = true, optional = true } -hyper-util = { workspace = true, optional = true } -bytes = { workspace = true, optional = true } +bytes = { workspace = true } # prod feature prometheus metrics-exporter-prometheus = { workspace = true, features = ["http-listener"], optional = true } @@ -243,6 +245,8 @@ proptest-derive = { workspace = true, optional = true } # test feature tokio-console console-subscriber = { workspace = true, optional = true } +derive-new.workspace = true + [build-dependencies] vergen-git2 = { workspace = true, features = ["cargo", "rustc"] } diff --git a/zebrad/src/commands/start.rs b/zebrad/src/commands/start.rs index ecf12716f0a..de1c550ac23 100644 --- a/zebrad/src/commands/start.rs +++ b/zebrad/src/commands/start.rs @@ -89,6 +89,7 @@ use zebra_rpc::{methods::RpcImpl, server::RpcServer, SubmitBlockChannel}; use crate::{ application::{build_version, user_agent, LAST_WARN_ERROR_LOG_SENDER}, components::{ + health, inbound::{self, InboundSetupData, MAX_INBOUND_RESPONSE_TIME}, mempool::{self, Mempool}, sync::{self, show_block_chain_progress, VERIFICATION_PIPELINE_SCALING_MULTIPLIER}, @@ -181,6 +182,8 @@ impl StartCmd { ) .await; + // Start health server if configured (after sync_status is available) + info!("initializing verifiers"); let (tx_verifier_setup_tx, tx_verifier_setup_rx) = oneshot::channel(); let (block_verifier_router, tx_verifier, consensus_task_handles, max_checkpoint_height) = @@ -321,15 +324,29 @@ impl StartCmd { ); info!("spawning progress logging task"); + let (chain_tip_metrics_sender, chain_tip_metrics_receiver) = + health::ChainTipMetrics::channel(); let progress_task_handle = tokio::spawn( show_block_chain_progress( config.network.network.clone(), latest_chain_tip.clone(), sync_status.clone(), + chain_tip_metrics_sender, ) .in_current_span(), ); + // Start health server if configured + info!("initializing health endpoints"); + let (health_task_handle, _) = health::init( + config.health.clone(), + config.network.network.clone(), + chain_tip_metrics_receiver, + sync_status.clone(), + address_book.clone(), + ) + .await; + // Spawn never ending end of support task. info!("spawning end of support checking task"); let end_of_support_task_handle = tokio::spawn( @@ -522,6 +539,7 @@ impl StartCmd { // ongoing tasks rpc_task_handle.abort(); rpc_tx_queue_handle.abort(); + health_task_handle.abort(); syncer_task_handle.abort(); block_gossip_task_handle.abort(); mempool_crawler_task_handle.abort(); diff --git a/zebrad/src/components.rs b/zebrad/src/components.rs index 43b051f1209..864c669f678 100644 --- a/zebrad/src/components.rs +++ b/zebrad/src/components.rs @@ -5,6 +5,7 @@ //! component and dependency injection models are designed to work together, but //! don't fit the async context well. +pub mod health; pub mod inbound; #[allow(missing_docs)] pub mod mempool; diff --git a/zebrad/src/components/health.rs b/zebrad/src/components/health.rs new file mode 100644 index 00000000000..87a99f60bab --- /dev/null +++ b/zebrad/src/components/health.rs @@ -0,0 +1,355 @@ +//! HTTP health and readiness endpoints for `zebrad`. +//! +//! Overview +//! +//! - This module exposes two small HTTP/1.1 endpoints for basic liveness/readiness checks, +//! suitable for Kubernetes probes and load balancers. +//! - Endpoints are opt-in, disabled by default. Enable by setting a `listen_addr` in the +//! `health` config section. +//! - Plain-text responses and small responses keep the checks fast and safe. +//! +//! Endpoints +//! +//! - `GET /healthy` — returns `200 OK` if the process is up and the node has at least +//! `min_connected_peers` recently live peers (default: 1). Otherwise `503 Service Unavailable`. +//! - `GET /ready` — returns `200 OK` if the node is near the chain tip, the estimated block lag is +//! within `ready_max_blocks_behind`, and the latest committed block is recent. On regtest/testnet, +//! readiness returns `200` unless `enforce_on_test_networks` is set. +//! +//! Security +//! +//! - Endpoints are unauthenticated by design. Bind to internal network interfaces, +//! and restrict exposure using network policy, firewall rules, and service configuration. +//! - The server does not access or return private data. It only summarises coarse node state. +//! +//! Configuration and examples +//! +//! - See the Zebra Book for configuration details and Kubernetes probe examples: +//! + +mod config; +#[cfg(test)] +mod tests; + +pub use config::Config; +use derive_new::new; + +use std::time::Instant; +use std::{convert::Infallible, net::SocketAddr, sync::Arc, time::Duration}; + +use bytes::Bytes; +use chrono::Utc; +use http_body_util::Full; +use hyper::header::{CONTENT_LENGTH, CONTENT_TYPE}; +use hyper::server::conn::http1; +use hyper::{ + body::Incoming, http::response::Builder as ResponseBuilder, Method, Request, Response, + StatusCode, +}; +use hyper_util::rt::TokioIo; +use tokio::{ + sync::watch, + task::JoinHandle, + time::{self, MissedTickBehavior}, +}; +use tracing::{debug, info, warn}; + +use zebra_chain::{chain_sync_status::ChainSyncStatus, parameters::Network}; +use zebra_network::AddressBookPeers; + +// Refresh peers on a short cadence so the cached snapshot tracks live network +// conditions closely without hitting the address book mutex on every request. +const PEER_METRICS_REFRESH_INTERVAL: Duration = Duration::from_secs(5); + +const METHOD_NOT_ALLOWED_MSG: &str = "method not allowed"; +const NOT_FOUND_MSG: &str = "not found"; + +/// The maximum number of requests that will be handled in a given time interval before requests are dropped. +const MAX_RECENT_REQUESTS: usize = 10_000; +const RECENT_REQUEST_INTERVAL: Duration = Duration::from_secs(5); + +#[derive(Clone)] +struct HealthCtx +where + SyncStatus: ChainSyncStatus + Clone + Send + Sync + 'static, +{ + config: Config, + network: Network, + chain_tip_metrics_receiver: watch::Receiver, + sync_status: SyncStatus, + num_live_peer_receiver: watch::Receiver, +} + +/// Metrics tracking how long it's been since +#[derive(Debug, Clone, PartialEq, Eq, new)] +pub struct ChainTipMetrics { + /// Last time the chain tip height increased. + pub last_chain_tip_grow_time: Instant, + /// Estimated distance between Zebra's chain tip and the network chain tip. + pub remaining_sync_blocks: Option, +} + +impl ChainTipMetrics { + /// Creates a new watch channel for reporting [`ChainTipMetrics`]. + pub fn channel() -> (watch::Sender, watch::Receiver) { + watch::channel(Self { + last_chain_tip_grow_time: Instant::now(), + remaining_sync_blocks: None, + }) + } +} + +/// Starts the health server if `listen_addr` is configured. +/// +/// Returns a task handle and the bound socket address. When disabled, returns a +/// pending task and `None` for the address. +/// +/// The server accepts HTTP/1.1 requests on a dedicated TCP listener and serves +/// two endpoints: `/healthy` and `/ready`. +/// +/// # Panics +/// +/// - If the configured `listen_addr` cannot be bound. +pub async fn init( + config: Config, + network: Network, + chain_tip_metrics_receiver: watch::Receiver, + sync_status: SyncStatus, + address_book: AddressBook, +) -> (JoinHandle<()>, Option) +where + AddressBook: AddressBookPeers + Clone + Send + Sync + 'static, + SyncStatus: ChainSyncStatus + Clone + Send + Sync + 'static, +{ + let Some(listen_addr) = config.listen_addr else { + return (tokio::spawn(std::future::pending()), None); + }; + + info!("opening health endpoint at {listen_addr}...",); + + let listener = tokio::net::TcpListener::bind(listen_addr) + .await + .unwrap_or_else(|e| panic!("Opening health endpoint listener {listen_addr:?} failed: {e:?}. Hint: Check if another zebrad is running, or change the health listen_addr in the config.")); + + let local = listener.local_addr().unwrap_or_else(|err| { + tracing::warn!(?err, "failed to read local addr from TcpListener"); + listen_addr + }); + + info!("opened health endpoint at {}", local); + + let (num_live_peer_sender, num_live_peer_receiver) = watch::channel(0); + + // Seed the watch channel with the first snapshot so early requests see + // a consistent view even before the refresher loop has ticked. + if let Some(metrics) = num_live_peers(&address_book).await { + let _ = num_live_peer_sender.send(metrics); + } + + // Refresh metrics in the background using a watch channel so request + // handlers can read the latest snapshot without taking locks. + let metrics_task = tokio::spawn(peer_metrics_refresh_task( + address_book.clone(), + num_live_peer_sender, + )); + + let shared = Arc::new(HealthCtx { + config, + network, + chain_tip_metrics_receiver, + sync_status, + num_live_peer_receiver, + }); + + let server_task = tokio::spawn(run_health_server(listener, shared)); + + // Keep both async tasks tied to a single JoinHandle so shutdown and + // abort semantics mirror other components. + let task = tokio::spawn(async move { + tokio::select! { + _ = metrics_task => {}, + _ = server_task => {}, + } + }); + + (task, Some(local)) +} + +async fn handle_request( + req: Request, + ctx: Arc>, +) -> Result>, Infallible> +where + SyncStatus: ChainSyncStatus + Clone + Send + Sync + 'static, +{ + // Hyper is already lightweight, but we still fence non-GET methods to keep + // these endpoints deterministic for probes. + if req.method() != Method::GET { + return Ok(simple_response( + StatusCode::METHOD_NOT_ALLOWED, + METHOD_NOT_ALLOWED_MSG, + )); + } + + let path = req.uri().path(); + let response = match path { + "/healthy" => healthy(&ctx).await, + "/ready" => ready(&ctx).await, + _ => simple_response(StatusCode::NOT_FOUND, NOT_FOUND_MSG), + }; + + Ok(response) +} + +// Liveness: ensure we still have the configured minimum of recently live peers, +// matching historical behaviour but fed from the cached snapshot to avoid +// mutex contention. +async fn healthy(ctx: &HealthCtx) -> Response> +where + SyncStatus: ChainSyncStatus + Clone + Send + Sync + 'static, +{ + if *ctx.num_live_peer_receiver.borrow() >= ctx.config.min_connected_peers { + simple_response(StatusCode::OK, "ok") + } else { + simple_response(StatusCode::SERVICE_UNAVAILABLE, "insufficient peers") + } +} + +// Readiness: combine peer availability, sync progress, estimated lag, and tip +// freshness to avoid the false positives called out in issue #4649 and the +// implementation plan. +async fn ready(ctx: &HealthCtx) -> Response> +where + SyncStatus: ChainSyncStatus + Clone + Send + Sync + 'static, +{ + if !ctx.config.enforce_on_test_networks && ctx.network.is_a_test_network() { + return simple_response(StatusCode::OK, "ok"); + } + + if *ctx.num_live_peer_receiver.borrow() < ctx.config.min_connected_peers { + return simple_response(StatusCode::SERVICE_UNAVAILABLE, "insufficient peers"); + } + + // Keep the historical sync-gate but feed it with the richer readiness + // checks so we respect the plan's "ensure recent block commits" item. + if !ctx.sync_status.is_close_to_tip() { + return simple_response(StatusCode::SERVICE_UNAVAILABLE, "syncing"); + } + + let ChainTipMetrics { + last_chain_tip_grow_time, + remaining_sync_blocks, + } = ctx.chain_tip_metrics_receiver.borrow().clone(); + + let Some(remaining_sync_blocks) = remaining_sync_blocks else { + tracing::warn!("syncer is getting block hashes from peers, but state is empty"); + return simple_response(StatusCode::SERVICE_UNAVAILABLE, "no tip"); + }; + + let tip_age = last_chain_tip_grow_time.elapsed(); + if tip_age > ctx.config.ready_max_tip_age { + return simple_response( + StatusCode::SERVICE_UNAVAILABLE, + &format!("tip_age={}s", tip_age.as_secs()), + ); + } + + if remaining_sync_blocks <= ctx.config.ready_max_blocks_behind { + simple_response(StatusCode::OK, "ok") + } else { + simple_response( + StatusCode::SERVICE_UNAVAILABLE, + &format!("lag={remaining_sync_blocks} blocks"), + ) + } +} + +// Measure peers on a blocking thread, mirroring the previous synchronous +// implementation but without holding the mutex on the request path. +async fn num_live_peers(address_book: &A) -> Option +where + A: AddressBookPeers + Clone + Send + Sync + 'static, +{ + let address_book = address_book.clone(); + tokio::task::spawn_blocking(move || address_book.recently_live_peers(Utc::now()).len()) + .await + .inspect_err(|err| warn!(?err, "failed to refresh peer metrics")) + .ok() +} + +// Periodically update the cached peer metrics for all handlers that hold a +// receiver. If receivers disappear we exit quietly so shutdown can proceed. +async fn peer_metrics_refresh_task(address_book: A, num_live_peer_sender: watch::Sender) +where + A: AddressBookPeers + Clone + Send + Sync + 'static, +{ + let mut interval = time::interval(PEER_METRICS_REFRESH_INTERVAL); + interval.set_missed_tick_behavior(MissedTickBehavior::Delay); + + loop { + // Updates are best-effort: if the snapshot fails or all receivers are + // dropped we exit quietly, letting the caller terminate the health task. + if let Some(metrics) = num_live_peers(&address_book).await { + if let Err(err) = num_live_peer_sender.send(metrics) { + tracing::warn!(?err, "failed to send to peer metrics channel"); + break; + } + } + + interval.tick().await; + } +} + +async fn run_health_server( + listener: tokio::net::TcpListener, + shared: Arc>, +) where + SyncStatus: ChainSyncStatus + Clone + Send + Sync + 'static, +{ + let mut num_recent_requests: usize = 0; + let mut last_request_count_reset_time = Instant::now(); + + // Dedicated accept loop to keep request handling small and predictable; we + // still spawn per-connection tasks but share the context clone. + + loop { + match listener.accept().await { + Ok((stream, _)) => { + if num_recent_requests < MAX_RECENT_REQUESTS { + num_recent_requests += 1; + } else if last_request_count_reset_time.elapsed() > RECENT_REQUEST_INTERVAL { + num_recent_requests = 0; + last_request_count_reset_time = Instant::now(); + } else { + // Drop the request if there have been too many recent requests + continue; + } + + let io = TokioIo::new(stream); + let svc_ctx = shared.clone(); + let service = + hyper::service::service_fn(move |req| handle_request(req, svc_ctx.clone())); + + tokio::spawn(async move { + if let Err(err) = http1::Builder::new().serve_connection(io, service).await { + debug!(?err, "health server connection closed with error"); + } + }); + } + Err(err) => { + warn!(?err, "health server accept failed"); + } + } + } +} + +fn simple_response(status: StatusCode, body: &str) -> Response> { + let bytes = Bytes::from(body.to_string()); + let len = bytes.len(); + ResponseBuilder::new() + .status(status) + .header(CONTENT_TYPE, "text/plain; charset=utf-8") + .header(CONTENT_LENGTH, len.to_string()) + .body(Full::new(bytes)) + .expect("valid response") +} diff --git a/zebrad/src/components/health/config.rs b/zebrad/src/components/health/config.rs new file mode 100644 index 00000000000..2d5a1002e0a --- /dev/null +++ b/zebrad/src/components/health/config.rs @@ -0,0 +1,42 @@ +use std::{net::SocketAddr, time::Duration}; + +use serde::{Deserialize, Serialize}; + +/// Health server configuration. +#[derive(Clone, Debug, Eq, PartialEq, Deserialize, Serialize)] +#[serde(deny_unknown_fields, default)] +pub struct Config { + /// Address to bind the health server to. + /// + /// The server is disabled when this is `None`. + pub listen_addr: Option, + /// Minimum number of recently live peers to consider the node healthy. + /// + /// Used by `/healthy`. + pub min_connected_peers: usize, + /// Maximum allowed estimated blocks behind the network tip for readiness. + /// + /// Used by `/ready`. Negative estimates are treated as 0. + pub ready_max_blocks_behind: i64, + /// Enforce readiness checks on test networks. + /// + /// If `false`, `/ready` always returns 200 on regtest and testnets. + pub enforce_on_test_networks: bool, + /// Maximum age of the last committed block before readiness fails. + #[serde(with = "humantime_serde")] + pub ready_max_tip_age: Duration, +} + +impl Default for Config { + fn default() -> Self { + Self { + listen_addr: None, + min_connected_peers: 1, + ready_max_blocks_behind: 2, + enforce_on_test_networks: false, + ready_max_tip_age: DEFAULT_READY_MAX_TIP_AGE, + } + } +} + +const DEFAULT_READY_MAX_TIP_AGE: Duration = Duration::from_secs(5 * 60); diff --git a/zebrad/src/components/health/tests.rs b/zebrad/src/components/health/tests.rs new file mode 100644 index 00000000000..2d0b332bb79 --- /dev/null +++ b/zebrad/src/components/health/tests.rs @@ -0,0 +1,223 @@ +use super::*; + +use std::{ + net::{IpAddr, Ipv4Addr, SocketAddr}, + time::Duration, +}; + +use tokio::{ + io::{AsyncReadExt, AsyncWriteExt}, + time::timeout, +}; +use zebra_chain::{chain_sync_status::mock::MockSyncStatus, parameters::Network}; +use zebra_network::{address_book_peers::MockAddressBookPeers, PeerSocketAddr}; + +// Build a config tailored for tests: enable the listener and disable the +// built-in rate limiter so assertions don't race the cooldown unless a test +// overrides it. +fn config_for(addr: SocketAddr) -> Config { + Config { + listen_addr: Some(addr), + enforce_on_test_networks: true, + ..Default::default() + } +} + +// Populate the mock address book with `count` live peers so we can exercise the +// peer threshold logic without spinning up real network state. +fn peers_with_count(count: usize) -> MockAddressBookPeers { + let mut peers = MockAddressBookPeers::default(); + for _ in 0..count { + let socket = SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), 0); + let peer = PeerSocketAddr::from(socket); + peers.add_peer(peer); + } + peers +} + +// Minimal HTTP client used by the tests; we intentionally avoid abstractions to +// verify the exact wire responses emitted by the health server. +async fn http_get(addr: SocketAddr, path: &str) -> Option<(u16, String)> { + let mut stream = timeout(Duration::from_secs(2), tokio::net::TcpStream::connect(addr)) + .await + .expect("connect timeout") + .expect("connect ok"); + let request = format!( + "GET {} HTTP/1.1\r\nHost: localhost\r\nConnection: close\r\n\r\n", + path + ); + timeout(Duration::from_secs(2), stream.write_all(request.as_bytes())) + .await + .expect("write timeout") + .expect("write ok"); + + let mut buf = Vec::new(); + timeout(Duration::from_secs(2), stream.read_to_end(&mut buf)) + .await + .expect("read timeout") + .ok()?; + + let text = String::from_utf8_lossy(&buf).to_string(); + let status = text + .lines() + .next() + .and_then(|line| line.split_whitespace().nth(1)) + .and_then(|s| s.parse::().ok()) + .unwrap_or(0); + + Some((status, text)) +} + +#[tokio::test] +async fn healthy_and_ready_ok() { + // Happy-path coverage: peers, sync status, lag, and tip age are all within + // thresholds, so both endpoints return 200. + let cfg = config_for(SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), 0)); + let mut sync_status = MockSyncStatus::default(); + sync_status.set_is_close_to_tip(true); + + let (chain_tip_metrics_sender, chain_tip_metrics_receiver) = ChainTipMetrics::channel(); + let _ = chain_tip_metrics_sender.send(ChainTipMetrics::new(Instant::now(), Some(0))); + + let (task, addr_opt) = init( + cfg, + Network::Mainnet, + chain_tip_metrics_receiver, + sync_status, + peers_with_count(1), + ) + .await; + let addr = addr_opt.expect("server bound addr"); + + let (status_h, body_h) = http_get(addr, "/healthy").await.unwrap(); + assert_eq!(status_h, 200, "healthy response: {}", body_h); + + let (status_r, body_r) = http_get(addr, "/ready").await.unwrap(); + assert_eq!(status_r, 200, "ready response: {}", body_r); + + task.abort(); +} + +#[tokio::test] +async fn not_ready_when_syncing_or_lagging() { + let cfg = config_for(SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), 0)); + + // Syncing -> not ready + let sync_status = MockSyncStatus::default(); + let (chain_tip_metrics_sender, chain_tip_metrics_receiver) = ChainTipMetrics::channel(); + let _ = chain_tip_metrics_sender.send(ChainTipMetrics::new(Instant::now(), Some(0))); + let (task_syncing, addr_syncing) = init( + cfg.clone(), + Network::Mainnet, + chain_tip_metrics_receiver.clone(), + sync_status, + peers_with_count(1), + ) + .await; + let addr_syncing = addr_syncing.expect("addr"); + let (status_syncing, body_syncing) = http_get(addr_syncing, "/ready").await.unwrap(); + assert_eq!(status_syncing, 503, "body: {}", body_syncing); + assert!(body_syncing.contains("syncing")); + task_syncing.abort(); + + // Lagging beyond threshold -> not ready + let mut sync_status = MockSyncStatus::default(); + sync_status.set_is_close_to_tip(true); + let _ = chain_tip_metrics_sender.send(ChainTipMetrics::new(Instant::now(), Some(5))); + let (task_lagging, addr_lagging) = init( + cfg, + Network::Mainnet, + chain_tip_metrics_receiver, + sync_status, + peers_with_count(1), + ) + .await; + let addr_lagging = addr_lagging.expect("addr"); + let (status_lagging, body_lagging) = http_get(addr_lagging, "/ready").await.unwrap(); + assert_eq!(status_lagging, 503, "body: {}", body_lagging); + assert!(body_lagging.contains("lag=5")); + task_lagging.abort(); +} + +#[tokio::test] +async fn not_ready_when_tip_is_too_old() { + // A stale block time must cause readiness to fail even if everything else + // looks healthy, preventing long-lived false positives. + let mut cfg = config_for(SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), 0)); + cfg.ready_max_tip_age = Duration::from_secs(1); + + let mut sync_status = MockSyncStatus::default(); + sync_status.set_is_close_to_tip(true); + + let (chain_tip_metrics_sender, chain_tip_metrics_receiver) = ChainTipMetrics::channel(); + let _ = chain_tip_metrics_sender.send(ChainTipMetrics::new( + Instant::now() + .checked_sub(Duration::from_secs(10)) + .expect("should not overflow"), + Some(0), + )); + + let (task, addr_opt) = init( + cfg, + Network::Mainnet, + chain_tip_metrics_receiver, + sync_status, + peers_with_count(1), + ) + .await; + let addr = addr_opt.expect("addr"); + + let (status, body) = http_get(addr, "/ready").await.unwrap(); + assert_eq!(status, 503, "body: {}", body); + assert!(body.contains("tip_age")); + + task.abort(); +} + +#[tokio::test] +#[cfg(not(target_os = "windows"))] +async fn rate_limiting_drops_bursts() { + // With a sleep shorter than the configured interval we should only be able + // to observe one successful request before the limiter responds with 429. + let cfg = config_for(SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), 0)); + + let mut sync_status = MockSyncStatus::default(); + sync_status.set_is_close_to_tip(true); + + let (chain_tip_metrics_sender, chain_tip_metrics_receiver) = ChainTipMetrics::channel(); + let _ = chain_tip_metrics_sender.send(ChainTipMetrics::new(Instant::now(), Some(0))); + + let (task, addr_opt) = init( + cfg, + Network::Mainnet, + chain_tip_metrics_receiver, + sync_status, + peers_with_count(1), + ) + .await; + let addr = addr_opt.expect("addr"); + + let (first_status, first_body) = http_get(addr, "/healthy").await.unwrap(); + assert_eq!(first_status, 200, "first response: {}", first_body); + + let mut was_request_dropped = false; + for i in 0..(MAX_RECENT_REQUESTS + 10) { + if http_get(addr, "/healthy").await.is_none() { + was_request_dropped = true; + println!("got expected status after some reqs: {i}"); + break; + } + } + + assert!( + was_request_dropped, + "requests should be dropped past threshold" + ); + + tokio::time::sleep(RECENT_REQUEST_INTERVAL + Duration::from_millis(100)).await; + + let (third_status, third_body) = http_get(addr, "/healthy").await.unwrap(); + assert_eq!(third_status, 200, "last response: {}", third_body); + + task.abort(); +} diff --git a/zebrad/src/components/sync/progress.rs b/zebrad/src/components/sync/progress.rs index 0131c798d0e..fa81f8025e6 100644 --- a/zebrad/src/components/sync/progress.rs +++ b/zebrad/src/components/sync/progress.rs @@ -9,6 +9,7 @@ use std::{ use chrono::Utc; use num_integer::div_ceil; +use tokio::sync::watch; use zebra_chain::{ block::{Height, HeightDiff}, chain_sync_status::ChainSyncStatus, @@ -18,7 +19,7 @@ use zebra_chain::{ }; use zebra_state::MAX_BLOCK_REORG_HEIGHT; -use crate::components::sync::SyncStatus; +use crate::components::{health::ChainTipMetrics, sync::SyncStatus}; /// The amount of time between progress logs. const LOG_INTERVAL: Duration = Duration::from_secs(60); @@ -69,6 +70,7 @@ pub async fn show_block_chain_progress( network: Network, latest_chain_tip: impl ChainTip, sync_status: SyncStatus, + chain_tip_metrics_sender: watch::Sender, ) -> ! { // The minimum number of extra blocks after the highest checkpoint, based on: // - the non-finalized state limit, and @@ -115,6 +117,7 @@ pub async fn show_block_chain_progress( // // Initialized to the start time to simplify the code. let mut last_state_change_time = Utc::now(); + let mut last_state_change_instant = Instant::now(); // The state tip height, when we last downloaded and verified at least one block. // @@ -155,6 +158,28 @@ pub async fn show_block_chain_progress( .set_len(u64::from(estimated_height.0)); } + let mut remaining_sync_blocks = estimated_height - current_height; + + if remaining_sync_blocks < 0 { + remaining_sync_blocks = 0; + } + + // Work out how long it has been since the state height has increased. + // + // Non-finalized forks can decrease the height, we only want to track increases. + if current_height > last_state_change_height { + last_state_change_height = current_height; + last_state_change_time = now; + last_state_change_instant = instant_now; + } + + if let Err(err) = chain_tip_metrics_sender.send(ChainTipMetrics::new( + last_state_change_instant, + Some(remaining_sync_blocks), + )) { + tracing::warn!(?err, "chain tip metrics channel closed"); + }; + // Skip logging and status updates if it isn't time for them yet. let elapsed_since_log = instant_now.saturating_duration_since(last_log_time); if elapsed_since_log < LOG_INTERVAL { @@ -174,19 +199,6 @@ pub async fn show_block_chain_progress( frac = SYNC_PERCENT_FRAC_DIGITS, ); - let mut remaining_sync_blocks = estimated_height - current_height; - if remaining_sync_blocks < 0 { - remaining_sync_blocks = 0; - } - - // Work out how long it has been since the state height has increased. - // - // Non-finalized forks can decrease the height, we only want to track increases. - if current_height > last_state_change_height { - last_state_change_height = current_height; - last_state_change_time = now; - } - let time_since_last_state_block_chrono = now.signed_duration_since(last_state_change_time); let time_since_last_state_block = humantime_seconds( diff --git a/zebrad/src/config.rs b/zebrad/src/config.rs index 597649c95ed..fd54029a7e5 100644 --- a/zebrad/src/config.rs +++ b/zebrad/src/config.rs @@ -77,6 +77,12 @@ pub struct ZebradConfig { /// Mining configuration pub mining: zebra_rpc::config::mining::Config, + + /// Health check HTTP server configuration. + /// + /// See the Zebra Book for details and examples: + /// + pub health: crate::components::health::Config, } impl ZebradConfig { diff --git a/zebrad/tests/common/configs/v2.6.0.toml b/zebrad/tests/common/configs/v2.6.0.toml index a2eb7a7e25b..82111845dc3 100644 --- a/zebrad/tests/common/configs/v2.6.0.toml +++ b/zebrad/tests/common/configs/v2.6.0.toml @@ -43,6 +43,12 @@ [consensus] checkpoint_sync = true +[health] +enforce_on_test_networks = false +min_connected_peers = 1 +ready_max_blocks_behind = 2 +ready_max_tip_age = "5m" + [mempool] eviction_memory_time = "1h" tx_cost_limit = 80000000 From 2c4d3e6adf8d2dc1bca67e5dcf51da0efd1a67ea Mon Sep 17 00:00:00 2001 From: Alfredo Garcia Date: Thu, 2 Oct 2025 17:46:39 -0300 Subject: [PATCH 010/431] test(fix): Uncomment & update `disconnects_from_misbehaving_peers` (#9735) * uncomment `disconnects_from_misbehaving_peers` test * fix import * add improvments to test * update `clear_checkpoints` fn * fix import --------- Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> --- zebra-chain/src/parameters/network/testnet.rs | 5 + zebrad/tests/acceptance.rs | 348 ++++++++++-------- 2 files changed, 192 insertions(+), 161 deletions(-) diff --git a/zebra-chain/src/parameters/network/testnet.rs b/zebra-chain/src/parameters/network/testnet.rs index ca6b9ccc967..d10c5ba1b7f 100644 --- a/zebra-chain/src/parameters/network/testnet.rs +++ b/zebra-chain/src/parameters/network/testnet.rs @@ -746,6 +746,11 @@ impl ParametersBuilder { self } + /// Clears checkpoints from the [`Parameters`] being built, keeping the genesis checkpoint. + pub fn clear_checkpoints(self) -> Self { + self.with_checkpoints(ConfiguredCheckpoints::Default(false)) + } + /// Converts the builder to a [`Parameters`] struct fn finish(self) -> Parameters { let Self { diff --git a/zebrad/tests/acceptance.rs b/zebrad/tests/acceptance.rs index 4d096f8fddf..a0478b356ed 100644 --- a/zebrad/tests/acceptance.rs +++ b/zebrad/tests/acceptance.rs @@ -4044,175 +4044,201 @@ async fn restores_non_finalized_state_and_commits_new_blocks() -> Result<()> { child.kill(true) } -// /// Check that Zebra will disconnect from misbehaving peers. -// #[tokio::test] -// #[cfg(not(target_os = "windows"))] -// async fn disconnects_from_misbehaving_peers() -> Result<()> { -// use std::sync::{atomic::AtomicBool, Arc}; - -// use common::regtest::MiningRpcMethods; -// use zebra_chain::parameters::testnet::{self, ConfiguredActivationHeights}; -// use zebra_rpc::methods::get_block_template_rpcs::types::peer_info::PeerInfo; - -// let _init_guard = zebra_test::init(); -// let network = testnet::Parameters::build() -// .with_activation_heights(ConfiguredActivationHeights { -// canopy: Some(1), -// nu5: Some(2), -// nu6: Some(3), -// ..Default::default() -// }) -// .with_slow_start_interval(Height::MIN) -// .with_disable_pow(true) -// .to_network(); - -// let test_type = LaunchWithEmptyState { -// launches_lightwalletd: false, -// }; -// let test_name = "disconnects_from_misbehaving_peers_test"; - -// if !common::launch::can_spawn_zebrad_for_test_type(test_name, test_type, false) { -// tracing::warn!("skipping disconnects_from_misbehaving_peers test"); -// return Ok(()); -// } - -// // Get the zebrad config -// let mut config = test_type -// .zebrad_config(test_name, false, None, &network) -// .expect("already checked config")?; - -// config.network.cache_dir = false.into(); -// config.network.listen_addr = format!("127.0.0.1:{}", random_known_port()).parse()?; - -// let rpc_listen_addr = config.rpc.listen_addr.unwrap(); -// let rpc_client_1 = RpcRequestClient::new(rpc_listen_addr); - -// tracing::info!( -// ?rpc_listen_addr, -// network_listen_addr = ?config.network.listen_addr, -// "starting a zebrad child on incompatible custom Testnet" -// ); - -// let is_finished = Arc::new(AtomicBool::new(false)); - -// { -// let is_finished = Arc::clone(&is_finished); -// let config = config.clone(); -// let (zebrad_failure_messages, zebrad_ignore_messages) = test_type.zebrad_failure_messages(); -// tokio::task::spawn_blocking(move || -> Result<()> { -// let mut zebrad_child = testdir()? -// .with_exact_config(&config)? -// .spawn_child(args!["start"])? -// .bypass_test_capture(true) -// .with_timeout(test_type.zebrad_timeout()) -// .with_failure_regex_iter(zebrad_failure_messages, zebrad_ignore_messages); - -// while !is_finished.load(std::sync::atomic::Ordering::SeqCst) { -// zebrad_child.wait_for_stdout_line(Some("zebraA1".to_string())); -// } - -// Ok(()) -// }); -// } - -// config.network.initial_testnet_peers = [config.network.listen_addr.to_string()].into(); -// config.network.network = Network::new_default_testnet(); -// config.network.listen_addr = "127.0.0.1:0".parse()?; -// config.rpc.listen_addr = Some(format!("127.0.0.1:{}", random_known_port()).parse()?); - -// let rpc_listen_addr = config.rpc.listen_addr.unwrap(); -// let rpc_client_2 = RpcRequestClient::new(rpc_listen_addr); - -// tracing::info!( -// ?rpc_listen_addr, -// network_listen_addr = ?config.network.listen_addr, -// "starting a zebrad child on the default Testnet" -// ); - -// { -// let is_finished = Arc::clone(&is_finished); -// tokio::task::spawn_blocking(move || -> Result<()> { -// let (zebrad_failure_messages, zebrad_ignore_messages) = -// test_type.zebrad_failure_messages(); -// let mut zebrad_child = testdir()? -// .with_exact_config(&config)? -// .spawn_child(args!["start"])? -// .bypass_test_capture(true) -// .with_timeout(test_type.zebrad_timeout()) -// .with_failure_regex_iter(zebrad_failure_messages, zebrad_ignore_messages); - -// while !is_finished.load(std::sync::atomic::Ordering::SeqCst) { -// zebrad_child.wait_for_stdout_line(Some("zebraB2".to_string())); -// } - -// Ok(()) -// }); -// } - -// tracing::info!("waiting for zebrad nodes to connect"); - -// // Wait a few seconds for Zebra to start up and make outbound peer connections -// tokio::time::sleep(LAUNCH_DELAY).await; - -// tracing::info!("checking for peers"); - -// // Call `getpeerinfo` to check that the zebrad instances have connected -// let peer_info: Vec = rpc_client_2 -// .json_result_from_call("getpeerinfo", "[]") -// .await -// .map_err(|err| eyre!(err))?; - -// assert!(!peer_info.is_empty(), "should have outbound peer"); - -// tracing::info!( -// ?peer_info, -// "found peer connection, committing genesis block" -// ); - -// let genesis_block = network.block_parsed_iter().next().unwrap(); -// rpc_client_1.submit_block(genesis_block.clone()).await?; -// rpc_client_2.submit_block(genesis_block).await?; - -// // Call the `generate` method to mine blocks in the zebrad instance where PoW is disabled -// tracing::info!("committed genesis block, mining blocks with invalid PoW"); -// tokio::time::sleep(Duration::from_secs(2)).await; - -// rpc_client_1.call("generate", "[500]").await?; +/// Check that Zebra will disconnect from misbehaving peers. +/// +/// In order to simulate a misbehaviour peer we start two zebrad instances: +/// - The first one is started with a custom Testnet where PoW is disabled. +/// - The second one is started with the default Testnet where PoW is enabled. +/// The second zebrad instance will connect to the first one, and when the first one mines +/// blocks with invalid PoW the second one should disconnect from it. +#[tokio::test] +#[cfg(not(target_os = "windows"))] +async fn disconnects_from_misbehaving_peers() -> Result<()> { + use std::sync::{atomic::AtomicBool, Arc}; + + use common::regtest::MiningRpcMethods; + use zebra_chain::parameters::testnet::{self, ConfiguredActivationHeights}; + use zebra_rpc::client::PeerInfo; + + let _init_guard = zebra_test::init(); + let network1 = testnet::Parameters::build() + .with_activation_heights(ConfiguredActivationHeights { + canopy: Some(1), + nu5: Some(2), + nu6: Some(3), + ..Default::default() + }) + .with_slow_start_interval(Height::MIN) + .with_disable_pow(true) + .clear_checkpoints() + .with_network_name("PoWDisabledTestnet") + .to_network(); + + let test_type = LaunchWithEmptyState { + launches_lightwalletd: false, + }; + let test_name = "disconnects_from_misbehaving_peers_test"; + + if !common::launch::can_spawn_zebrad_for_test_type(test_name, test_type, false) { + tracing::warn!("skipping disconnects_from_misbehaving_peers test"); + return Ok(()); + } + + // Get the zebrad config + let mut config = test_type + .zebrad_config(test_name, false, None, &network1) + .expect("already checked config")?; + + config.network.cache_dir = false.into(); + config.network.listen_addr = format!("127.0.0.1:{}", random_known_port()).parse()?; + config.state.ephemeral = true; + config.network.initial_testnet_peers = [].into(); + config.network.crawl_new_peer_interval = Duration::from_secs(5); + + let rpc_listen_addr = config.rpc.listen_addr.unwrap(); + let rpc_client_1 = RpcRequestClient::new(rpc_listen_addr); + + tracing::info!( + ?rpc_listen_addr, + network_listen_addr = ?config.network.listen_addr, + "starting a zebrad child on incompatible custom Testnet" + ); + + let is_finished = Arc::new(AtomicBool::new(false)); + + { + let is_finished = Arc::clone(&is_finished); + let config = config.clone(); + let (zebrad_failure_messages, zebrad_ignore_messages) = test_type.zebrad_failure_messages(); + tokio::task::spawn_blocking(move || -> Result<()> { + let mut zebrad_child = testdir()? + .with_exact_config(&config)? + .spawn_child(args!["start"])? + .bypass_test_capture(true) + .with_timeout(test_type.zebrad_timeout()) + .with_failure_regex_iter(zebrad_failure_messages, zebrad_ignore_messages); + + while !is_finished.load(std::sync::atomic::Ordering::SeqCst) { + zebrad_child.wait_for_stdout_line(Some("zebraA1".to_string())); + } + + Ok(()) + }); + } + + let network2 = testnet::Parameters::build() + .with_activation_heights(ConfiguredActivationHeights { + canopy: Some(1), + nu5: Some(2), + nu6: Some(3), + ..Default::default() + }) + .with_slow_start_interval(Height::MIN) + .clear_checkpoints() + .with_network_name("PoWEnabledTestnet") + .to_network(); + + config.network.network = network2; + config.network.initial_testnet_peers = [config.network.listen_addr.to_string()].into(); + config.network.listen_addr = "127.0.0.1:0".parse()?; + config.rpc.listen_addr = Some(format!("127.0.0.1:{}", random_known_port()).parse()?); + config.network.crawl_new_peer_interval = Duration::from_secs(5); + config.network.cache_dir = false.into(); + config.state.ephemeral = true; + + let rpc_listen_addr = config.rpc.listen_addr.unwrap(); + let rpc_client_2 = RpcRequestClient::new(rpc_listen_addr); + + tracing::info!( + ?rpc_listen_addr, + network_listen_addr = ?config.network.listen_addr, + "starting a zebrad child on the default Testnet" + ); -// tracing::info!("wait for misbehavior messages to flush into address updater channel"); + { + let is_finished = Arc::clone(&is_finished); + tokio::task::spawn_blocking(move || -> Result<()> { + let (zebrad_failure_messages, zebrad_ignore_messages) = + test_type.zebrad_failure_messages(); + let mut zebrad_child = testdir()? + .with_exact_config(&config)? + .spawn_child(args!["start"])? + .bypass_test_capture(true) + .with_timeout(test_type.zebrad_timeout()) + .with_failure_regex_iter(zebrad_failure_messages, zebrad_ignore_messages); + + while !is_finished.load(std::sync::atomic::Ordering::SeqCst) { + zebrad_child.wait_for_stdout_line(Some("zebraB2".to_string())); + } -// tokio::time::sleep(Duration::from_secs(30)).await; + Ok(()) + }); + } -// tracing::info!("calling getpeerinfo to confirm Zebra has dropped the peer connection"); - -// // Call `getpeerinfo` to check that the zebrad instances have disconnected -// for i in 0..600 { -// let peer_info: Vec = rpc_client_2 -// .json_result_from_call("getpeerinfo", "[]") -// .await -// .map_err(|err| eyre!(err))?; + tracing::info!("waiting for zebrad nodes to connect"); -// if peer_info.is_empty() { -// break; -// } else if i % 10 == 0 { -// tracing::info!(?peer_info, "has not yet disconnected from misbehaving peer"); -// } + // Wait a few seconds for Zebra to start up and make outbound peer connections + tokio::time::sleep(LAUNCH_DELAY).await; + + tracing::info!("checking for peers"); + + // Call `getpeerinfo` to check that the zebrad instances have connected + let peer_info: Vec = rpc_client_2 + .json_result_from_call("getpeerinfo", "[]") + .await + .map_err(|err| eyre!(err))?; + + assert!(!peer_info.is_empty(), "should have outbound peer"); + + tracing::info!( + ?peer_info, + "found peer connection, committing genesis block" + ); -// rpc_client_1.call("generate", "[1]").await?; + let genesis_block = network1.block_parsed_iter().next().unwrap(); + rpc_client_1.submit_block(genesis_block.clone()).await?; + rpc_client_2.submit_block(genesis_block).await?; -// tokio::time::sleep(Duration::from_secs(1)).await; -// } + // Call the `generate` method to mine blocks in the zebrad instance where PoW is disabled + tracing::info!("committed genesis block, mining blocks with invalid PoW"); + tokio::time::sleep(Duration::from_secs(2)).await; -// let peer_info: Vec = rpc_client_2 -// .json_result_from_call("getpeerinfo", "[]") -// .await -// .map_err(|err| eyre!(err))?; + rpc_client_1.call("generate", "[500]").await?; -// tracing::info!(?peer_info, "called getpeerinfo"); + tracing::info!("wait for misbehavior messages to flush into address updater channel"); -// assert!(peer_info.is_empty(), "should have no peers"); + tokio::time::sleep(Duration::from_secs(30)).await; -// is_finished.store(true, std::sync::atomic::Ordering::SeqCst); + tracing::info!("calling getpeerinfo to confirm Zebra has dropped the peer connection"); -// Ok(()) -// } + // Call `getpeerinfo` to check that the zebrad instances have disconnected + for i in 0..600 { + let peer_info: Vec = rpc_client_2 + .json_result_from_call("getpeerinfo", "[]") + .await + .map_err(|err| eyre!(err))?; + + if peer_info.is_empty() { + break; + } else if i % 10 == 0 { + tracing::info!(?peer_info, "has not yet disconnected from misbehaving peer"); + } + + rpc_client_1.call("generate", "[1]").await?; + + tokio::time::sleep(Duration::from_secs(1)).await; + } + + let peer_info: Vec = rpc_client_2 + .json_result_from_call("getpeerinfo", "[]") + .await + .map_err(|err| eyre!(err))?; + + tracing::info!(?peer_info, "called getpeerinfo"); + + assert!(peer_info.is_empty(), "should have no peers"); + + is_finished.store(true, std::sync::atomic::Ordering::SeqCst); + + Ok(()) +} From f5395b5d763451d38d413d53277731f935c83d2b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 3 Oct 2025 09:25:00 +0000 Subject: [PATCH 011/431] build(deps): bump the devops group across 1 directory with 5 updates (#9946) Bumps the devops group with 5 updates in the / directory: | Package | From | To | | --- | --- | --- | | [actions-rust-lang/setup-rust-toolchain](https://github.com/actions-rust-lang/setup-rust-toolchain) | `1.14.0` | `1.15.1` | | [taiki-e/install-action](https://github.com/taiki-e/install-action) | `2.58.23` | `2.62.15` | | [codecov/codecov-action](https://github.com/codecov/codecov-action) | `5.5.0` | `5.5.1` | | [peter-evans/dockerhub-description](https://github.com/peter-evans/dockerhub-description) | `4.0.2` | `5.0.0` | | [docker/login-action](https://github.com/docker/login-action) | `3.5.0` | `3.6.0` | Updates `actions-rust-lang/setup-rust-toolchain` from 1.14.0 to 1.15.1 - [Release notes](https://github.com/actions-rust-lang/setup-rust-toolchain/releases) - [Changelog](https://github.com/actions-rust-lang/setup-rust-toolchain/blob/main/CHANGELOG.md) - [Commits](https://github.com/actions-rust-lang/setup-rust-toolchain/compare/ab6845274e2ff01cd4462007e1a9d9df1ab49f42...02be93da58aa71fb456aa9c43b301149248829d8) Updates `taiki-e/install-action` from 2.58.23 to 2.62.15 - [Release notes](https://github.com/taiki-e/install-action/releases) - [Changelog](https://github.com/taiki-e/install-action/blob/main/CHANGELOG.md) - [Commits](https://github.com/taiki-e/install-action/compare/3ee5d63d29478156148c0b53e9f3447829b47bc2...d6d752794628f1e1fffa3c4d3c8874e06f043d50) Updates `codecov/codecov-action` from 5.5.0 to 5.5.1 - [Release notes](https://github.com/codecov/codecov-action/releases) - [Changelog](https://github.com/codecov/codecov-action/blob/main/CHANGELOG.md) - [Commits](https://github.com/codecov/codecov-action/compare/fdcc8476540edceab3de004e990f80d881c6cc00...5a1091511ad55cbe89839c7260b706298ca349f7) Updates `peter-evans/dockerhub-description` from 4.0.2 to 5.0.0 - [Release notes](https://github.com/peter-evans/dockerhub-description/releases) - [Commits](https://github.com/peter-evans/dockerhub-description/compare/432a30c9e07499fd01da9f8a49f0faf9e0ca5b77...1b9a80c056b620d92cedb9d9b5a223409c68ddfa) Updates `docker/login-action` from 3.5.0 to 3.6.0 - [Release notes](https://github.com/docker/login-action/releases) - [Commits](https://github.com/docker/login-action/compare/184bdaa0721073962dff0199f1fb9940f07167d1...5e57cd118135c172c3672efd75eb46360885c0ef) --- updated-dependencies: - dependency-name: actions-rust-lang/setup-rust-toolchain dependency-version: 1.15.1 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: devops - dependency-name: taiki-e/install-action dependency-version: 2.62.15 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: devops - dependency-name: codecov/codecov-action dependency-version: 5.5.1 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: devops - dependency-name: peter-evans/dockerhub-description dependency-version: 5.0.0 dependency-type: direct:production update-type: version-update:semver-major dependency-group: devops - dependency-name: docker/login-action dependency-version: 3.6.0 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: devops ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/book.yml | 2 +- .github/workflows/coverage.yml | 6 ++--- .github/workflows/lint.yml | 26 +++++++++---------- .github/workflows/release-binaries.yml | 2 +- .github/workflows/test-crates.yml | 4 +-- .github/workflows/tests-unit.yml | 8 +++--- .github/workflows/zfnd-build-docker-image.yml | 4 +-- .../workflows/zfnd-delete-gcp-resources.yml | 2 +- 8 files changed, 27 insertions(+), 27 deletions(-) diff --git a/.github/workflows/book.yml b/.github/workflows/book.yml index 2d92972fe99..955e02a6db7 100644 --- a/.github/workflows/book.yml +++ b/.github/workflows/book.yml @@ -48,7 +48,7 @@ jobs: uses: arduino/setup-protoc@c65c819552d16ad3c9b72d9dfd5ba5237b9c906b #v3.0.0 with: repo-token: ${{ secrets.GITHUB_TOKEN }} - - uses: actions-rust-lang/setup-rust-toolchain@ab6845274e2ff01cd4462007e1a9d9df1ab49f42 #v1.14.0 + - uses: actions-rust-lang/setup-rust-toolchain@02be93da58aa71fb456aa9c43b301149248829d8 #v1.15.1 with: toolchain: nightly cache-on-failure: true diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml index 243ed9b8653..d0f53604327 100644 --- a/.github/workflows/coverage.yml +++ b/.github/workflows/coverage.yml @@ -45,12 +45,12 @@ jobs: uses: arduino/setup-protoc@c65c819552d16ad3c9b72d9dfd5ba5237b9c906b #v3.0.0 with: repo-token: ${{ secrets.GITHUB_TOKEN }} - - uses: actions-rust-lang/setup-rust-toolchain@ab6845274e2ff01cd4462007e1a9d9df1ab49f42 #v1.14.0 + - uses: actions-rust-lang/setup-rust-toolchain@02be93da58aa71fb456aa9c43b301149248829d8 #v1.15.1 with: toolchain: stable components: llvm-tools-preview cache-on-failure: true - - uses: taiki-e/install-action@3ee5d63d29478156148c0b53e9f3447829b47bc2 #v2.58.23 + - uses: taiki-e/install-action@d6d752794628f1e1fffa3c4d3c8874e06f043d50 #v2.62.15 with: tool: cargo-llvm-cov,nextest - name: Run coverage tests @@ -67,7 +67,7 @@ jobs: PROPTEST_MAX_SHRINK_ITERS: 0 - name: Upload coverage to Codecov - uses: codecov/codecov-action@fdcc8476540edceab3de004e990f80d881c6cc00 #v5.5.0 + uses: codecov/codecov-action@5a1091511ad55cbe89839c7260b706298ca349f7 #v5.5.1 with: files: lcov.info diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 0e655134f44..763e85a1f33 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -60,7 +60,7 @@ jobs: - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 #v5.0.0 with: persist-credentials: false - - uses: actions-rust-lang/setup-rust-toolchain@ab6845274e2ff01cd4462007e1a9d9df1ab49f42 #v1.14.0 + - uses: actions-rust-lang/setup-rust-toolchain@02be93da58aa71fb456aa9c43b301149248829d8 #v1.15.1 with: components: clippy toolchain: ${{ matrix.rust-version }} @@ -80,13 +80,13 @@ jobs: - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 #v5.0.0 with: persist-credentials: false - - uses: actions-rust-lang/setup-rust-toolchain@ab6845274e2ff01cd4462007e1a9d9df1ab49f42 #v1.14.0 + - uses: actions-rust-lang/setup-rust-toolchain@02be93da58aa71fb456aa9c43b301149248829d8 #v1.15.1 with: cache-on-failure: true - uses: arduino/setup-protoc@c65c819552d16ad3c9b72d9dfd5ba5237b9c906b #v3.0.0 with: repo-token: ${{ secrets.GITHUB_TOKEN }} - - uses: taiki-e/install-action@3ee5d63d29478156148c0b53e9f3447829b47bc2 #v2.58.23 + - uses: taiki-e/install-action@d6d752794628f1e1fffa3c4d3c8874e06f043d50 #v2.62.15 with: tool: cargo-hack - run: cargo hack check --workspace @@ -109,7 +109,7 @@ jobs: - uses: arduino/setup-protoc@c65c819552d16ad3c9b72d9dfd5ba5237b9c906b #v3.0.0 with: repo-token: ${{ secrets.GITHUB_TOKEN }} - - uses: actions-rust-lang/setup-rust-toolchain@ab6845274e2ff01cd4462007e1a9d9df1ab49f42 #v1.14.0 + - uses: actions-rust-lang/setup-rust-toolchain@02be93da58aa71fb456aa9c43b301149248829d8 #v1.15.1 with: toolchain: 1.89 # MSRV cache-on-failure: true @@ -129,7 +129,7 @@ jobs: uses: arduino/setup-protoc@c65c819552d16ad3c9b72d9dfd5ba5237b9c906b #v3.0.0 with: repo-token: ${{ secrets.GITHUB_TOKEN }} - - uses: actions-rust-lang/setup-rust-toolchain@ab6845274e2ff01cd4462007e1a9d9df1ab49f42 #v1.14.0 + - uses: actions-rust-lang/setup-rust-toolchain@02be93da58aa71fb456aa9c43b301149248829d8 #v1.15.1 with: toolchain: nightly cache-on-failure: true @@ -150,7 +150,7 @@ jobs: - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 #v5.0.0 with: persist-credentials: false - - uses: actions-rust-lang/setup-rust-toolchain@ab6845274e2ff01cd4462007e1a9d9df1ab49f42 #v1.14.0 + - uses: actions-rust-lang/setup-rust-toolchain@02be93da58aa71fb456aa9c43b301149248829d8 #v1.15.1 with: toolchain: nightly components: rustfmt @@ -172,11 +172,11 @@ jobs: uses: arduino/setup-protoc@c65c819552d16ad3c9b72d9dfd5ba5237b9c906b #v3.0.0 with: repo-token: ${{ secrets.GITHUB_TOKEN }} - - uses: actions-rust-lang/setup-rust-toolchain@ab6845274e2ff01cd4462007e1a9d9df1ab49f42 #v1.14.0 + - uses: actions-rust-lang/setup-rust-toolchain@02be93da58aa71fb456aa9c43b301149248829d8 #v1.15.1 with: toolchain: nightly cache-on-failure: true - - uses: taiki-e/install-action@3ee5d63d29478156148c0b53e9f3447829b47bc2 #v2.58.23 + - uses: taiki-e/install-action@d6d752794628f1e1fffa3c4d3c8874e06f043d50 #v2.62.15 with: tool: cargo-udeps - run: cargo udeps --workspace --all-targets --all-features --locked @@ -189,7 +189,7 @@ jobs: with: persist-credentials: false - - uses: actions-rust-lang/setup-rust-toolchain@ab6845274e2ff01cd4462007e1a9d9df1ab49f42 #v1.14.0 + - uses: actions-rust-lang/setup-rust-toolchain@02be93da58aa71fb456aa9c43b301149248829d8 #v1.15.1 with: toolchain: stable cache-on-failure: true @@ -212,12 +212,12 @@ jobs: uses: arduino/setup-protoc@c65c819552d16ad3c9b72d9dfd5ba5237b9c906b #v3.0.0 with: repo-token: ${{ secrets.GITHUB_TOKEN }} - - uses: actions-rust-lang/setup-rust-toolchain@ab6845274e2ff01cd4462007e1a9d9df1ab49f42 #v1.14.0 + - uses: actions-rust-lang/setup-rust-toolchain@02be93da58aa71fb456aa9c43b301149248829d8 #v1.15.1 with: toolchain: nightly cache-on-failure: true - name: cargo install cargo-hack - uses: taiki-e/install-action@3ee5d63d29478156148c0b53e9f3447829b47bc2 #v2.58.23 + uses: taiki-e/install-action@d6d752794628f1e1fffa3c4d3c8874e06f043d50 #v2.62.15 with: tool: cargo-hack - run: cargo hack check --all @@ -234,7 +234,7 @@ jobs: - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 #v5.0.0 with: persist-credentials: false - - uses: actions-rust-lang/setup-rust-toolchain@ab6845274e2ff01cd4462007e1a9d9df1ab49f42 #v1.14.0 + - uses: actions-rust-lang/setup-rust-toolchain@02be93da58aa71fb456aa9c43b301149248829d8 #v1.15.1 with: toolchain: stable cache-on-failure: true @@ -267,7 +267,7 @@ jobs: - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 #v5.0.0 with: persist-credentials: false - - uses: actions-rust-lang/setup-rust-toolchain@ab6845274e2ff01cd4462007e1a9d9df1ab49f42 #v1.14.0 + - uses: actions-rust-lang/setup-rust-toolchain@02be93da58aa71fb456aa9c43b301149248829d8 #v1.15.1 with: cache-on-failure: true - name: Check ${{ matrix.checks }} with features ${{ matrix.features }} diff --git a/.github/workflows/release-binaries.yml b/.github/workflows/release-binaries.yml index e55065fe559..6084908eee7 100644 --- a/.github/workflows/release-binaries.yml +++ b/.github/workflows/release-binaries.yml @@ -46,7 +46,7 @@ jobs: persist-credentials: false - name: Docker Hub Description - uses: peter-evans/dockerhub-description@432a30c9e07499fd01da9f8a49f0faf9e0ca5b77 #v4.0.2 + uses: peter-evans/dockerhub-description@1b9a80c056b620d92cedb9d9b5a223409c68ddfa #v5.0.0 with: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_PASSWORD }} diff --git a/.github/workflows/test-crates.yml b/.github/workflows/test-crates.yml index c74a80c864d..14b7305143d 100644 --- a/.github/workflows/test-crates.yml +++ b/.github/workflows/test-crates.yml @@ -55,7 +55,7 @@ jobs: with: persist-credentials: false - - uses: actions-rust-lang/setup-rust-toolchain@ab6845274e2ff01cd4462007e1a9d9df1ab49f42 #v1.14.0 + - uses: actions-rust-lang/setup-rust-toolchain@02be93da58aa71fb456aa9c43b301149248829d8 #v1.15.1 with: toolchain: stable components: clippy @@ -106,7 +106,7 @@ jobs: with: persist-credentials: false - - uses: actions-rust-lang/setup-rust-toolchain@ab6845274e2ff01cd4462007e1a9d9df1ab49f42 #v1.14.0 + - uses: actions-rust-lang/setup-rust-toolchain@02be93da58aa71fb456aa9c43b301149248829d8 #v1.15.1 with: toolchain: stable components: clippy diff --git a/.github/workflows/tests-unit.yml b/.github/workflows/tests-unit.yml index 6cc2d2faad1..f4ef3b2fce7 100644 --- a/.github/workflows/tests-unit.yml +++ b/.github/workflows/tests-unit.yml @@ -63,12 +63,12 @@ jobs: - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 #v5.0.0 with: persist-credentials: false - - uses: actions-rust-lang/setup-rust-toolchain@ab6845274e2ff01cd4462007e1a9d9df1ab49f42 #v1.14.0 + - uses: actions-rust-lang/setup-rust-toolchain@02be93da58aa71fb456aa9c43b301149248829d8 #v1.15.1 with: toolchain: ${{ matrix.rust-version }} cache-key: unit-tests-${{ matrix.os }}-${{ matrix.rust-version }}-${{ matrix.features }} cache-on-failure: true - - uses: taiki-e/install-action@3ee5d63d29478156148c0b53e9f3447829b47bc2 #v2.58.23 + - uses: taiki-e/install-action@d6d752794628f1e1fffa3c4d3c8874e06f043d50 #v2.62.15 with: tool: cargo-nextest - uses: arduino/setup-protoc@c65c819552d16ad3c9b72d9dfd5ba5237b9c906b #v3.0.0 @@ -110,11 +110,11 @@ jobs: - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 #v5.0.0 with: persist-credentials: false - - uses: actions-rust-lang/setup-rust-toolchain@ab6845274e2ff01cd4462007e1a9d9df1ab49f42 #v1.14.0 + - uses: actions-rust-lang/setup-rust-toolchain@02be93da58aa71fb456aa9c43b301149248829d8 #v1.15.1 with: toolchain: stable cache-on-failure: true - - uses: taiki-e/install-action@3ee5d63d29478156148c0b53e9f3447829b47bc2 #v2.58.23 + - uses: taiki-e/install-action@d6d752794628f1e1fffa3c4d3c8874e06f043d50 #v2.62.15 with: tool: cargo-nextest - uses: arduino/setup-protoc@c65c819552d16ad3c9b72d9dfd5ba5237b9c906b #v3.0.0 diff --git a/.github/workflows/zfnd-build-docker-image.yml b/.github/workflows/zfnd-build-docker-image.yml index 0e4d0cfaf17..c36072a8bf6 100644 --- a/.github/workflows/zfnd-build-docker-image.yml +++ b/.github/workflows/zfnd-build-docker-image.yml @@ -131,14 +131,14 @@ jobs: access_token_lifetime: 10800s - name: Login to Google Artifact Registry - uses: docker/login-action@184bdaa0721073962dff0199f1fb9940f07167d1 #v3.5.0 + uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef #v3.6.0 with: registry: us-docker.pkg.dev username: oauth2accesstoken password: ${{ steps.auth.outputs.access_token }} - name: Login to DockerHub - uses: docker/login-action@184bdaa0721073962dff0199f1fb9940f07167d1 #v3.5.0 + uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef #v3.6.0 with: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} diff --git a/.github/workflows/zfnd-delete-gcp-resources.yml b/.github/workflows/zfnd-delete-gcp-resources.yml index 2cf826ca133..375175b913b 100644 --- a/.github/workflows/zfnd-delete-gcp-resources.yml +++ b/.github/workflows/zfnd-delete-gcp-resources.yml @@ -133,7 +133,7 @@ jobs: token_format: access_token - name: Login to Google Artifact Registry - uses: docker/login-action@184bdaa0721073962dff0199f1fb9940f07167d1 #v3.5.0 + uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef #v3.6.0 with: registry: us-docker.pkg.dev username: oauth2accesstoken From 64cad2b9eb62a99f28706dda6a52daaf851c2ec7 Mon Sep 17 00:00:00 2001 From: Arya Date: Fri, 3 Oct 2025 13:51:19 -0400 Subject: [PATCH 012/431] fix(state): Avoid heap allocations in `expand_zero_be_bytes()` (#9951) * Avoids unnecessary allocations `expand_zero_be_bytes()` * fixes clippy lint --- .../service/finalized_state/disk_format.rs | 26 ++++++++++--------- .../finalized_state/disk_format/block.rs | 6 ++--- .../disk_format/transparent.rs | 6 ++--- 3 files changed, 18 insertions(+), 20 deletions(-) diff --git a/zebra-state/src/service/finalized_state/disk_format.rs b/zebra-state/src/service/finalized_state/disk_format.rs index dc7732a6c2b..f1a72122bec 100644 --- a/zebra-state/src/service/finalized_state/disk_format.rs +++ b/zebra-state/src/service/finalized_state/disk_format.rs @@ -5,7 +5,7 @@ //! [`crate::constants::state_database_format_version_in_code()`] must be incremented //! each time the database format (column, serialization, etc) changes. -use std::sync::Arc; +use std::{io::Write, sync::Arc}; pub mod block; pub mod chain; @@ -182,25 +182,27 @@ pub fn truncate_zero_be_bytes(mem_bytes: &[u8], disk_len: usize) -> Option<&[u8] Some(truncated) } -/// Expands `disk_bytes` to `mem_len`, by adding zero bytes at the start of the slice. +/// Expands `disk_bytes` to `MEM_SIZE`, by adding zero bytes at the start of the slice. /// Used to zero-fill bytes that were discarded during serialization. /// /// # Panics /// /// - if `disk_bytes` is longer than `mem_len` -pub fn expand_zero_be_bytes(disk_bytes: &[u8], mem_len: usize) -> Vec { - let extra_bytes = mem_len - .checked_sub(disk_bytes.len()) - .expect("unexpected `disk_bytes` length: must not exceed `mem_len`"); - - if extra_bytes == 0 { - return disk_bytes.to_vec(); +#[inline] +pub fn expand_zero_be_bytes(disk_bytes: &[u8]) -> [u8; MEM_SIZE] { + // Return early if disk_bytes is already the expected length + if let Ok(disk_bytes_array) = disk_bytes.try_into() { + return disk_bytes_array; } - let mut expanded = vec![0; extra_bytes]; - expanded.extend(disk_bytes); + let extra_bytes = MEM_SIZE + .checked_sub(disk_bytes.len()) + .expect("unexpected `disk_bytes` length: must not exceed `MEM_SIZE`"); - assert_eq!(expanded.len(), mem_len); + let mut expanded = [0; MEM_SIZE]; + let _ = (&mut expanded[extra_bytes..]) + .write(disk_bytes) + .expect("should fit"); expanded } diff --git a/zebra-state/src/service/finalized_state/disk_format/block.rs b/zebra-state/src/service/finalized_state/disk_format/block.rs index ed57dae03fc..3b1b5b58156 100644 --- a/zebra-state/src/service/finalized_state/disk_format/block.rs +++ b/zebra-state/src/service/finalized_state/disk_format/block.rs @@ -255,11 +255,9 @@ impl IntoDisk for Height { impl FromDisk for Height { fn from_bytes(disk_bytes: impl AsRef<[u8]>) -> Self { - let mem_len = u32::BITS / 8; - let mem_len = mem_len.try_into().unwrap(); + const MEM_LEN: usize = size_of::(); - let mem_bytes = expand_zero_be_bytes(disk_bytes.as_ref(), mem_len); - let mem_bytes = mem_bytes.try_into().unwrap(); + let mem_bytes = expand_zero_be_bytes::(disk_bytes.as_ref()); Height(u32::from_be_bytes(mem_bytes)) } } diff --git a/zebra-state/src/service/finalized_state/disk_format/transparent.rs b/zebra-state/src/service/finalized_state/disk_format/transparent.rs index 30ab87c94b7..3e4fddc34fc 100644 --- a/zebra-state/src/service/finalized_state/disk_format/transparent.rs +++ b/zebra-state/src/service/finalized_state/disk_format/transparent.rs @@ -682,11 +682,9 @@ impl IntoDisk for OutputIndex { impl FromDisk for OutputIndex { fn from_bytes(disk_bytes: impl AsRef<[u8]>) -> Self { - let mem_len = u32::BITS / 8; - let mem_len = mem_len.try_into().unwrap(); + const MEM_LEN: usize = size_of::(); - let mem_bytes = expand_zero_be_bytes(disk_bytes.as_ref(), mem_len); - let mem_bytes = mem_bytes.try_into().unwrap(); + let mem_bytes = expand_zero_be_bytes::(disk_bytes.as_ref()); OutputIndex::from_index(u32::from_be_bytes(mem_bytes)) } } From 8c093c6a0638685356de1b141c6f8ca183172079 Mon Sep 17 00:00:00 2001 From: Arya Date: Fri, 3 Oct 2025 17:17:48 -0400 Subject: [PATCH 013/431] Add(state): Adds a `MappedRequest` helper trait and refactors error types used by `CommitSemanticallyVerifiedBlock` requests (#9923) * - Updates `CommitSemanticallyVerifiedError` to be an enum instead of a newtype wrapping a boxed `ValidateContextError` - Documents the expected error type for CommitSemanticallyVerifiedBlock requests * Adds a `MappedRequest` helper trait for converting response and error types * Updates zebra-state changelog * fixes clippy lints * Update zebra-state/src/request.rs * Apply suggestions from code review Co-authored-by: Alfredo Garcia --------- Co-authored-by: Alfredo Garcia --- Cargo.lock | 1 + zebra-state/CHANGELOG.md | 14 +++ zebra-state/Cargo.toml | 1 + zebra-state/src/error.rs | 96 ++++++++++++------- zebra-state/src/lib.rs | 3 +- zebra-state/src/request.rs | 66 ++++++++++++- zebra-state/src/response.rs | 4 +- zebra-state/src/service.rs | 77 +++++++-------- .../src/service/check/tests/nullifier.rs | 30 ++---- zebra-state/src/service/check/tests/utxo.rs | 21 ++-- zebra-state/src/service/queued_blocks.rs | 12 +-- zebra-state/src/service/write.rs | 32 +++---- zebrad/tests/acceptance.rs | 18 ++-- 13 files changed, 220 insertions(+), 155 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 92ce4c0cbe5..053df805838 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6974,6 +6974,7 @@ dependencies = [ "color-eyre", "crossbeam-channel", "derive-getters", + "derive-new", "dirs", "elasticsearch", "futures", diff --git a/zebra-state/CHANGELOG.md b/zebra-state/CHANGELOG.md index 221e7b3329b..7ea81193596 100644 --- a/zebra-state/CHANGELOG.md +++ b/zebra-state/CHANGELOG.md @@ -5,6 +5,20 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [2.1.0] - 2025-XX-XX + +### Breaking Changes + +- Updated error messages in response to failed `CommitSemanticallyVerifiedBlock` state requests ([#9923](https://github.com/ZcashFoundation/zebra/pull/9923)) + +## Added + +- Added `MappedRequest` trait and `CommitSemanticallyVerifiedBlockRequest` for convenient state response and error type conversions ([#9923](https://github.com/ZcashFoundation/zebra/pull/9923)) + +## Fixed + +- Replaced boxed-string errors in response to failed `CommitSemanticallyVerifiedBlock` and `ReconsiderBlock` state requests with concrete error type ([#9848](https://github.com/ZcashFoundation/zebra/pull/9848), [#9923](https://github.com/ZcashFoundation/zebra/pull/9923), [#9919](https://github.com/ZcashFoundation/zebra/pull/9919)) + ## [2.0.0] - 2025-08-07 ### Breaking Changes diff --git a/zebra-state/Cargo.toml b/zebra-state/Cargo.toml index 169a18a4718..b2230e72de0 100644 --- a/zebra-state/Cargo.toml +++ b/zebra-state/Cargo.toml @@ -87,6 +87,7 @@ zebra-test = { path = "../zebra-test/", version = "1.0.1", optional = true } proptest = { workspace = true, optional = true } proptest-derive = { workspace = true, optional = true } derive-getters.workspace = true +derive-new.workspace = true [dev-dependencies] color-eyre = { workspace = true } diff --git a/zebra-state/src/error.rs b/zebra-state/src/error.rs index 4c249524ede..d964c963de6 100644 --- a/zebra-state/src/error.rs +++ b/zebra-state/src/error.rs @@ -3,6 +3,7 @@ use std::sync::Arc; use chrono::{DateTime, Utc}; +use derive_new::new; use thiserror::Error; use zebra_chain::{ @@ -41,10 +42,69 @@ impl From for CloneError { /// A boxed [`std::error::Error`]. pub type BoxError = Box; -/// An error describing the reason a semantically verified block could not be committed to the state. -#[derive(Debug, Clone, Error, PartialEq, Eq)] -#[error("block is not contextually valid: {}", .0)] -pub struct CommitSemanticallyVerifiedError(#[from] Box); +/// An error describing why a block could not be queued to be committed to the state. +#[derive(Debug, Error, Clone, PartialEq, Eq, new)] +pub enum QueueAndCommitError { + #[error("block hash {block_hash} has already been sent to be committed to the state")] + #[non_exhaustive] + Duplicate { block_hash: block::Hash }, + + #[error("block height {block_height:?} is already committed in the finalized state")] + #[non_exhaustive] + AlreadyFinalized { block_height: block::Height }, + + #[error("block hash {block_hash} was replaced by a newer commit request")] + #[non_exhaustive] + Replaced { block_hash: block::Hash }, + + #[error("pruned block at or below the finalized tip height: {block_height:?}")] + #[non_exhaustive] + Pruned { block_height: block::Height }, + + #[error("block {block_hash} was dropped from the queue of non-finalized blocks")] + #[non_exhaustive] + Dropped { block_hash: block::Hash }, + + #[error("block commit task exited. Is Zebra shutting down?")] + #[non_exhaustive] + CommitTaskExited, + + #[error("dropping the state: dropped unused non-finalized state queue block")] + #[non_exhaustive] + DroppedUnusedBlock, +} + +/// An error describing why a `CommitSemanticallyVerified` request failed. +#[derive(Debug, Error, Clone, PartialEq, Eq)] +#[non_exhaustive] +pub enum CommitSemanticallyVerifiedError { + /// Queuing/commit step failed. + #[error("could not queue and commit semantically verified block")] + QueueAndCommitError(#[from] QueueAndCommitError), + /// Contextual validation failed. + #[error("could not contextually validate semantically verified block")] + ValidateContextError(#[from] ValidateContextError), + /// The write task exited (likely during shutdown). + #[error("block write task has exited. Is Zebra shutting down?")] + WriteTaskExited, +} + +#[derive(Debug, Error)] +pub enum LayeredStateError { + #[error("{0}")] + State(E), + #[error("{0}")] + Layer(BoxError), +} + +impl From for LayeredStateError { + fn from(err: BoxError) -> Self { + match err.downcast::() { + Ok(state_err) => Self::State(*state_err), + Err(layer_error) => Self::Layer(layer_error), + } + } +} /// An error describing the reason a block or its descendants could not be reconsidered after /// potentially being invalidated from the chain_set. @@ -294,34 +354,6 @@ pub enum ValidateContextError { tx_index_in_block: Option, transaction_hash: transaction::Hash, }, - - #[error("block hash {block_hash} has already been sent to be committed to the state")] - #[non_exhaustive] - DuplicateCommitRequest { block_hash: block::Hash }, - - #[error("block height {block_height:?} is already committed in the finalized state")] - #[non_exhaustive] - AlreadyFinalized { block_height: block::Height }, - - #[error("block hash {block_hash} was replaced by a newer commit request")] - #[non_exhaustive] - ReplacedByNewerRequest { block_hash: block::Hash }, - - #[error("pruned block at or below the finalized tip height: {block_height:?}")] - #[non_exhaustive] - PrunedBelowFinalizedTip { block_height: block::Height }, - - #[error("block {block_hash} was dropped from the queue of non-finalized blocks")] - #[non_exhaustive] - DroppedCommitRequest { block_hash: block::Hash }, - - #[error("block commit task exited. Is Zebra shutting down?")] - #[non_exhaustive] - CommitTaskExited, - - #[error("dropping the state: dropped unused non-finalized state queue block")] - #[non_exhaustive] - DroppedUnusedBlock, } /// Trait for creating the corresponding duplicate nullifier error from a nullifier. diff --git a/zebra-state/src/lib.rs b/zebra-state/src/lib.rs index 5d156e7bb92..9e594eaf0b8 100644 --- a/zebra-state/src/lib.rs +++ b/zebra-state/src/lib.rs @@ -42,7 +42,8 @@ pub use error::{ ValidateContextError, }; pub use request::{ - CheckpointVerifiedBlock, HashOrHeight, ReadRequest, Request, SemanticallyVerifiedBlock, + CheckpointVerifiedBlock, CommitSemanticallyVerifiedBlockRequest, HashOrHeight, MappedRequest, + ReadRequest, Request, SemanticallyVerifiedBlock, }; #[cfg(feature = "indexer")] diff --git a/zebra-state/src/request.rs b/zebra-state/src/request.rs index d77a32b3f60..f293aa234d3 100644 --- a/zebra-state/src/request.rs +++ b/zebra-state/src/request.rs @@ -6,6 +6,7 @@ use std::{ sync::Arc, }; +use tower::{BoxError, Service, ServiceExt}; use zebra_chain::{ amount::{DeferredPoolBalanceChange, NegativeAllowed}, block::{self, Block, HeightDiff}, @@ -28,6 +29,7 @@ use crate::{ constants::{MAX_FIND_BLOCK_HASHES_RESULTS, MAX_FIND_BLOCK_HEADERS_RESULTS}, ReadResponse, Response, }; +use crate::{error::LayeredStateError, CommitSemanticallyVerifiedError}; /// Identify a spend by a transparent outpoint or revealed nullifier. /// @@ -327,7 +329,8 @@ pub struct Treestate { } impl Treestate { - pub fn new( + #[allow(missing_docs)] + pub(crate) fn new( sprout: Arc, sapling: Arc, orchard: Arc, @@ -352,6 +355,7 @@ impl Treestate { /// /// Zebra's state service passes this `enum` over to the finalized state /// when committing a block. +#[allow(missing_docs)] pub enum FinalizableBlock { Checkpoint { checkpoint_verified: CheckpointVerifiedBlock, @@ -629,6 +633,60 @@ impl DerefMut for CheckpointVerifiedBlock { } } +/// Helper trait for convenient access to expected response and error types. +pub trait MappedRequest: Sized + Send + 'static { + /// Expected response type for this state request. + type MappedResponse; + /// Expected error type for this state request. + type Error: std::error::Error + std::fmt::Display + 'static; + + /// Maps the request type to a [`Request`]. + fn map_request(self) -> Request; + + /// Maps the expected [`Response`] variant for this request to the mapped response type. + fn map_response(response: Response) -> Self::MappedResponse; + + /// Accepts a state service to call, maps this request to a [`Request`], waits for the state to be ready, + /// calls the state with the mapped request, then maps the success or error response to the expected response + /// or error type for this request. + /// + /// Returns a [`Result>`]. + #[allow(async_fn_in_trait)] + async fn mapped_oneshot( + self, + state: &mut State, + ) -> Result> + where + State: Service, + State::Future: Send, + { + let response = state.ready().await?.call(self.map_request()).await?; + Ok(Self::map_response(response)) + } +} + +/// Performs contextual validation of the given semantically verified block, +/// committing it to the state if successful. +/// +/// See the [`crate`] documentation and [`Request::CommitSemanticallyVerifiedBlock`] for details. +pub struct CommitSemanticallyVerifiedBlockRequest(pub SemanticallyVerifiedBlock); + +impl MappedRequest for CommitSemanticallyVerifiedBlockRequest { + type MappedResponse = block::Hash; + type Error = CommitSemanticallyVerifiedError; + + fn map_request(self) -> Request { + Request::CommitSemanticallyVerifiedBlock(self.0) + } + + fn map_response(response: Response) -> Self::MappedResponse { + match response { + Response::Committed(hash) => hash, + _ => unreachable!("wrong response variant for request"), + } + } +} + #[derive(Clone, Debug, PartialEq, Eq)] /// A query about or modification to the chain state, via the /// [`StateService`](crate::service::StateService). @@ -640,8 +698,8 @@ pub enum Request { /// until its parent is ready. /// /// Returns [`Response::Committed`] with the hash of the block when it is - /// committed to the state, or an error if the block fails contextual - /// validation or has already been committed to the state. + /// committed to the state, or a [`CommitSemanticallyVerifiedBlockError`][0] if + /// the block fails contextual validation or otherwise could not be committed. /// /// This request cannot be cancelled once submitted; dropping the response /// future will have no effect on whether it is eventually processed. A @@ -653,6 +711,8 @@ pub enum Request { /// Block commit requests should be wrapped in a timeout, so that /// out-of-order and invalid requests do not hang indefinitely. See the [`crate`] /// documentation for details. + /// + /// [0]: (crate::error::CommitSemanticallyVerifiedBlockError) CommitSemanticallyVerifiedBlock(SemanticallyVerifiedBlock), /// Commit a checkpointed block to the state, skipping most but not all diff --git a/zebra-state/src/response.rs b/zebra-state/src/response.rs index 3a33820e9e5..1d1f13229e4 100644 --- a/zebra-state/src/response.rs +++ b/zebra-state/src/response.rs @@ -30,8 +30,8 @@ use crate::{service::read::AddressUtxos, NonFinalizedState, TransactionLocation, #[derive(Clone, Debug, PartialEq, Eq)] /// A response to a [`StateService`](crate::service::StateService) [`Request`]. pub enum Response { - /// Response to [`Request::CommitSemanticallyVerifiedBlock`] indicating that a block was - /// successfully committed to the state. + /// Response to [`Request::CommitSemanticallyVerifiedBlock`] and [`Request::CommitCheckpointVerifiedBlock`] + /// indicating that a block was successfully committed to the state. Committed(block::Hash), /// Response to [`Request::InvalidateBlock`] indicating that a block was found and diff --git a/zebra-state/src/service.rs b/zebra-state/src/service.rs index 4da4b5bcc69..06361a1773a 100644 --- a/zebra-state/src/service.rs +++ b/zebra-state/src/service.rs @@ -45,7 +45,7 @@ use crate::{ constants::{ MAX_FIND_BLOCK_HASHES_RESULTS, MAX_FIND_BLOCK_HEADERS_RESULTS, MAX_LEGACY_CHAIN_BLOCKS, }, - error::ReconsiderError, + error::{QueueAndCommitError, ReconsiderError}, response::NonFinalizedBlocksListener, service::{ block_iter::any_ancestor_blocks, @@ -57,7 +57,7 @@ use crate::{ watch_receiver::WatchReceiver, }, BoxError, CheckpointVerifiedBlock, CommitSemanticallyVerifiedError, Config, ReadRequest, - ReadResponse, Request, Response, SemanticallyVerifiedBlock, ValidateContextError, + ReadResponse, Request, Response, SemanticallyVerifiedBlock, }; pub mod block_iter; @@ -235,9 +235,7 @@ impl Drop for StateService { self.clear_finalized_block_queue( "dropping the state: dropped unused finalized state queue block", ); - self.clear_non_finalized_block_queue(CommitSemanticallyVerifiedError::from(Box::new( - ValidateContextError::DroppedUnusedBlock, - ))); + self.clear_non_finalized_block_queue(CommitSemanticallyVerifiedError::WriteTaskExited); // Log database metrics before shutting down info!("dropping the state: logging database metrics"); @@ -635,38 +633,36 @@ impl StateService { /// in RFC0005. /// /// [1]: https://zebra.zfnd.org/dev/rfcs/0005-state-updates.html#committing-non-finalized-blocks - #[instrument(level = "debug", skip(self, semantically_verrified))] + #[instrument(level = "debug", skip(self, semantically_verified))] fn queue_and_commit_to_non_finalized_state( &mut self, - semantically_verrified: SemanticallyVerifiedBlock, + semantically_verified: SemanticallyVerifiedBlock, ) -> oneshot::Receiver> { - tracing::debug!(block = %semantically_verrified.block, "queueing block for contextual verification"); - let parent_hash = semantically_verrified.block.header.previous_block_hash; + tracing::debug!(block = %semantically_verified.block, "queueing block for contextual verification"); + let parent_hash = semantically_verified.block.header.previous_block_hash; if self .non_finalized_block_write_sent_hashes - .contains(&semantically_verrified.hash) + .contains(&semantically_verified.hash) { let (rsp_tx, rsp_rx) = oneshot::channel(); - let _ = rsp_tx.send(Err(CommitSemanticallyVerifiedError::from(Box::new( - ValidateContextError::DuplicateCommitRequest { - block_hash: semantically_verrified.hash, - }, - )))); + let _ = rsp_tx.send(Err(QueueAndCommitError::new_duplicate( + semantically_verified.hash, + ) + .into())); return rsp_rx; } if self .read_service .db - .contains_height(semantically_verrified.height) + .contains_height(semantically_verified.height) { let (rsp_tx, rsp_rx) = oneshot::channel(); - let _ = rsp_tx.send(Err(CommitSemanticallyVerifiedError::from(Box::new( - ValidateContextError::AlreadyFinalized { - block_height: semantically_verrified.height, - }, - )))); + let _ = rsp_tx.send(Err(QueueAndCommitError::new_already_finalized( + semantically_verified.height, + ) + .into())); return rsp_rx; } @@ -675,21 +671,20 @@ impl StateService { // it with the newer request. let rsp_rx = if let Some((_, old_rsp_tx)) = self .non_finalized_state_queued_blocks - .get_mut(&semantically_verrified.hash) + .get_mut(&semantically_verified.hash) { tracing::debug!("replacing older queued request with new request"); let (mut rsp_tx, rsp_rx) = oneshot::channel(); std::mem::swap(old_rsp_tx, &mut rsp_tx); - let _ = rsp_tx.send(Err(CommitSemanticallyVerifiedError::from(Box::new( - ValidateContextError::ReplacedByNewerRequest { - block_hash: semantically_verrified.hash, - }, - )))); + let _ = rsp_tx.send(Err(QueueAndCommitError::new_replaced( + semantically_verified.hash, + ) + .into())); rsp_rx } else { let (rsp_tx, rsp_rx) = oneshot::channel(); self.non_finalized_state_queued_blocks - .queue((semantically_verrified, rsp_tx)); + .queue((semantically_verified, rsp_tx)); rsp_rx }; @@ -785,15 +780,11 @@ impl StateService { // If Zebra is shutting down, drop blocks and return an error. Self::send_semantically_verified_block_error( queued, - CommitSemanticallyVerifiedError::from(Box::new( - ValidateContextError::CommitTaskExited, - )), + CommitSemanticallyVerifiedError::WriteTaskExited, ); self.clear_non_finalized_block_queue( - CommitSemanticallyVerifiedError::from(Box::new( - ValidateContextError::CommitTaskExited, - )), + CommitSemanticallyVerifiedError::WriteTaskExited, ); return; @@ -977,6 +968,8 @@ impl Service for StateService { match req { // Uses non_finalized_state_queued_blocks and pending_utxos in the StateService // Accesses shared writeable state in the StateService, NonFinalizedState, and ZebraDb. + // + // The expected error type for this request is `CommitSemanticallyVerifiedError`. Request::CommitSemanticallyVerifiedBlock(semantically_verified) => { self.assert_block_can_be_validated(&semantically_verified); @@ -1007,20 +1000,16 @@ impl Service for StateService { // The work is all done, the future just waits on a channel for the result timer.finish(module_path!(), line!(), "CommitSemanticallyVerifiedBlock"); - // Await the channel response, mapping any receive error into a BoxError. - // Then flatten the nested Result by converting the inner CommitSemanticallyVerifiedError into a BoxError. + // Await the channel response, flatten the result, map receive errors to + // `CommitSemanticallyVerifiedError::WriteTaskExited`. + // Then flatten the nested Result and convert any errors to a BoxError. let span = Span::current(); async move { rsp_rx .await - .map_err(|_recv_error| { - BoxError::from(CommitSemanticallyVerifiedError::from(Box::new( - ValidateContextError::NotReadyToBeCommitted, - ))) - }) - // TODO: replace with Result::flatten once it stabilises - // https://github.com/rust-lang/rust/issues/70142 - .and_then(|res| res.map_err(BoxError::from)) + .map_err(|_recv_error| CommitSemanticallyVerifiedError::WriteTaskExited) + .flatten() + .map_err(BoxError::from) .map(Response::Committed) } .instrument(span) diff --git a/zebra-state/src/service/check/tests/nullifier.rs b/zebra-state/src/service/check/tests/nullifier.rs index 1f4000b4db1..7ca829e6038 100644 --- a/zebra-state/src/service/check/tests/nullifier.rs +++ b/zebra-state/src/service/check/tests/nullifier.rs @@ -165,11 +165,10 @@ proptest! { // we might need to just check `is_err()` here prop_assert_eq!( commit_result, - Err(Box::new(DuplicateSproutNullifier { + Err(DuplicateSproutNullifier { nullifier: duplicate_nullifier, in_finalized_state: false, }) - .into()) ); // block was rejected prop_assert_eq!(Some((Height(0), genesis.hash)), read::best_tip(&non_finalized_state, &finalized_state.db)); @@ -224,11 +223,10 @@ proptest! { prop_assert_eq!( commit_result, - Err(Box::new(DuplicateSproutNullifier { + Err(DuplicateSproutNullifier { nullifier: duplicate_nullifier, in_finalized_state: false, }) - .into()) ); prop_assert_eq!(Some((Height(0), genesis.hash)), read::best_tip(&non_finalized_state, &finalized_state.db)); prop_assert!(non_finalized_state.eq_internal_state(&previous_mem)); @@ -285,11 +283,10 @@ proptest! { prop_assert_eq!( commit_result, - Err(Box::new(DuplicateSproutNullifier { + Err(DuplicateSproutNullifier { nullifier: duplicate_nullifier, in_finalized_state: false, }) - .into()) ); prop_assert_eq!(Some((Height(0), genesis.hash)), read::best_tip(&non_finalized_state, &finalized_state.db)); prop_assert!(non_finalized_state.eq_internal_state(&previous_mem)); @@ -392,11 +389,10 @@ proptest! { prop_assert_eq!( commit_result, - Err(Box::new(DuplicateSproutNullifier { + Err(DuplicateSproutNullifier { nullifier: duplicate_nullifier, in_finalized_state: duplicate_in_finalized_state, }) - .into()) ); let check_tx_no_duplicates_in_chain = @@ -517,11 +513,10 @@ proptest! { prop_assert_eq!( commit_result, - Err(Box::new(DuplicateSaplingNullifier { + Err(DuplicateSaplingNullifier { nullifier: duplicate_nullifier, in_finalized_state: false, }) - .into()) ); prop_assert_eq!(Some((Height(0), genesis.hash)), read::best_tip(&non_finalized_state, &finalized_state.db)); prop_assert!(non_finalized_state.eq_internal_state(&previous_mem)); @@ -573,11 +568,10 @@ proptest! { prop_assert_eq!( commit_result, - Err(Box::new(DuplicateSaplingNullifier { + Err(DuplicateSaplingNullifier { nullifier: duplicate_nullifier, in_finalized_state: false, }) - .into()) ); prop_assert_eq!(Some((Height(0), genesis.hash)), read::best_tip(&non_finalized_state, &finalized_state.db)); prop_assert!(non_finalized_state.eq_internal_state(&previous_mem)); @@ -670,11 +664,10 @@ proptest! { prop_assert_eq!( commit_result, - Err(Box::new(DuplicateSaplingNullifier { + Err(DuplicateSaplingNullifier { nullifier: duplicate_nullifier, in_finalized_state: duplicate_in_finalized_state, }) - .into()) ); let check_tx_no_duplicates_in_chain = @@ -798,11 +791,10 @@ proptest! { prop_assert_eq!( commit_result, - Err(Box::new(DuplicateOrchardNullifier { + Err(DuplicateOrchardNullifier { nullifier: duplicate_nullifier, in_finalized_state: false, }) - .into()) ); prop_assert_eq!(Some((Height(0), genesis.hash)), read::best_tip(&non_finalized_state, &finalized_state.db)); prop_assert!(non_finalized_state.eq_internal_state(&previous_mem)); @@ -858,11 +850,10 @@ proptest! { prop_assert_eq!( commit_result, - Err(Box::new(DuplicateOrchardNullifier { + Err(DuplicateOrchardNullifier { nullifier: duplicate_nullifier, in_finalized_state: false, }) - .into()) ); prop_assert_eq!(Some((Height(0), genesis.hash)), read::best_tip(&non_finalized_state, &finalized_state.db)); prop_assert!(non_finalized_state.eq_internal_state(&previous_mem)); @@ -958,11 +949,10 @@ proptest! { prop_assert_eq!( commit_result, - Err(Box::new(DuplicateOrchardNullifier { + Err(DuplicateOrchardNullifier { nullifier: duplicate_nullifier, in_finalized_state: duplicate_in_finalized_state, }) - .into()) ); let check_tx_no_duplicates_in_chain = diff --git a/zebra-state/src/service/check/tests/utxo.rs b/zebra-state/src/service/check/tests/utxo.rs index d878f6290b4..dd9017bea20 100644 --- a/zebra-state/src/service/check/tests/utxo.rs +++ b/zebra-state/src/service/check/tests/utxo.rs @@ -374,11 +374,10 @@ proptest! { // the block was rejected prop_assert_eq!( commit_result, - Err(Box::new(DuplicateTransparentSpend { + Err(DuplicateTransparentSpend { outpoint: expected_outpoint, location: "the same block", }) - .into()) ); prop_assert_eq!(Some((Height(0), genesis.hash)), read::best_tip(&non_finalized_state, &finalized_state.db)); @@ -438,11 +437,10 @@ proptest! { // the block was rejected prop_assert_eq!( commit_result, - Err(Box::new(DuplicateTransparentSpend { + Err(DuplicateTransparentSpend { outpoint: expected_outpoint, location: "the same block", }) - .into()) ); prop_assert_eq!(Some((Height(1), block1.hash())), read::best_tip(&non_finalized_state, &finalized_state.db)); @@ -523,11 +521,10 @@ proptest! { // the block was rejected prop_assert_eq!( commit_result, - Err(Box::new(DuplicateTransparentSpend { + Err(DuplicateTransparentSpend { outpoint: expected_outpoint, location: "the same block", }) - .into()) ); prop_assert_eq!(Some((Height(1), block1.hash())), read::best_tip(&non_finalized_state, &finalized_state.db)); @@ -674,20 +671,18 @@ proptest! { if use_finalized_state_spend { prop_assert_eq!( commit_result, - Err(Box::new(MissingTransparentOutput { + Err(MissingTransparentOutput { outpoint: expected_outpoint, location: "the non-finalized and finalized chain", }) - .into()) ); } else { prop_assert_eq!( commit_result, - Err(Box::new(DuplicateTransparentSpend { + Err(DuplicateTransparentSpend { outpoint: expected_outpoint, location: "the non-finalized chain", }) - .into()) ); } prop_assert_eq!(Some((Height(2), block2.hash())), read::best_tip(&non_finalized_state, &finalized_state.db)); @@ -749,11 +744,10 @@ proptest! { // the block was rejected prop_assert_eq!( commit_result, - Err(Box::new(MissingTransparentOutput { + Err(MissingTransparentOutput { outpoint: expected_outpoint, location: "the non-finalized and finalized chain", }) - .into()) ); prop_assert_eq!(Some((Height(0), genesis.hash)), read::best_tip(&non_finalized_state, &finalized_state.db)); @@ -816,10 +810,9 @@ proptest! { // the block was rejected prop_assert_eq!( commit_result, - Err(Box::new(EarlyTransparentSpend { + Err(EarlyTransparentSpend { outpoint: expected_outpoint, }) - .into()) ); prop_assert_eq!(Some((Height(0), genesis.hash)), read::best_tip(&non_finalized_state, &finalized_state.db)); diff --git a/zebra-state/src/service/queued_blocks.rs b/zebra-state/src/service/queued_blocks.rs index 1e99716b879..8128f59bc0c 100644 --- a/zebra-state/src/service/queued_blocks.rs +++ b/zebra-state/src/service/queued_blocks.rs @@ -11,8 +11,8 @@ use tracing::instrument; use zebra_chain::{block, transparent}; use crate::{ - BoxError, CheckpointVerifiedBlock, CommitSemanticallyVerifiedError, NonFinalizedState, - SemanticallyVerifiedBlock, ValidateContextError, + error::QueueAndCommitError, BoxError, CheckpointVerifiedBlock, CommitSemanticallyVerifiedError, + NonFinalizedState, SemanticallyVerifiedBlock, }; #[cfg(test)] @@ -145,11 +145,9 @@ impl QueuedBlocks { let parent_hash = &expired_block.block.header.previous_block_hash; // we don't care if the receiver was dropped - let _ = expired_sender.send(Err(CommitSemanticallyVerifiedError::from(Box::new( - ValidateContextError::PrunedBelowFinalizedTip { - block_height: expired_block.height, - }, - )))); + let _ = expired_sender.send(Err( + QueueAndCommitError::new_pruned(expired_block.height).into() + )); // TODO: only remove UTXOs if there are no queued blocks with that UTXO // (known_utxos is best-effort, so this is ok for now) diff --git a/zebra-state/src/service/write.rs b/zebra-state/src/service/write.rs index f6daa74add4..4292c3a29ed 100644 --- a/zebra-state/src/service/write.rs +++ b/zebra-state/src/service/write.rs @@ -23,7 +23,7 @@ use crate::{ queued_blocks::{QueuedCheckpointVerified, QueuedSemanticallyVerified}, BoxError, ChainTipBlock, ChainTipSender, ReconsiderError, }, - CommitSemanticallyVerifiedError, SemanticallyVerifiedBlock, + SemanticallyVerifiedBlock, ValidateContextError, }; // These types are used in doc links @@ -53,19 +53,14 @@ pub(crate) fn validate_and_commit_non_finalized( finalized_state: &ZebraDb, non_finalized_state: &mut NonFinalizedState, prepared: SemanticallyVerifiedBlock, -) -> Result<(), CommitSemanticallyVerifiedError> { - check::initial_contextual_validity(finalized_state, non_finalized_state, &prepared) - .map_err(Box::new)?; +) -> Result<(), ValidateContextError> { + check::initial_contextual_validity(finalized_state, non_finalized_state, &prepared)?; let parent_hash = prepared.block.header.previous_block_hash; if finalized_state.finalized_tip_hash() == parent_hash { - non_finalized_state - .commit_new_chain(prepared, finalized_state) - .map_err(Box::new)?; + non_finalized_state.commit_new_chain(prepared, finalized_state)?; } else { - non_finalized_state - .commit_block(prepared, finalized_state) - .map_err(Box::new)?; + non_finalized_state.commit_block(prepared, finalized_state)?; } Ok(()) @@ -336,8 +331,7 @@ impl WriteBlockWorkerTask { } // Save any errors to propagate down to queued child blocks - let mut parent_error_map: IndexMap = - IndexMap::new(); + let mut parent_error_map: IndexMap = IndexMap::new(); while let Some(msg) = non_finalized_block_write_receiver.blocking_recv() { let queued_child_and_rsp_tx = match msg { @@ -369,23 +363,21 @@ impl WriteBlockWorkerTask { let parent_hash = queued_child.block.header.previous_block_hash; let parent_error = parent_error_map.get(&parent_hash); - let result; - // If the parent block was marked as rejected, also reject all its children. // // At this point, we know that all the block's descendants // are invalid, because we checked all the consensus rules before // committing the failing ancestor block to the non-finalized state. - if let Some(parent_error) = parent_error { - result = Err(parent_error.clone()); + let result = if let Some(parent_error) = parent_error { + Err(parent_error.clone()) } else { tracing::trace!(?child_hash, "validating queued child"); - result = validate_and_commit_non_finalized( + validate_and_commit_non_finalized( &finalized_state.db, non_finalized_state, queued_child, ) - } + }; // TODO: fix the test timing bugs that require the result to be sent // after `update_latest_chain_channels()`, @@ -393,7 +385,7 @@ impl WriteBlockWorkerTask { if let Err(ref error) = result { // Update the caller with the error. - let _ = rsp_tx.send(result.clone().map(|()| child_hash)); + let _ = rsp_tx.send(result.clone().map(|()| child_hash).map_err(Into::into)); // If the block is invalid, mark any descendant blocks as rejected. parent_error_map.insert(child_hash, error.clone()); @@ -422,7 +414,7 @@ impl WriteBlockWorkerTask { ); // Update the caller with the result. - let _ = rsp_tx.send(result.clone().map(|()| child_hash)); + let _ = rsp_tx.send(result.clone().map(|()| child_hash).map_err(Into::into)); while non_finalized_state .best_chain_len() diff --git a/zebrad/tests/acceptance.rs b/zebrad/tests/acceptance.rs index a0478b356ed..59b7fc1048b 100644 --- a/zebrad/tests/acceptance.rs +++ b/zebrad/tests/acceptance.rs @@ -3649,9 +3649,7 @@ async fn has_spending_transaction_ids() -> Result<()> { use std::sync::Arc; use tower::Service; use zebra_chain::{chain_tip::ChainTip, transparent::Input}; - use zebra_state::{ - ReadRequest, ReadResponse, Request, Response, SemanticallyVerifiedBlock, Spend, - }; + use zebra_state::{ReadRequest, ReadResponse, SemanticallyVerifiedBlock, Spend}; use common::cached_state::future_blocks; @@ -3676,18 +3674,14 @@ async fn has_spending_transaction_ids() -> Result<()> { tracing::info!("committing blocks to non-finalized state"); for block in non_finalized_blocks { + use zebra_state::{CommitSemanticallyVerifiedBlockRequest, MappedRequest}; + let expected_hash = block.hash(); let block = SemanticallyVerifiedBlock::with_hash(Arc::new(block), expected_hash); - let Response::Committed(block_hash) = state - .ready() - .await - .map_err(|err| eyre!(err))? - .call(Request::CommitSemanticallyVerifiedBlock(block)) + let block_hash = CommitSemanticallyVerifiedBlockRequest(block) + .mapped_oneshot(&mut state) .await - .map_err(|err| eyre!(err))? - else { - panic!("unexpected response to Block request"); - }; + .map_err(|err| eyre!(err))?; assert_eq!( expected_hash, block_hash, From dba92fad94ecbb4a3a9c901aa95c41679b95777c Mon Sep 17 00:00:00 2001 From: Jack Grigg Date: Sat, 4 Oct 2025 00:43:11 +0100 Subject: [PATCH 014/431] Migrate to `zcash_primitives 0.25` etc. (#9927) * Migrate to `zcash_primitives 0.25` etc. * Replace `zcash_primitives::legacy` with `zcash_transparent::address` The former is deprecated and re-exports the latter. --------- Co-authored-by: Conrado Gouvea --- Cargo.lock | 123 +++++++----------- Cargo.toml | 12 +- zebra-chain/Cargo.toml | 1 + .../src/primitives/zcash_primitives.rs | 21 +-- zebra-chain/src/transparent/script.rs | 2 +- zebra-rpc/Cargo.toml | 1 + .../src/methods/types/get_block_template.rs | 13 +- .../types/get_block_template/zip317/tests.rs | 2 +- zebra-script/Cargo.toml | 1 + zebra-script/src/lib.rs | 102 ++++++++------- 10 files changed, 135 insertions(+), 143 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 053df805838..91217551729 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -454,7 +454,7 @@ dependencies = [ "bitflags 2.9.4", "cexpr", "clang-sys", - "itertools 0.12.1", + "itertools 0.10.5", "lazy_static", "lazycell", "proc-macro2", @@ -474,9 +474,7 @@ dependencies = [ "bitflags 2.9.4", "cexpr", "clang-sys", - "itertools 0.13.0", - "log", - "prettyplease", + "itertools 0.10.5", "proc-macro2", "quote", "regex", @@ -606,6 +604,15 @@ dependencies = [ "subtle", ] +[[package]] +name = "bounded-vec" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09dc0086e469182132244e9b8d313a0742e1132da43a08c24b9dd3c18e0faf3a" +dependencies = [ + "thiserror 2.0.16", +] + [[package]] name = "bs58" version = "0.5.1" @@ -773,7 +780,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "145052bdd345b87320e369255277e3fb5152762ad123a901ef5c262dd38fe8d2" dependencies = [ "iana-time-zone", - "num-traits 0.2.19", + "num-traits", "serde", "windows-link 0.2.0", ] @@ -1112,7 +1119,7 @@ dependencies = [ "criterion-plot", "is-terminal", "itertools 0.10.5", - "num-traits 0.2.19", + "num-traits", "once_cell", "oorandom", "plotters", @@ -1489,15 +1496,6 @@ dependencies = [ "cfg-if", ] -[[package]] -name = "enum_primitive" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "be4551092f4d519593039259a9ed8daedf0da12e5109c5280338073eaeb81180" -dependencies = [ - "num-traits 0.1.43", -] - [[package]] name = "env_home" version = "0.1.0" @@ -1673,7 +1671,7 @@ dependencies = [ "libm", "num-bigint", "num-integer", - "num-traits 0.2.19", + "num-traits", ] [[package]] @@ -2003,7 +2001,7 @@ dependencies = [ "byteorder", "flate2", "nom", - "num-traits 0.2.19", + "num-traits", ] [[package]] @@ -2545,24 +2543,6 @@ dependencies = [ "either", ] -[[package]] -name = "itertools" -version = "0.12.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ba291022dbbd398a455acf126c1e341954079855bc60dfdda641363bd6922569" -dependencies = [ - "either", -] - -[[package]] -name = "itertools" -version = "0.13.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "413ee7dfc52ee1a4949ceeb7dbc8a33f2d6c088194d9f922fb8318faf1f01186" -dependencies = [ - "either", -] - [[package]] name = "itertools" version = "0.14.0" @@ -2815,6 +2795,19 @@ dependencies = [ "vcpkg", ] +[[package]] +name = "libzcash_script" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f8ce05b56f3cbc65ec7d0908adb308ed91281e022f61c8c3a0c9388b5380b17" +dependencies = [ + "bindgen 0.72.1", + "cc", + "thiserror 2.0.16", + "tracing", + "zcash_script", +] + [[package]] name = "linux-raw-sys" version = "0.11.0" @@ -3051,7 +3044,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a5e44f723f1133c9deac646763579fdb3ac745e418f2a7af9cd0c431da1f20b9" dependencies = [ "num-integer", - "num-traits 0.2.19", + "num-traits", ] [[package]] @@ -3076,16 +3069,7 @@ version = "0.1.46" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7969661fd2958a5cb096e56c8e1ad0444ac2bbcd0061bd28660485a44879858f" dependencies = [ - "num-traits 0.2.19", -] - -[[package]] -name = "num-traits" -version = "0.1.43" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "92e5113e9fd4cc14ded8e499429f396a20f98c772a47cc8622a736e1ec843c31" -dependencies = [ - "num-traits 0.2.19", + "num-traits", ] [[package]] @@ -3451,7 +3435,7 @@ version = "0.3.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5aeb6f403d7a4911efb1e33402027fc44f29b5bf6def3effcc22d7bb75f2b747" dependencies = [ - "num-traits 0.2.19", + "num-traits", "plotters-backend", "plotters-svg", "wasm-bindgen", @@ -3609,7 +3593,7 @@ dependencies = [ "bit-vec", "bitflags 2.9.4", "lazy_static", - "num-traits 0.2.19", + "num-traits", "rand 0.9.2", "rand_chacha 0.9.0", "rand_xorshift", @@ -4691,17 +4675,6 @@ dependencies = [ "version_check", ] -[[package]] -name = "sha-1" -version = "0.10.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f5058ada175748e33390e40e872bd0fe59a19f265d0158daa551c5a88a76009c" -dependencies = [ - "cfg-if", - "cpufeatures", - "digest 0.10.7", -] - [[package]] name = "sha1" version = "0.10.6" @@ -6568,9 +6541,9 @@ dependencies = [ [[package]] name = "zcash_keys" -version = "0.10.1" +version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c6c8d3d5a08a66f76264c72172e692ec362218b091181cda30c04d00a4561cd8" +checksum = "650436433a6636ba41175a7537c44baddf8b09a38bf05c4e066b260a39729117" dependencies = [ "bech32", "blake2b_simd", @@ -6607,9 +6580,9 @@ dependencies = [ [[package]] name = "zcash_primitives" -version = "0.24.1" +version = "0.25.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "76362b79e432bde2f22b3defcb6919d4fb50446985997169da3cc3ae4035a6d9" +checksum = "08e60678c8119d878276c9b4f605f9dbe1f0c1b7ab69925f4d694c404b1cefdc" dependencies = [ "bip32", "blake2b_simd", @@ -6642,6 +6615,7 @@ dependencies = [ "zcash_encoding", "zcash_note_encryption", "zcash_protocol", + "zcash_script", "zcash_spec", "zcash_transparent", "zip32", @@ -6649,9 +6623,9 @@ dependencies = [ [[package]] name = "zcash_proofs" -version = "0.24.0" +version = "0.25.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9f90d9521161f7308c2fe6bddf771947f1a0fcd01b9e8a3b624c30a5661ad945" +checksum = "4202009c0e54662b8218e9161e5129e6875d58387d6080495107018c10af599f" dependencies = [ "bellman", "blake2b_simd", @@ -6685,20 +6659,17 @@ dependencies = [ [[package]] name = "zcash_script" -version = "0.3.2" +version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "caf6e76f310bb2d3cc233086a97c1710ba1de7ffbbf8198b8113407d0f427dfc" +checksum = "7c1d06ec5990ad2c51c3052fd41f87aa5ccab92d02d1ab5173712c2e37c24658" dependencies = [ - "bindgen 0.72.1", "bitflags 2.9.4", - "cc", - "enum_primitive", + "bounded-vec", "ripemd 0.1.3", "secp256k1", - "sha-1", + "sha1", "sha2 0.10.9", "thiserror 2.0.16", - "tracing", ] [[package]] @@ -6712,9 +6683,9 @@ dependencies = [ [[package]] name = "zcash_transparent" -version = "0.4.0" +version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3a7c162a8aa6f708e842503ed5157032465dadfb1d7f63adf9db2d45213a0b11" +checksum = "5ed1d3b5d7bdb547689bf78a2ca134455cf9d813956c1c734623fdb66446d0c8" dependencies = [ "bip32", "blake2b_simd", @@ -6730,6 +6701,7 @@ dependencies = [ "zcash_address", "zcash_encoding", "zcash_protocol", + "zcash_script", "zcash_spec", "zip32", ] @@ -6796,6 +6768,7 @@ dependencies = [ "zcash_note_encryption", "zcash_primitives", "zcash_protocol", + "zcash_script", "zcash_transparent", "zebra-test", ] @@ -6942,6 +6915,7 @@ dependencies = [ "zcash_keys", "zcash_primitives", "zcash_protocol", + "zcash_script", "zcash_transparent", "zebra-chain", "zebra-consensus", @@ -6958,6 +6932,7 @@ version = "2.0.0" dependencies = [ "hex", "lazy_static", + "libzcash_script", "thiserror 2.0.16", "zcash_primitives", "zcash_script", diff --git a/Cargo.toml b/Cargo.toml index 5938475f950..427db890442 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -24,13 +24,12 @@ incrementalmerkletree = { version = "0.8.2", features = ["legacy-api"] } orchard = "0.11" sapling-crypto = "0.5" zcash_address = "0.9" -zcash_client_backend = "0.19" zcash_encoding = "0.3" zcash_history = "0.4" -zcash_keys = "0.10" -zcash_primitives = "0.24" -zcash_proofs = "0.24" -zcash_transparent = "0.4" +zcash_keys = "0.11" +zcash_primitives = "0.25" +zcash_proofs = "0.25" +zcash_transparent = "0.5" zcash_protocol = "0.6" zip32 = "0.2" abscissa_core = "0.7" @@ -86,6 +85,7 @@ jsonrpsee-proc-macros = "0.24.9" jsonrpsee-types = "0.24.9" jubjub = "0.10" lazy_static = "1.4" +libzcash_script = "0.1" log = "0.4.27" metrics = "0.24" metrics-exporter-prometheus = { version = "0.16", default-features = false } @@ -152,7 +152,7 @@ uint = "0.10" vergen-git2 = { version = "1.0", default-features = false } x25519-dalek = "2.0.1" zcash_note_encryption = "0.4.1" -zcash_script = "0.3.2" +zcash_script = "0.4" config = { version = "0.15.14", features = ["toml"] } which = "8.0.0" diff --git a/zebra-chain/Cargo.toml b/zebra-chain/Cargo.toml index aa9ff2bc090..ad0857cea61 100644 --- a/zebra-chain/Cargo.toml +++ b/zebra-chain/Cargo.toml @@ -79,6 +79,7 @@ sha2 = { workspace = true, features = ["compress"] } uint = { workspace = true } x25519-dalek = { workspace = true, features = ["serde"] } bech32 = { workspace = true } +zcash_script.workspace = true # ECC deps halo2 = { package = "halo2_proofs", version = "0.3" } diff --git a/zebra-chain/src/primitives/zcash_primitives.rs b/zebra-chain/src/primitives/zcash_primitives.rs index 886ce6ea4b5..27314292ae2 100644 --- a/zebra-chain/src/primitives/zcash_primitives.rs +++ b/zebra-chain/src/primitives/zcash_primitives.rs @@ -5,6 +5,7 @@ use std::{io, ops::Deref, sync::Arc}; use zcash_primitives::transaction::{self as zp_tx, TxDigests}; use zcash_protocol::value::{BalanceError, ZatBalance, Zatoshis}; +use zcash_script::script; use crate::{ amount::{Amount, NonNegative}, @@ -25,7 +26,7 @@ struct TransparentAuth { } impl zcash_transparent::bundle::Authorization for TransparentAuth { - type ScriptSig = zcash_primitives::legacy::Script; + type ScriptSig = zcash_transparent::address::Script; } // In this block we convert our Output to a librustzcash to TxOut. @@ -43,11 +44,13 @@ impl zcash_transparent::sighash::TransparentAuthorizingContext for TransparentAu .collect() } - fn input_scriptpubkeys(&self) -> Vec { + fn input_scriptpubkeys(&self) -> Vec { self.all_prev_outputs .iter() .map(|prevout| { - zcash_primitives::legacy::Script(prevout.lock_script.as_raw_bytes().into()) + zcash_transparent::address::Script(script::Code( + prevout.lock_script.as_raw_bytes().into(), + )) }) .collect() } @@ -189,14 +192,14 @@ impl TryFrom for ZatBalance { } /// Convert a Zebra Script into a librustzcash one. -impl From<&Script> for zcash_primitives::legacy::Script { +impl From<&Script> for zcash_transparent::address::Script { fn from(script: &Script) -> Self { - zcash_primitives::legacy::Script(script.as_raw_bytes().to_vec()) + zcash_transparent::address::Script(script::Code(script.as_raw_bytes().to_vec())) } } /// Convert a Zebra Script into a librustzcash one. -impl From + + + + + diff --git a/book/valarbook/pages/zcashd-compat.md b/book/valarbook/pages/zcashd-compat.md deleted file mode 100644 index 70a352ab0e8..00000000000 --- a/book/valarbook/pages/zcashd-compat.md +++ /dev/null @@ -1,230 +0,0 @@ -# zebra-zcashd-compat - -Run Zebra for consensus and P2P, while keeping zcashd for wallet and zcashd-compatible RPC surfaces. - -**Status:** Alpha - -## Integration Summary - -`zebra-zcashd-compat` is a Zebra binary that manages a zcashd child process. Zebra is responsible for consensus and P2P. zcashd is kept for wallet and zcashd-compatible RPC surfaces. zcashd ingests chain and mempool data from Zebra over authenticated RPC. The primary purpose of this integration is to allow exchanges to smoothly migrate to a Zebra node without disrupting their existing zcashd setup. - -- **Trust boundary:** zcashd trusts the configured Zebra node for block source and transaction forwarding. -- **Transport:** zcashd talks to Zebra over RPC only. There is no zcashd P2P in compat mode. -- **Auth:** keep Zebra cookie auth enabled and avoid exposing RPC publicly. -- **Deployment:** deploy zcashd and Zebra on the same machine. -- **Lifecycle:** supervised mode manages the zcashd process lifecycle at runtime. For container builds, zcashd artifact fetch and SHA verification happen outside the Dockerfile before image build. - -## Prerequisites - -Because this setup runs a Zebra node alongside your zcashd wallet API, the hardware requirements are increased. Provision the recommended hardware before restarting from the updated binary. - -### Hardware Requirements - -- **CPU:** 8 logical CPUs available to the process. -- **RAM:** 32 GiB effective memory. -- **Disk:** at least 1 TiB combined capacity across the filesystems used by Zebra state and zcashd datadir. - -Startup fails below the minimums: 4 logical CPUs, 16 GiB RAM, 300 GiB per data volume, or 600 GiB if Zebra state and the zcashd datadir share a filesystem. Bypass only with `--unsafe-low-specs`. Linux x86_64 only. - -### Downloads - -Binaries: - -- [Zebra with zcashd compatibility](https://github.com/valargroup/zebra/releases) - -Snapshots: - -- [Zebra](https://zebra.valargroup.org/) -- [zcashd](https://zcashd.valargroup.org/) - -## Configuration - -Choose one of the setup modes below. - -### Supervised - -Supervised mode is the default path: one Zebra binary. Zebra resolves a hash-pinned compatible zcashd build, starts it as a child process, and supervises restarts. - -Start Zebra in zcashd-compat mode: - -```bash -zebrad start --zcashd-compat -``` - -Use this minimal configuration to keep supervision enabled so Zebra manages install and lifecycle: - -```text -[zcashd_compat] -enabled = true -manage_zcashd = true -zcashd_source = "managed" - -# Optionally, customize zcashd data directory. -zcashd_datadir = "/path/to/zcashd/datadir" - -# Optionally, propagate zcashd CLI arguments via env var. -ZEBRA_ZCASHD_COMPAT__ZCASHD_EXTRA_ARGS='["-printtoconsole"]' -``` - -For full configuration reference, see the [full documentation](https://github.com/valargroup/zebra/blob/ironwood-main/book/src/user/zcashd-compat.md). - -### Build From Source - -Build Zebra and zcashd from source: - -```bash -# in zebrad tree -cargo build --release - -# in zcashd tree -./zcutil/build.sh -j"$(nproc)" -``` - -Configure Zebra for manual operation without zcashd supervision: - -```toml -[zcashd_compat] -enabled = true -manage_zcashd = false -``` - -Then run zcashd against Zebra RPC manually: - -```bash -zebrad start --zcashd-compat - -# Default location, to which Zebra writes an auth cookie -ZEBRA_COMPAT_COOKIE="$HOME/.cache/zebra/.zcashd-compat.cookie" - -# Ensure Zebra has written it -test -f "$ZEBRA_COMPAT_COOKIE" && echo "cookie ready: $ZEBRA_COMPAT_COOKIE" - -# One-time zcashd datadir setup (zcashd refuses to start without this line) -mkdir -p ~/.zcash -echo 'i-am-aware-zcashd-will-be-replaced-by-zebrad-and-zallet-in-2025=1' >> ~/.zcash/zcash.conf - -# Start zcashd in compat mode, pointed at Zebra's compat listener -./src/zcashd -zebra-compat \ - -zebra-compat-url=http://127.0.0.1:28232 \ - -zebra-compat-cookiefile="$HOME/.cache/zebra/.zcashd-compat.cookie" \ - -printtoconsole - -# verify -./src/zcash-cli getzebracompatinfo -``` - -### Docker - -Artifacts are fetched and hash-verified externally, then passed in as build context. You can also follow the build-from-source instructions if desired. - -Install Docker Buildx and verify it before using the Docker setup mode: - -```bash -apt-get update && apt-get install -y docker-buildx && docker buildx version -``` - -Set the image tag, choose a hash-pinned zcashd artifact for your platform, and define the local build context path: - -```bash -export ZEBRA_DOCKER_IMAGE="zebra:zcashd-compat" - -# Pick the matching artifact for your host architecture. -# x86_64 example: -export ZCASHD_COMPAT_URL="https://github.com/valargroup/zcashd/releases/download/v6.2.1-alpha/zcashd-zebra-compat-v6.2.1-alpha-linux-x86_64.tar.gz" -export ZCASHD_COMPAT_SHA256="09e640b55c9af91dee5742e5e9bb6712f92d7073f0fe899ca58d43f62eb9d13c" - -# Local Docker build context that will contain ./bin/zcashd. -export ZCASHD_COMPAT_CONTEXT="${PWD}/target/zcashd-compat/context" - -# State directories for Zebra and zcashd. -export ZEBRA_STATE_CACHE_DIR="/mnt/data/zebra-state" -export ZCASHD_DATADIR="/mnt/data/.zcashd" -``` - -Download the artifact, verify its SHA256, extract it into a local Docker build context, and confirm the expected binary exists: - -```bash -mkdir -p "${ZCASHD_COMPAT_CONTEXT}" -curl -fsSL "${ZCASHD_COMPAT_URL}" -o /tmp/zcashd-compat.tar.gz -echo "${ZCASHD_COMPAT_SHA256} /tmp/zcashd-compat.tar.gz" | sha256sum -c - -tar -xzf /tmp/zcashd-compat.tar.gz -C "${ZCASHD_COMPAT_CONTEXT}" - -# Guardrail: this fails early if ./bin/zcashd is missing or not executable. -test -x "${ZCASHD_COMPAT_CONTEXT}/bin/zcashd" -``` - -Build the runtime-zcashd-compat target and inject the prepared zcashd binary via `--build-context`: - -```bash -docker build \ - -f ./docker/Dockerfile \ - --target runtime-zcashd-compat \ - --build-context "zcashd_compat=${ZCASHD_COMPAT_CONTEXT}" \ - --tag "${ZEBRA_DOCKER_IMAGE}" \ - . -``` - -Ensure Zebra and zcashd data folders are accessible inside the container: - -```bash -# 10001 is the default Dockerfile container user -# Ensure that zebra and zcashd data folders are accessible inside the container -mkdir -p "${ZEBRA_STATE_CACHE_DIR}" "${ZCASHD_DATADIR}" -chown -R 10001:10001 "${ZEBRA_STATE_CACHE_DIR}" "${ZCASHD_DATADIR}" -``` - -Start Zebra in zcashd compatibility mode with persistent Zebra and zcashd state mounted from the host: - -```bash -docker run --rm -it \ - -e ZCASHD_COMPAT_ENABLED=true \ - -e ZEBRA_NETWORK__LISTEN_ADDR="[::]:18233" \ - -e ZEBRA_STATE__CACHE_DIR="/home/zebra/.cache/zebra" \ - -e ZEBRA_ZCASHD_COMPAT__ZCASHD_DATADIR="/home/zebra/.cache/zcashd" \ - -e ZEBRA_ZCASHD_COMPAT__ZCASHD_EXTRA_ARGS='["-rpcbind=0.0.0.0","-rpcallowip=0.0.0.0/0"]' \ - --mount type=bind,src="${ZEBRA_STATE_CACHE_DIR}",dst="/home/zebra/.cache/zebra" \ - --mount type=bind,src="${ZCASHD_DATADIR}",dst="/home/zebra/.cache/zcashd" \ - -p 18233:18233 \ - -p 127.0.0.1:8232:8232 \ - "${ZEBRA_DOCKER_IMAGE}" \ - zebrad start --zcashd-compat -``` - -## FAQ - -### How do I preserve my original zcashd configuration? - -Point Zebra at your existing zcashd data directory with `zcashd_datadir`, and pass any required zcashd CLI arguments through `zcashd_extra_args`. These settings can be provided in Zebra configuration, or through equivalent runtime overrides where supported. See the [full instructions](https://github.com/valargroup/zebra/blob/ironwood-main/book/src/user/zcashd-compat.md) for details. - -Remove peer-directing options from your existing `zcash.conf` first: - -- `bind=` -- `whitebind=` -- `connect=` -- `addnode=` -- `seednode=` - -Startup fails validation if they are present. - -### If my original hardware is below the recommended specification, do I have to provision more resources? - -Yes. Because this mode runs Zebra alongside the zcashd wallet API, nodes below the minimum specification fail startup preflight. Between minimum and recommended, Zebra starts with warnings but is not supported for production. Provision hardware that meets the requirements in this guide before restarting with the updated binary. - -### I have a restricted setup that does not allow IPC and also restricts networking. What do you recommend? - -Build from source and configure the components manually. This gives you direct control over how Zebra and zcashd communicate in your restricted environment. - -### How is my old zcashd config managed? - -Even if present, the system force-disables the following zcashd config parameters because P2P is unused: - -- `p2p` -- `listen` -- `dnsseed` -- `listenonion` - -Otherwise, zcashd would fail to start. - -### What happens if I do not have a prior config? - -A default one with sane defaults is created for you. See the [full instructions](https://github.com/valargroup/zebra/blob/ironwood-main/book/src/user/zcashd-compat.md) for details about location. diff --git a/book/valarbook/static/zcash-logo.svg b/book/valarbook/static/zcash-logo.svg new file mode 100644 index 00000000000..de0f51c413c --- /dev/null +++ b/book/valarbook/static/zcash-logo.svg @@ -0,0 +1,5 @@ + + + + + From eda405dae51d7462c458d99884ac1115643a613e Mon Sep 17 00:00:00 2001 From: Roman Akhtariev Date: Fri, 12 Jun 2026 22:06:51 -0300 Subject: [PATCH 302/431] docs: improve instructions (#66) --- book/valarbook/index.html | 88 +++++++++++++++++++++++++++++++++------ 1 file changed, 76 insertions(+), 12 deletions(-) diff --git a/book/valarbook/index.html b/book/valarbook/index.html index e03903eef16..00e936a43a0 100644 --- a/book/valarbook/index.html +++ b/book/valarbook/index.html @@ -574,7 +574,7 @@

Integration Summary

  • Transport: zcashd talks to Zebra over RPC only (no zcashd P2P in compat mode).
  • Auth: keep Zebra cookie auth enabled and avoid exposing RPC publicly.
  • Deployment: deploy zcashd and Zebra on the same machine.
  • -
  • Lifecycle: supervised mode manages zcashd process lifecycle at runtime. For container builds, zcashd artifact fetch and SHA verification happen outside the Dockerfile before image build.
  • +
  • Lifecycle: managed mode manages zcashd process lifecycle at runtime. For container builds, zcashd artifact fetch and SHA verification happen outside the Dockerfile before image build.
  • @@ -620,27 +620,91 @@

    Configuration

    data-panel-scope=".panel"> + data-panel-target="path-panel" + aria-pressed="true">Path + -
    + + + -
    +

    Install zcashd-compat

      -
    1. +
    2. Run Installer

      From 921f9c498c811ba658c91a773be90be21338ddc2 Mon Sep 17 00:00:00 2001 From: Roman Akhtariev Date: Sat, 13 Jun 2026 13:47:23 -0300 Subject: [PATCH 311/431] fix(ci): repair release asset publishing (#74) --- .github/workflows/release-binaries.yml | 35 +++++++++++++++++++++++--- 1 file changed, 32 insertions(+), 3 deletions(-) diff --git a/.github/workflows/release-binaries.yml b/.github/workflows/release-binaries.yml index 2c7aea4cbeb..117e2b974f0 100644 --- a/.github/workflows/release-binaries.yml +++ b/.github/workflows/release-binaries.yml @@ -229,9 +229,37 @@ jobs: find ./publish -type f -name 'zebrad-*.tar.gz' -exec cp {} ./publish/final/ \; cp ./scripts/install-zcashd-compat.sh ./publish/final/ + RELEASE_TAG="$RELEASE_TAG" python3 - <<'PY' + import os + import re + from pathlib import Path + + release_tag = os.environ["RELEASE_TAG"] + image_version = release_tag.removeprefix("v") + installer_path = Path("./publish/final/install-zcashd-compat.sh") + installer = installer_path.read_text(encoding="utf-8") + replacements = { + "ZEBRA_RELEASE_TAG": release_tag, + "ZEBRA_DOCKER_IMAGE": f"valaroman/zebra:{image_version}", + "ZEBRA_COMPAT_DOCKER_IMAGE": f"valaroman/zebra:zcashd-compat-{image_version}", + } + for name, value in replacements.items(): + installer, count = re.subn( + rf'^{name}="[^"]*"$', + f'{name}="{value}"', + installer, + count=1, + flags=re.MULTILINE, + ) + if count != 1: + raise SystemExit(f"Expected exactly one {name} assignment in installer.") + installer_path.write_text(installer, encoding="utf-8") + PY ( cd ./publish/final - sha256sum ./* > SHA256SUMS.txt + rm -f SHA256SUMS.txt + sha256sum ./* > ../SHA256SUMS.txt + mv ../SHA256SUMS.txt SHA256SUMS.txt ) - name: Ensure release exists @@ -241,7 +269,7 @@ jobs: GH_TOKEN: ${{ github.token }} REPOSITORY: ${{ github.repository }} RELEASE_TAG: ${{ needs.resolve-release.outputs.release_tag }} - TARGET_SHA: ${{ needs.resolve-release.outputs.checkout_ref }} + TARGET_REF: ${{ needs.resolve-release.outputs.checkout_ref }} run: | set -euo pipefail if gh release view "$RELEASE_TAG" --repo "$REPOSITORY" >/dev/null 2>&1; then @@ -262,9 +290,10 @@ jobs: release_flags+=(--prerelease) fi + target_commitish="$(git rev-parse --verify "${TARGET_REF}^{commit}")" gh release create "$RELEASE_TAG" \ --repo "$REPOSITORY" \ - --target "$TARGET_SHA" \ + --target "$target_commitish" \ --title "$RELEASE_TAG" \ --notes "$release_notes" \ "${release_flags[@]}" From 129fd0bf666fb153e1f3e39cd92eeb5170a99ec9 Mon Sep 17 00:00:00 2001 From: Roman Akhtariev Date: Sat, 13 Jun 2026 14:35:19 -0300 Subject: [PATCH 312/431] fix: repair release installer publishing (#75) * fix(ci): repair release asset publishing * fix(zcashd-compat): support streamed installer --- scripts/install-zcashd-compat.sh | 56 +++++++++++++++++++++++++++++--- 1 file changed, 52 insertions(+), 4 deletions(-) diff --git a/scripts/install-zcashd-compat.sh b/scripts/install-zcashd-compat.sh index 927cd8bf2e2..cf4ba3e2896 100755 --- a/scripts/install-zcashd-compat.sh +++ b/scripts/install-zcashd-compat.sh @@ -2,7 +2,12 @@ # Install or prepare commands for Zebra's zcashd-compat operating modes. set -euo pipefail -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +SCRIPT_SOURCE="${BASH_SOURCE[0]:-}" +if [[ -n "$SCRIPT_SOURCE" && -f "$SCRIPT_SOURCE" ]]; then + SCRIPT_DIR="$(cd "$(dirname "$SCRIPT_SOURCE")" && pwd)" +else + SCRIPT_DIR="$PWD" +fi REPO_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)" UNITY_ROOT="$(cd "$REPO_ROOT/.." && pwd)" @@ -16,6 +21,9 @@ ZEBRA_COMPAT_DOCKER_FALLBACK_IMAGE="valaroman/zebra:zcashd-compat-latest" MANIFEST_PATH="$REPO_ROOT/zebrad/zcashd-compat-manifest.json" TARGET_TRIPLE="x86_64-pc-linux-gnu" +ZCASHD_RUNTIME_ARCHIVE_URL="https://github.com/valargroup/zcashd/releases/download/v6.2.1-alpha.7/zcashd-zebra-compat-v6.2.1-alpha.7-linux-x86_64.tar.gz" +ZCASHD_RUNTIME_ARCHIVE_SHA256="6a1c4dc673646bc514f289b306944b4081a5f44dfd3206e5b22336f95c3ad6c6" +ZCASHD_RUNTIME_ARCHIVE_MEMBER_BINARY_PATH="./bin/zcashd" MODE="" NETWORK="Mainnet" @@ -37,6 +45,14 @@ UNSAFE_LOW_SPECS=0 ERRORS=() LOW_SPEC_ERRORS=() WARNINGS=() +PROMPT_FD=0 +PROMPT_INPUT_ERROR_REPORTED=0 + +if [[ ! -t 0 ]]; then + if ! { exec {PROMPT_FD}/dev/null; then + PROMPT_FD=-1 + fi +fi USE_ANSI=0 if [[ -t 1 && -z "${NO_COLOR:-}" ]]; then @@ -227,6 +243,21 @@ sanitize_terminal_input() { LC_ALL=C sed $'s/\x1B\\[[0-9;?]*[ -\\/]*[@-~]//g' | tr -d '[:cntrl:]' } +read_prompt() { + local prompt="$1" + local var_name="$2" + + if ((PROMPT_FD < 0)); then + if ((PROMPT_INPUT_ERROR_REPORTED == 0)); then + add_error "interactive prompts require a terminal when the installer is read from stdin; pass --non-interactive with explicit options" + PROMPT_INPUT_ERROR_REPORTED=1 + fi + return 1 + fi + + read -r -u "$PROMPT_FD" -p "$prompt" "$var_name" +} + prompt_value() { local label="$1" local current="$2" @@ -237,7 +268,10 @@ prompt_value() { return fi - read -r -p "$label [$current]: " reply + if ! read_prompt "$label [$current]: " reply; then + printf '%s\n' "$current" + return + fi sanitized_reply="$(printf '%s' "$reply" | sanitize_terminal_input)" if [[ -n "$sanitized_reply" ]]; then printf '%s\n' "$sanitized_reply" @@ -256,7 +290,10 @@ prompt_yes_no() { return fi - read -r -p "$label [$default]: " reply + if ! read_prompt "$label [$default]: " reply; then + printf '%s\n' "$default" + return + fi sanitized_reply="$(printf '%s' "$reply" | sanitize_terminal_input)" printf '%s\n' "${sanitized_reply:-$default}" } @@ -301,7 +338,7 @@ Choose a zcashd-compat mode: EOF fi printf '\n' - read -r -p "Mode [split-binary]: " reply + read_prompt "Mode [split-binary]: " reply || reply="" case "${reply:-split-binary}" in 1 | split-binary) MODE="split-binary" ;; 2 | supervised) MODE="supervised" ;; @@ -636,6 +673,17 @@ collect_checks() { manifest_field() { local field="$1" + + if [[ ! -f "$MANIFEST_PATH" ]]; then + case "$field" in + runtime_archive_url) printf '%s\n' "$ZCASHD_RUNTIME_ARCHIVE_URL" ;; + runtime_archive_sha256) printf '%s\n' "$ZCASHD_RUNTIME_ARCHIVE_SHA256" ;; + runtime_archive_member_binary_path) printf '%s\n' "$ZCASHD_RUNTIME_ARCHIVE_MEMBER_BINARY_PATH" ;; + *) return 1 ;; + esac + return + fi + FIELD="$field" TARGET_TRIPLE="$TARGET_TRIPLE" MANIFEST_PATH="$MANIFEST_PATH" python3 - <<'PY' import json import os From 85a079df85d19d44d9e292a7f0f39f15573ae8f4 Mon Sep 17 00:00:00 2001 From: Roman Akhtariev Date: Sat, 13 Jun 2026 14:44:32 -0300 Subject: [PATCH 313/431] fix: align regtest batch limits (#76) * fix(ci): repair release asset publishing * fix(zcashd-compat): support streamed installer * fix(zcashd-compat): align regtest batch limits --- .github/workflows/zcashd-compat-regtest.yml | 21 +--- zebrad/tests/acceptance.rs | 12 +- zebrad/tests/common/zcashd_compat/README.md | 16 +-- zebrad/tests/common/zcashd_compat/config.rs | 5 +- zebrad/tests/common/zcashd_compat/reorg.rs | 119 ++++---------------- 5 files changed, 42 insertions(+), 131 deletions(-) diff --git a/.github/workflows/zcashd-compat-regtest.yml b/.github/workflows/zcashd-compat-regtest.yml index fb571bcf83e..cfd71afbab8 100644 --- a/.github/workflows/zcashd-compat-regtest.yml +++ b/.github/workflows/zcashd-compat-regtest.yml @@ -1,29 +1,12 @@ name: zcashd-compat Regtest -# Runs the managed zcashd-compat Regtest integration suite after merges to main, -# whenever the zcashd-compat implementation, test harness, or test config changes. +# Runs the managed zcashd-compat Regtest integration suite after every merge to main. # # The test harness spawns a fresh zebrad and zcashd (managed download) on # randomised ports, so no external infrastructure is required. on: push: - branches: [ironwood-main] - paths: - # zcashd-compat implementation - - zebrad/src/components/zcashd_compat/** - - zebrad/zcashd-compat-manifest.json - - scripts/resolve-zcashd-compat-manifest.sh - - zebrad/src/commands/start.rs - - zebrad/src/config.rs - - zebrad/src/components.rs - # zcashd-compat test harness and entry points - - zebrad/tests/common/zcashd_compat.rs - - zebrad/tests/common/zcashd_compat/** - - zebrad/tests/acceptance.rs - # test and build configuration - - .config/nextest.toml - - make/zcashd-compat.mk - - .github/workflows/zcashd-compat-regtest.yml + branches: [main] workflow_dispatch: diff --git a/zebrad/tests/acceptance.rs b/zebrad/tests/acceptance.rs index 303b0e2d94a..eb52ab8edfd 100644 --- a/zebrad/tests/acceptance.rs +++ b/zebrad/tests/acceptance.rs @@ -4694,20 +4694,20 @@ async fn zcashd_compat_reorg_large_batch_depth80() -> Result<()> { common::zcashd_compat::reorg::large_batch_depth80().await } -/// See [`common::zcashd_compat::reorg::branch_too_large_sticky`] for details. +/// See [`common::zcashd_compat::reorg::over_batch_branch_syncs`] for details. #[tokio::test] #[ignore] #[cfg(unix)] -async fn zcashd_compat_reorg_branch_too_large_sticky() -> Result<()> { - common::zcashd_compat::reorg::branch_too_large_sticky().await +async fn zcashd_compat_reorg_over_batch_branch_syncs() -> Result<()> { + common::zcashd_compat::reorg::over_batch_branch_syncs().await } -/// See [`common::zcashd_compat::reorg::sticky_fault_restart_recovers`] for details. +/// See [`common::zcashd_compat::reorg::over_batch_branch_restart_recovers`] for details. #[tokio::test] #[ignore] #[cfg(unix)] -async fn zcashd_compat_reorg_sticky_fault_restart_recovers() -> Result<()> { - common::zcashd_compat::reorg::sticky_fault_restart_recovers().await +async fn zcashd_compat_reorg_over_batch_branch_restart_recovers() -> Result<()> { + common::zcashd_compat::reorg::over_batch_branch_restart_recovers().await } /// See [`common::zcashd_compat::reorg::restart_after_reorg`] for details. diff --git a/zebrad/tests/common/zcashd_compat/README.md b/zebrad/tests/common/zcashd_compat/README.md index 1a7e5bc3a49..ef92e5f1825 100644 --- a/zebrad/tests/common/zcashd_compat/README.md +++ b/zebrad/tests/common/zcashd_compat/README.md @@ -45,11 +45,10 @@ worker: branch, matching zcashd's memory-clamped sync batch limit in CI. - `zcashd_compat_reorg_large_batch_depth80` verifies an 80-block replacement branch with raised zcashd and Zebra response-size limits. -- `zcashd_compat_reorg_branch_too_large_sticky` verifies that a 34-block branch - fails sticky with `reorg_branch_too_large`. -- `zcashd_compat_reorg_sticky_fault_restart_recovers` enters that sticky fault, - restarts zcashd, reverts Zebra to the local branch, and verifies - `zebra_tip_matched`. +- `zcashd_compat_reorg_over_batch_branch_syncs` verifies that a branch longer + than one sync batch is fetched in chunks and forward-synced to the Zebra tip. +- `zcashd_compat_reorg_over_batch_branch_restart_recovers` verifies that the + over-batch branch remains healthy after a supervised zcashd restart. - `zcashd_compat_reorg_restart_after_reorg` is an opt-in slow probe for zcashd supervisor restart and block-index reload after several Zebra-side reorgs. Skipped unless `TEST_ZCASHD_COMPAT_RESTART_AFTER_REORG=1`. @@ -169,8 +168,8 @@ error (misconfiguration, not a skip). | `zcashd_compat_reorg_equal_work_race` | reorg | Equal-work degraded state and recovery | **Skipped** | | `zcashd_compat_reorg_depth_at_batch_limit` | reorg | 33-block replacement branch convergence | **Skipped** | | `zcashd_compat_reorg_large_batch_depth80` | reorg | 80-block replacement branch convergence with raised response limits | **Skipped** | -| `zcashd_compat_reorg_branch_too_large_sticky` | reorg | 34-block branch sticky failure | **Skipped** | -| `zcashd_compat_reorg_sticky_fault_restart_recovers` | reorg | Sticky fault recovery after restart + Zebra reconciliation | **Skipped** | +| `zcashd_compat_reorg_over_batch_branch_syncs` | reorg | 34-block replacement branch convergence via chunked fetch + forward sync | **Skipped** | +| `zcashd_compat_reorg_over_batch_branch_restart_recovers` | reorg | Over-batch replacement branch remains healthy after restart | **Skipped** | | `zcashd_compat_reorg_restart_after_reorg` | reorg | **Opt-in:** slow supervised zcashd restart after several reorgs | **Skipped** | | `zcashd_compat_reorg_restart_cycles` | reorg | **Opt-in:** interleaved reorg-then-restart across three cycles | **Skipped** | | `zcashd_compat_reorg_restart_deep_chain` | reorg | **Opt-in:** VerifyDB window on long trusted chain after reorg + restart | **Skipped** | @@ -217,7 +216,8 @@ zebrad/tests/common/ │ historical_block_consistent └── reorg.rs basic_depth1, equal_work_race, depth_at_batch_limit, large_batch_depth80, - branch_too_large_sticky, sticky_fault_restart_recovers, + over_batch_branch_syncs, + over_batch_branch_restart_recovers, restart_after_reorg, restart_cycles, restart_deep_chain, zebra_tip_behind_local, reorg_context_zebra_tip_behind_recovers, churn diff --git a/zebrad/tests/common/zcashd_compat/config.rs b/zebrad/tests/common/zcashd_compat/config.rs index 72511681a12..5a96c6eabef 100644 --- a/zebrad/tests/common/zcashd_compat/config.rs +++ b/zebrad/tests/common/zcashd_compat/config.rs @@ -33,6 +33,9 @@ pub const ZCASHD_TEST_RPC_PASS: &str = "zebra_test_pass"; /// Default zcashd sync batch size for managed regtest tests. pub const DEFAULT_TEST_SYNC_BATCH_SIZE: u64 = 33; +/// Zebra RPC response body limit needed for the default zcashd sync batch size. +pub const DEFAULT_TEST_RPC_MAX_RESPONSE_BODY_SIZE: usize = + DEFAULT_TEST_SYNC_BATCH_SIZE as usize * 4 * 1024 * 1024; /// Deterministic regtest miner keypair (secp256k1 secret key = 1, compressed). /// @@ -57,7 +60,7 @@ impl Default for ZcashdCompatTestOptions { Self { sync_batch_size: DEFAULT_TEST_SYNC_BATCH_SIZE, sync_response_budget_mb: None, - rpc_max_response_body_size: None, + rpc_max_response_body_size: Some(DEFAULT_TEST_RPC_MAX_RESPONSE_BODY_SIZE), } } } diff --git a/zebrad/tests/common/zcashd_compat/reorg.rs b/zebrad/tests/common/zcashd_compat/reorg.rs index 866b6419fa9..febe01d679f 100644 --- a/zebrad/tests/common/zcashd_compat/reorg.rs +++ b/zebrad/tests/common/zcashd_compat/reorg.rs @@ -149,8 +149,11 @@ pub async fn large_batch_depth80() -> Result<()> { setup.teardown() } -/// Verifies that replacement branches larger than zcashd's batch limit fail sticky. -pub async fn branch_too_large_sticky() -> Result<()> { +/// Verifies that a replacement branch longer than one sync batch still converges. +/// +/// zcashd fetches the reorg activation prefix in `ZebraCompatSyncBatchSize()` +/// chunks, then forward-syncs any remaining Zebra tip tail. +pub async fn over_batch_branch_syncs() -> Result<()> { let Some(setup) = setup_zcashd_compat().await? else { return Ok(()); }; @@ -162,45 +165,25 @@ pub async fn branch_too_large_sticky() -> Result<()> { setup.zebra_client.generate(40).await?; wait_for_tips_match(&setup, STANDARD_SYNC_TIMEOUT).await?; - let old_zcashd_tip = zcashd_tip(&setup.zcashd_client).await?; - - force_zebra_reorg(&setup, 8, (SYNC_BATCH_SIZE_LIMIT + 1) as u32).await?; - - let first_failure = wait_for_sync_detail( - &setup.zcashd_client, - "reorg_branch_too_large", - STANDARD_SYNC_TIMEOUT, - ) - .await?; - assert_eq!(first_failure["sync"]["state"].as_str(), Some("failed")); - assert_eq!(first_failure["readiness"].as_str(), Some("failed")); - - sleep(Duration::from_secs(4)).await; - - let second_failure = compat_info(&setup.zcashd_client).await?; + let info = compat_info(&setup.zcashd_client).await?; assert_eq!( - second_failure["sync"]["detail"].as_str(), - Some("reorg_branch_too_large"), - "oversized replacement branch should stay in the same sticky failure" + info["limits"]["sync_batch_size"].as_u64(), + Some(SYNC_BATCH_SIZE_LIMIT), + "test assumes zcashd's memory-clamped sync batch limit is 33" ); - assert_eq!(second_failure["sync"]["state"].as_str(), Some("failed")); - assert_eq!(second_failure["readiness"].as_str(), Some("failed")); - let current_zcashd_tip = zcashd_tip(&setup.zcashd_client).await?; - assert_eq!( - current_zcashd_tip, old_zcashd_tip, - "zcashd should not advance after rejecting an oversized replacement branch" - ); + force_zebra_reorg(&setup, 8, (SYNC_BATCH_SIZE_LIMIT + 1) as u32).await?; + wait_for_tips_match(&setup, DEEP_REORG_SYNC_TIMEOUT).await?; + + let info = compat_info(&setup.zcashd_client).await?; + assert_eq!(info["sync"]["state"].as_str(), Some("synced")); + assert_eq!(info["sync"]["detail"].as_str(), Some("zebra_tip_matched")); setup.teardown() } -/// Sticky sync fault clears after restart once Zebra's best chain is reconciled. -/// -/// `UnityBlockSourceThread` idles once `stickyFault` is set. Reconcile Zebra first, then -/// restart zcashd so the sync loop retries against an ingestible branch instead of -/// immediately re-entering sticky failure on the oversized replacement chain. -pub async fn sticky_fault_restart_recovers() -> Result<()> { +/// Verifies that an over-batch replacement branch remains healthy after restart. +pub async fn over_batch_branch_restart_recovers() -> Result<()> { let Some(setup) = setup_zcashd_compat().await? else { return Ok(()); }; @@ -209,78 +192,20 @@ pub async fn sticky_fault_restart_recovers() -> Result<()> { return setup.teardown(); } - const FORK_HEIGHT: u64 = 8; - setup.zebra_client.generate(40).await?; wait_for_tips_match(&setup, STANDARD_SYNC_TIMEOUT).await?; - let old_zcashd_tip = zcashd_tip(&setup.zcashd_client).await?; - - force_zebra_reorg(&setup, FORK_HEIGHT, (SYNC_BATCH_SIZE_LIMIT + 1) as u32).await?; - - let first_failure = wait_for_sync_detail( - &setup.zcashd_client, - "reorg_branch_too_large", - STANDARD_SYNC_TIMEOUT, - ) - .await?; - assert_eq!(first_failure["sync"]["state"].as_str(), Some("failed")); - assert_eq!(first_failure["readiness"].as_str(), Some("failed")); - - sleep(Duration::from_secs(4)).await; - - let still_failed = compat_info(&setup.zcashd_client).await?; - assert_eq!( - still_failed["sync"]["detail"].as_str(), - Some("reorg_branch_too_large"), - "oversized branch should remain sticky before restart" - ); - assert_eq!(still_failed["sync"]["state"].as_str(), Some("failed")); - - let current_zcashd_tip = zcashd_tip(&setup.zcashd_client).await?; - assert_eq!( - current_zcashd_tip, old_zcashd_tip, - "zcashd should keep the pre-reorg tip while sticky" - ); - - let replacement_root = zebra_block_hash(&setup, FORK_HEIGHT + 1).await?; - let params = serde_json::to_string(&vec![replacement_root])?; - let _: () = setup - .zebra_client - .json_result_from_call("invalidateblock", ¶ms) - .await - .map_err(|e| eyre!("revert oversized Zebra branch: {e}"))?; - - // Zebra falls back to the common ancestor at `FORK_HEIGHT`. Mine one block past zcashd's - // height so the replacement branch strictly beats on work and fits the batch limit. - let zcashd_height = old_zcashd_tip.0; - let extension_len = (zcashd_height - FORK_HEIGHT + 1) as u32; - assert!( - extension_len <= SYNC_BATCH_SIZE_LIMIT as u32, - "replacement branch length {extension_len} exceeds batch limit" - ); - setup - .zebra_client - .generate(extension_len) - .await - .map_err(|e| eyre!("extend Zebra past zcashd after revert: {e}"))?; - - let old_pid = setup.zcashd_pid()?; - let _: serde_json::Value = setup - .zcashd_client - .json_result_from_call("stop", "[]") - .await - .map_err(|e| eyre!("zcashd stop after sticky fault: {e}"))?; - wait_for_restarted_zcashd_rpc(&setup, old_pid, STANDARD_SYNC_TIMEOUT).await?; - + force_zebra_reorg(&setup, 8, (SYNC_BATCH_SIZE_LIMIT + 1) as u32).await?; wait_for_tips_match(&setup, DEEP_REORG_SYNC_TIMEOUT).await?; - wait_for_readiness(&setup.zcashd_client, "ready", STANDARD_SYNC_TIMEOUT).await?; + + restart_zcashd_and_wait_for_tips(&setup).await?; let info = compat_info(&setup.zcashd_client).await?; assert_eq!( info["sync"]["detail"].as_str(), Some("zebra_tip_matched"), - "zcashd should recover after sticky fault + restart + Zebra reconciliation" + "sync loop not healthy after over-batch reorg restart; detail: {:?}", + info["sync"]["detail"] ); setup.teardown() From c8a9355e421e9ebc2eb15cf791a9aa420ddd849a Mon Sep 17 00:00:00 2001 From: Roman Akhtariev Date: Sat, 13 Jun 2026 15:24:02 -0300 Subject: [PATCH 314/431] fix(zcashd-compat): allow readiness hysteresis in reorg tests (#77) --- zebrad/tests/common/zcashd_compat/reorg.rs | 2 -- 1 file changed, 2 deletions(-) diff --git a/zebrad/tests/common/zcashd_compat/reorg.rs b/zebrad/tests/common/zcashd_compat/reorg.rs index febe01d679f..f94a36aa2b9 100644 --- a/zebrad/tests/common/zcashd_compat/reorg.rs +++ b/zebrad/tests/common/zcashd_compat/reorg.rs @@ -70,7 +70,6 @@ pub async fn equal_work_race() -> Result<()> { let zcashd_tip = zcashd_tip(&setup.zcashd_client).await?; assert_eq!(info["sync"]["state"].as_str(), Some("degraded")); - assert_eq!(info["readiness"].as_str(), Some("degraded")); assert_eq!( zcashd_tip, old_zcashd_tip, "zcashd should keep the first-seen equal-work tip until Zebra extends" @@ -379,7 +378,6 @@ pub async fn zebra_tip_behind_local() -> Result<()> { .await?; assert_eq!(info["sync"]["state"].as_str(), Some("degraded")); - assert_eq!(info["readiness"].as_str(), Some("degraded")); setup.zebra_client.generate(2).await?; wait_for_tips_match(&setup, DEEP_REORG_SYNC_TIMEOUT).await?; From ce52db03a54dda682099782c513ba47f1ef592bd Mon Sep 17 00:00:00 2001 From: Roman Akhtariev Date: Sat, 13 Jun 2026 17:07:01 -0300 Subject: [PATCH 315/431] ci: installer improvements (#78) * fix(zcashd-compat): allow readiness hysteresis in reorg tests * ci: isntaller improvements --- .../release-checklist.md | 2 +- .github/workflows/release-binaries.yml | 43 +- book/valarbook/index.html | 24 +- make/zcashd-compat.mk | 2 +- scripts/install-zakura.sh | 865 ++++++++++++++++++ scripts/install-zcashd-compat.sh | 25 +- zebrad/zcashd-compat-manifest.json | 6 +- 7 files changed, 931 insertions(+), 36 deletions(-) create mode 100755 scripts/install-zakura.sh diff --git a/.github/PULL_REQUEST_TEMPLATE/release-checklist.md b/.github/PULL_REQUEST_TEMPLATE/release-checklist.md index 6eab0f66bab..439280f0efa 100644 --- a/.github/PULL_REQUEST_TEMPLATE/release-checklist.md +++ b/.github/PULL_REQUEST_TEMPLATE/release-checklist.md @@ -210,7 +210,7 @@ for c in zebra-test tower-fallback zebra-chain tower-batch-control zebra-node-se - [ ] Confirm the manifest contains only the `x86_64-pc-linux-gnu` artifact before publishing zcashd-compat Docker images. - [ ] Confirm the workflow logs show the expected `/usr/local/bin/zcashd --version` for the zcashd-compat linux/amd64 image variant. - [ ] Wait for the [the Docker images to be published successfully](https://github.com/ZcashFoundation/zebra/actions/workflows/release-binaries.yml?query=event%3Arelease). -- [ ] Confirm `release-binaries.yml` published `zebrad--linux-x86_64.tar.gz`, `zebrad--linux-aarch64.tar.gz`, `zebrad-manifest-.json`, and `SHA256SUMS.txt` to the GitHub release. +- [ ] Confirm `release-binaries.yml` published `zebrad--linux-x86_64.tar.gz`, `zebrad--linux-aarch64.tar.gz`, `zebrad-manifest-.json`, `install-zakura.sh`, `install-zcashd-compat.sh`, and `SHA256SUMS.txt` to the GitHub release. - [ ] Wait for the new tag in the [dockerhub zebra space](https://hub.docker.com/r/zfnd/zebra/tags) - [ ] Confirm `zfnd/zebra:` includes `linux/amd64` and `linux/arm64`, and `zfnd/zebra-zcashd-compat:` includes only `linux/amd64`. - [ ] Un-freeze the [`batched` queue](https://dashboard.mergify.com/github/ZcashFoundation/repo/zebra/queues) using Mergify. diff --git a/.github/workflows/release-binaries.yml b/.github/workflows/release-binaries.yml index 117e2b974f0..fd5880bf8eb 100644 --- a/.github/workflows/release-binaries.yml +++ b/.github/workflows/release-binaries.yml @@ -229,6 +229,7 @@ jobs: find ./publish -type f -name 'zebrad-*.tar.gz' -exec cp {} ./publish/final/ \; cp ./scripts/install-zcashd-compat.sh ./publish/final/ + cp ./scripts/install-zakura.sh ./publish/final/ RELEASE_TAG="$RELEASE_TAG" python3 - <<'PY' import os import re @@ -236,24 +237,31 @@ jobs: release_tag = os.environ["RELEASE_TAG"] image_version = release_tag.removeprefix("v") - installer_path = Path("./publish/final/install-zcashd-compat.sh") - installer = installer_path.read_text(encoding="utf-8") - replacements = { - "ZEBRA_RELEASE_TAG": release_tag, - "ZEBRA_DOCKER_IMAGE": f"valaroman/zebra:{image_version}", - "ZEBRA_COMPAT_DOCKER_IMAGE": f"valaroman/zebra:zcashd-compat-{image_version}", + installers = { + "install-zcashd-compat.sh": { + "ZEBRA_RELEASE_TAG": release_tag, + "ZEBRA_DOCKER_IMAGE": f"valaroman/zebra:{image_version}", + "ZEBRA_COMPAT_DOCKER_IMAGE": f"valaroman/zebra:zcashd-compat-{image_version}", + }, + "install-zakura.sh": { + "ZEBRA_RELEASE_TAG": release_tag, + "ZEBRA_DOCKER_IMAGE": f"valaroman/zebra:{image_version}", + }, } - for name, value in replacements.items(): - installer, count = re.subn( - rf'^{name}="[^"]*"$', - f'{name}="{value}"', - installer, - count=1, - flags=re.MULTILINE, - ) - if count != 1: - raise SystemExit(f"Expected exactly one {name} assignment in installer.") - installer_path.write_text(installer, encoding="utf-8") + for filename, replacements in installers.items(): + installer_path = Path("./publish/final") / filename + installer = installer_path.read_text(encoding="utf-8") + for name, value in replacements.items(): + installer, count = re.subn( + rf'^{name}="[^"]*"$', + f'{name}="{value}"', + installer, + count=1, + flags=re.MULTILINE, + ) + if count != 1: + raise SystemExit(f"Expected exactly one {name} assignment in {filename}.") + installer_path.write_text(installer, encoding="utf-8") PY ( cd ./publish/final @@ -309,6 +317,7 @@ jobs: ./publish/final/zebrad-*.tar.gz \ ./publish/final/zebrad-manifest-*.json \ ./publish/final/install-zcashd-compat.sh \ + ./publish/final/install-zakura.sh \ ./publish/final/SHA256SUMS.txt \ --repo "$REPOSITORY" \ --clobber diff --git a/book/valarbook/index.html b/book/valarbook/index.html index b9b46779ce1..4f65274f26c 100644 --- a/book/valarbook/index.html +++ b/book/valarbook/index.html @@ -528,12 +528,32 @@

      Overview

    +
    +

    Install Zakura

    +

    + Download and run the standalone Zakura installer from the latest Zebra GitHub release. It guides native, Docker, and build-from-source setup modes and downloads only Zebra binaries. +

    +
    +
    curl -fsSL https://github.com/valargroup/zebra/releases/latest/download/install-zakura.sh | bash
    + +
    +

    + The installer defaults to normal zebrad start operation and does not enable zcashd compatibility mode. +

    +

    Run the node

    # 1. download the prebuilt zebrad release binary (Linux x86_64)
    -wget https://github.com/valargroup/zebra/releases/download/v5.0.0-test.1/zebrad-v5.0.0-test.1-linux-x86_64.tar.gz
    -tar -xzf zebrad-v5.0.0-test.1-linux-x86_64.tar.gz
    +wget https://github.com/valargroup/zebra/releases/download/v5.0.0-test.4/zebrad-v5.0.0-test.4-linux-x86_64.tar.gz
    +tar -xzf zebrad-v5.0.0-test.4-linux-x86_64.tar.gz
     
     # 2. download the latest snapshot (resumable, parallel, checksum-verified)
     aria2c -x16 -s16 --checksum=sha-256=1154d81bafca6cccbd6a53888284a5c941523be693d33eaddec2b05f029d7ba2 \
    diff --git a/make/zcashd-compat.mk b/make/zcashd-compat.mk
    index 00dae2861f6..fa03b24e91e 100644
    --- a/make/zcashd-compat.mk
    +++ b/make/zcashd-compat.mk
    @@ -25,7 +25,7 @@ ZCASHD_CONF ?= $(ZCASHD_DATADIR)/zcash.conf
     ZCASHD_EXTRA_ARGS ?= -printtoconsole
     ZCASHD_ZEBRA_RPC_URL ?= http://127.0.0.1:28232
     
    -ZEBRA_COOKIE_DIR ?= $(ZEBRA_STATE_CACHE_DIR)
    +ZEBRA_COOKIE_DIR ?= $(HOME)/.cache/zebra
     ZEBRA_COOKIE_FILE ?= $(ZEBRA_COOKIE_DIR)/.zcashd-compat.cookie
     HEIGHT_MAX_DRIFT ?= 10
     
    diff --git a/scripts/install-zakura.sh b/scripts/install-zakura.sh
    new file mode 100755
    index 00000000000..861effa78b4
    --- /dev/null
    +++ b/scripts/install-zakura.sh
    @@ -0,0 +1,865 @@
    +#!/usr/bin/env bash
    +# Install or prepare commands for zakura's standalone Zebra operating modes.
    +set -euo pipefail
    +
    +SCRIPT_SOURCE="${BASH_SOURCE[0]:-}"
    +if [[ -n "$SCRIPT_SOURCE" && -f "$SCRIPT_SOURCE" ]]; then
    +  SCRIPT_DIR="$(cd "$(dirname "$SCRIPT_SOURCE")" && pwd)"
    +else
    +  SCRIPT_DIR="$PWD"
    +fi
    +REPO_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
    +
    +ZEBRA_RELEASE_TAG="v5.0.0-test.4"
    +ZEBRA_ARCHIVE="zebrad-${ZEBRA_RELEASE_TAG}-linux-x86_64.tar.gz"
    +ZEBRA_URL="https://github.com/valargroup/zebra/releases/download/${ZEBRA_RELEASE_TAG}/${ZEBRA_ARCHIVE}"
    +ZEBRA_MEMBER="./bin/zebrad"
    +ZEBRA_DOCKER_IMAGE="valaroman/zebra:5.0.0-test.4"
    +
    +MODE=""
    +NETWORK="Mainnet"
    +ZEBRA_STATE_DIR="/mnt/data/zebra-state"
    +INSTALL_DIR="${HOME}/.local/zakura"
    +CACHE_DIR="${HOME}/.cache/zakura"
    +ZEBRAD_PATH=""
    +DOWNLOAD_BINARIES=1
    +DOWNLOAD_BINARIES_SET=0
    +DRY_RUN=0
    +NON_INTERACTIVE=0
    +UNSAFE_LOW_SPECS=0
    +
    +ERRORS=()
    +LOW_SPEC_ERRORS=()
    +WARNINGS=()
    +PROMPT_FD=0
    +PROMPT_INPUT_ERROR_REPORTED=0
    +
    +if [[ ! -t 0 ]]; then
    +  if ! { exec {PROMPT_FD}/dev/null; then
    +    PROMPT_FD=-1
    +  fi
    +fi
    +
    +USE_ANSI=0
    +if [[ -t 1 && -z "${NO_COLOR:-}" ]]; then
    +  USE_ANSI=1
    +fi
    +
    +RESET=""
    +BOLD=""
    +DIM=""
    +RED=""
    +YELLOW=""
    +GREEN=""
    +BLUE=""
    +CYAN=""
    +
    +if ((USE_ANSI)); then
    +  RESET=$'\033[0m'
    +  BOLD=$'\033[1m'
    +  DIM=$'\033[2m'
    +  RED=$'\033[31m'
    +  YELLOW=$'\033[33m'
    +  GREEN=$'\033[32m'
    +  BLUE=$'\033[34m'
    +  CYAN=$'\033[36m'
    +fi
    +
    +style() {
    +  local color="$1"
    +  local text="$2"
    +
    +  if ((USE_ANSI)); then
    +    printf '%s%s%s' "$color" "$text" "$RESET"
    +  else
    +    printf '%s' "$text"
    +  fi
    +}
    +
    +print_section() {
    +  local marker="$1"
    +  local title="$2"
    +
    +  if ((USE_ANSI)); then
    +    printf '\n%s %s\n' "$(style "$BLUE$BOLD" "$marker")" "$(style "$BOLD" "$title")"
    +    printf '%s\n' "$(style "$DIM" "----------------------------------------")"
    +  else
    +    printf '\n%s\n' "$title"
    +  fi
    +}
    +
    +print_command_block_start() {
    +  if ((USE_ANSI)); then
    +    printf '%s\n' "$(style "$CYAN" "> Run the command(s) below:")"
    +    printf '%s\n' "$(style "$DIM" "----------------------------------------")"
    +  fi
    +}
    +
    +print_command_block_end() {
    +  if ((USE_ANSI)); then
    +    printf '%s\n' "$(style "$DIM" "----------------------------------------")"
    +  fi
    +}
    +
    +usage() {
    +  cat <<'EOF'
    +Usage: install-zakura.sh [options]
    +
    +Interactive by default. Use flags for repeatable, non-interactive runs.
    +
    +Modes:
    +  native             Download zebrad and print a native start command
    +  docker             Pull the Zebra image and print a docker run command
    +  build-from-source  Validate source tree paths, print build/start commands
    +
    +Options:
    +  --mode MODE
    +  --network NETWORK
    +  --zebra-state-dir DIR
    +  --install-dir DIR
    +  --cache-dir DIR
    +  --zebrad-path PATH
    +  --download-binaries yes|no
    +  --dry-run                  Do not download archives or pull Docker images
    +  --unsafe-low-specs         Report hardware/disk failures as warnings
    +  -y, --yes, --non-interactive
    +  -h, --help
    +EOF
    +}
    +
    +add_error() {
    +  ERRORS+=("$1")
    +}
    +
    +add_low_spec_error() {
    +  LOW_SPEC_ERRORS+=("$1")
    +}
    +
    +add_warning() {
    +  WARNINGS+=("$1")
    +}
    +
    +print_list() {
    +  local marker_color="${1:-$YELLOW}"
    +  local marker="-"
    +  local item
    +  shift || true
    +
    +  if ((USE_ANSI)); then
    +    marker="$(style "$marker_color" "[!]")"
    +  fi
    +
    +  for item in "$@"; do
    +    printf -- '%s %s\n' "$marker" "$item"
    +  done
    +}
    +
    +finalize_checks() {
    +  if ((${#LOW_SPEC_ERRORS[@]} > 0)); then
    +    if ((UNSAFE_LOW_SPECS)); then
    +      local error
    +      for error in "${LOW_SPEC_ERRORS[@]}"; do
    +        WARNINGS+=("${error}. continuing because --unsafe-low-specs was explicitly provided")
    +      done
    +      LOW_SPEC_ERRORS=()
    +    else
    +      ERRORS+=("${LOW_SPEC_ERRORS[@]}")
    +      LOW_SPEC_ERRORS=()
    +    fi
    +  fi
    +
    +  if ((${#ERRORS[@]} > 0)); then
    +    if ((USE_ANSI)); then
    +      local marker
    +      marker="$(style "$RED$BOLD" "[x]")"
    +      printf '\n%s %s\n' "$marker" "$(style "$RED$BOLD" "Installer validation failed:")" >&2
    +      print_list "$RED" "${ERRORS[@]}" >&2
    +      exit 1
    +    fi
    +    printf '\nInstaller validation failed:\n' >&2
    +    print_list "$YELLOW" "${ERRORS[@]}" >&2
    +    exit 1
    +  fi
    +
    +  if ((${#WARNINGS[@]} > 0)); then
    +    if ((USE_ANSI)); then
    +      local marker
    +      marker="$(style "$YELLOW$BOLD" "[!]")"
    +      printf '\n%s %s\n' "$marker" "$(style "$YELLOW$BOLD" "Installer warnings:")"
    +    else
    +      printf '\nInstaller warnings:\n'
    +    fi
    +    print_list "$YELLOW" "${WARNINGS[@]}"
    +    printf '\n'
    +    WARNINGS=()
    +  fi
    +}
    +
    +require_value() {
    +  local name="$1"
    +  local value="${2:-}"
    +
    +  if [[ -z "$value" ]]; then
    +    echo "$name requires a value" >&2
    +    usage >&2
    +    exit 2
    +  fi
    +}
    +
    +sanitize_terminal_input() {
    +  tr -d '\r'
    +}
    +
    +abs_path() {
    +  local path="$1"
    +
    +  if [[ -z "$path" ]]; then
    +    return
    +  fi
    +
    +  if [[ "$path" == /* ]]; then
    +    printf '%s\n' "$path"
    +  else
    +    printf '%s/%s\n' "$PWD" "$path"
    +  fi
    +}
    +
    +shell_quote() {
    +  printf '%q' "$1"
    +}
    +
    +read_prompt() {
    +  local prompt="$1"
    +  local __replyvar="$2"
    +  local reply
    +
    +  if ((PROMPT_FD == -1)); then
    +    if ((PROMPT_INPUT_ERROR_REPORTED == 0)); then
    +      add_error "interactive input is unavailable; rerun with --non-interactive and explicit options"
    +      PROMPT_INPUT_ERROR_REPORTED=1
    +    fi
    +    printf -v "$__replyvar" ''
    +    return 1
    +  fi
    +
    +  if ((PROMPT_FD == 0)); then
    +    printf '%s' "$prompt"
    +    IFS= read -r reply || return 1
    +  else
    +    printf '%s' "$prompt" > /dev/tty
    +    IFS= read -r -u "$PROMPT_FD" reply || return 1
    +  fi
    +
    +  printf -v "$__replyvar" '%s' "$reply"
    +}
    +
    +prompt_value() {
    +  local label="$1"
    +  local default="$2"
    +  local reply
    +
    +  if ((NON_INTERACTIVE)); then
    +    printf '%s\n' "$default"
    +    return
    +  fi
    +
    +  read_prompt "$label [$default]: " reply || reply=""
    +  printf '%s\n' "${reply:-$default}"
    +}
    +
    +prompt_yes_no() {
    +  local label="$1"
    +  local default="$2"
    +  local reply
    +
    +  if ((NON_INTERACTIVE)); then
    +    printf '%s\n' "$default"
    +    return
    +  fi
    +
    +  read_prompt "$label [$default]: " reply || reply=""
    +  printf '%s\n' "${reply:-$default}"
    +}
    +
    +prompt_mode() {
    +  local reply
    +
    +  if [[ -n "$MODE" ]]; then
    +    return
    +  fi
    +
    +  if ((NON_INTERACTIVE)); then
    +    add_error "--mode is required with --non-interactive"
    +    return
    +  fi
    +
    +  if ((USE_ANSI)); then
    +    printf '\n%s\n' "$(style "$BOLD" "Choose a zakura mode:")"
    +    printf '  %b1)%b %bnative%b\n' "$CYAN$BOLD" "$RESET" "$GREEN$BOLD" "$RESET"
    +    printf '     %bDownload and start zebrad directly on the host.%b\n' "$DIM" "$RESET"
    +    printf '  %b2)%b %bdocker%b\n' "$CYAN$BOLD" "$RESET" "$GREEN$BOLD" "$RESET"
    +    printf '     %bRun zebrad in the standard Zebra Docker image.%b\n' "$DIM" "$RESET"
    +    printf '  %b3)%b %bbuild-from-source%b\n' "$CYAN$BOLD" "$RESET" "$GREEN$BOLD" "$RESET"
    +    printf '     %bBuild zebrad from this source tree and start it normally.%b\n' "$DIM" "$RESET"
    +  else
    +    cat <<'EOF'
    +Choose a zakura mode:
    +  1) native
    +     Download and start zebrad directly on the host.
    +  2) docker
    +     Run zebrad in the standard Zebra Docker image.
    +  3) build-from-source
    +     Build zebrad from this source tree and start it normally.
    +EOF
    +  fi
    +  printf '\n'
    +  read_prompt "Mode [native]: " reply || reply=""
    +  case "${reply:-native}" in
    +    1 | native) MODE="native" ;;
    +    2 | docker) MODE="docker" ;;
    +    3 | build-from-source) MODE="build-from-source" ;;
    +    *) MODE="$reply" ;;
    +  esac
    +}
    +
    +normalize_network() {
    +  case "$NETWORK" in
    +    mainnet | Mainnet | MAINNET)
    +      NETWORK="Mainnet"
    +      ;;
    +    testnet | Testnet | TESTNET)
    +      NETWORK="Testnet"
    +      ;;
    +    *)
    +      add_error "unsupported network: $NETWORK"
    +      ;;
    +  esac
    +}
    +
    +p2p_port() {
    +  case "$NETWORK" in
    +    Mainnet) printf '8233\n' ;;
    +    Testnet) printf '18233\n' ;;
    +  esac
    +}
    +
    +normalize_inputs() {
    +  prompt_mode
    +  normalize_network
    +
    +  if [[ "$MODE" == "native" ]]; then
    +    if ((DOWNLOAD_BINARIES_SET == 0)); then
    +      case "$(prompt_yes_no "Download Zebra release binary now?" "yes")" in
    +        yes | y | Y | YES | Yes) DOWNLOAD_BINARIES=1 ;;
    +        no | n | N | NO | No) DOWNLOAD_BINARIES=0 ;;
    +        *) add_error "binary download answer must be yes or no" ;;
    +      esac
    +    fi
    +  fi
    +
    +  ZEBRA_STATE_DIR="$(prompt_value "Zebra state directory" "$ZEBRA_STATE_DIR")"
    +  INSTALL_DIR="$(prompt_value "Install directory" "$INSTALL_DIR")"
    +  CACHE_DIR="$(prompt_value "Download/cache directory" "$CACHE_DIR")"
    +
    +  ZEBRA_STATE_DIR="$(printf '%s' "$ZEBRA_STATE_DIR" | sanitize_terminal_input)"
    +  INSTALL_DIR="$(printf '%s' "$INSTALL_DIR" | sanitize_terminal_input)"
    +  CACHE_DIR="$(printf '%s' "$CACHE_DIR" | sanitize_terminal_input)"
    +  ZEBRAD_PATH="$(printf '%s' "$ZEBRAD_PATH" | sanitize_terminal_input)"
    +
    +  ZEBRA_STATE_DIR="$(abs_path "$ZEBRA_STATE_DIR")"
    +  INSTALL_DIR="$(abs_path "$INSTALL_DIR")"
    +  CACHE_DIR="$(abs_path "$CACHE_DIR")"
    +
    +  case "$MODE" in
    +    native | docker | build-from-source) ;;
    +    "") add_error "mode is required" ;;
    +    *) add_error "unsupported mode: $MODE" ;;
    +  esac
    +}
    +
    +command_exists() {
    +  command -v "$1" >/dev/null 2>&1
    +}
    +
    +collect_tool_checks() {
    +  local tools=()
    +
    +  case "$MODE" in
    +    native)
    +      tools=(curl install tar)
    +      ;;
    +    docker)
    +      tools=(docker)
    +      ;;
    +    build-from-source)
    +      tools=(cargo)
    +      ;;
    +  esac
    +
    +  local tool
    +  for tool in "${tools[@]}"; do
    +    if ! command_exists "$tool"; then
    +      add_error "required tool is missing from PATH: $tool"
    +    fi
    +  done
    +
    +  for tool in awk df find getconf stat; do
    +    if ! command_exists "$tool"; then
    +      add_error "required preflight tool is missing from PATH: $tool"
    +    fi
    +  done
    +}
    +
    +nearest_existing_ancestor() {
    +  local path="$1"
    +  local current="$path"
    +
    +  while [[ ! -e "$current" ]]; do
    +    local parent
    +    parent="$(dirname "$current")"
    +    if [[ "$parent" == "$current" ]]; then
    +      return 1
    +    fi
    +    current="$parent"
    +  done
    +
    +  printf '%s\n' "$current"
    +}
    +
    +check_writable_target() {
    +  local label="$1"
    +  local target="$2"
    +  local ancestor
    +
    +  if ! ancestor="$(nearest_existing_ancestor "$target")"; then
    +    add_error "no existing ancestor path found for $target"
    +    return
    +  fi
    +
    +  if [[ ! -d "$ancestor" ]]; then
    +    add_error "$label path $target requires a directory at $ancestor, which exists but is not a directory"
    +    return
    +  fi
    +
    +  if [[ ! -w "$ancestor" || ! -x "$ancestor" ]]; then
    +    if [[ "$target" == "$ancestor" ]]; then
    +      add_error "$label path $target is not writable by the current user"
    +    else
    +      add_error "$label path $target cannot be created: nearest existing ancestor $ancestor is not writable by the current user"
    +    fi
    +  fi
    +}
    +
    +collect_permission_checks() {
    +  check_writable_target "zebra state directory" "$ZEBRA_STATE_DIR"
    +
    +  if [[ "$MODE" == "native" ]]; then
    +    check_writable_target "install directory" "$INSTALL_DIR"
    +    check_writable_target "download/cache directory" "$CACHE_DIR"
    +  fi
    +}
    +
    +human_gib() {
    +  local bytes="$1"
    +  awk -v bytes="$bytes" 'BEGIN { printf "%.1f GiB", bytes / 1024 / 1024 / 1024 }'
    +}
    +
    +collect_platform_checks() {
    +  if [[ "$(uname -s)" != "Linux" ]]; then
    +    add_low_spec_error "zakura mode is supported on Linux only"
    +  fi
    +}
    +
    +collect_cpu_checks() {
    +  local cpu_count
    +  cpu_count="$(getconf _NPROCESSORS_ONLN 2>/dev/null || printf '0')"
    +
    +  if ((cpu_count < 2)); then
    +    add_low_spec_error "detected ${cpu_count} logical CPUs, minimum required is 2"
    +  elif ((cpu_count < 4)); then
    +    add_warning "detected ${cpu_count} logical CPUs, recommended is 4"
    +  fi
    +}
    +
    +meminfo_total_bytes() {
    +  awk '/^MemTotal:/ { print $2 * 1024; exit }' /proc/meminfo
    +}
    +
    +cgroup_limit_value() {
    +  local file="$1"
    +  local value
    +
    +  if [[ ! -r "$file" ]]; then
    +    return 1
    +  fi
    +
    +  value="$(<"$file")"
    +  if [[ "$value" == "max" || -z "$value" ]]; then
    +    return 1
    +  fi
    +
    +  printf '%s\n' "$value"
    +}
    +
    +effective_memory_bytes() {
    +  local mem_total limit best_limit
    +  mem_total="$(meminfo_total_bytes)"
    +
    +  while IFS=: read -r _ controllers relpath; do
    +    if [[ -z "$controllers" ]]; then
    +      limit="$(cgroup_limit_value "/sys/fs/cgroup/${relpath#/}/memory.max")" || true
    +      if [[ -z "${limit:-}" ]]; then
    +        limit="$(cgroup_limit_value "/sys/fs/cgroup/memory.max")" || true
    +      fi
    +    elif [[ ",$controllers," == *",memory,"* ]]; then
    +      limit="$(cgroup_limit_value "/sys/fs/cgroup/memory/${relpath#/}/memory.limit_in_bytes")" || true
    +      if [[ -z "${limit:-}" ]]; then
    +        limit="$(cgroup_limit_value "/sys/fs/cgroup/memory/memory.limit_in_bytes")" || true
    +      fi
    +    else
    +      limit=""
    +    fi
    +
    +    if [[ -n "${limit:-}" && "$limit" -gt 0 && ( -z "${best_limit:-}" || "$limit" -lt "$best_limit" ) ]]; then
    +      best_limit="$limit"
    +    fi
    +  done < /proc/self/cgroup
    +
    +  if [[ -n "${best_limit:-}" && "$best_limit" -lt "$mem_total" ]]; then
    +    printf '%s\n' "$best_limit"
    +  else
    +    printf '%s\n' "$mem_total"
    +  fi
    +}
    +
    +collect_memory_checks() {
    +  local memory min recommended
    +  min=$((4 * 1024 * 1024 * 1024))
    +  recommended=$((16 * 1024 * 1024 * 1024))
    +  memory="$(effective_memory_bytes)"
    +
    +  if ((memory < min)); then
    +    add_low_spec_error "detected effective memory $(human_gib "$memory"), minimum required is $(human_gib "$min")"
    +  elif ((memory < recommended)); then
    +    add_warning "detected effective memory $(human_gib "$memory"), recommended is $(human_gib "$recommended")"
    +  fi
    +}
    +
    +disk_device_and_size() {
    +  local path="$1"
    +  local ancestor device size
    +  ancestor="$(nearest_existing_ancestor "$path")" || return 1
    +  device="$(stat -c '%d' "$ancestor")"
    +  size="$(df -PB1 "$ancestor" | awk 'NR == 2 { gsub(/B$/, "", $2); print $2 }')"
    +  printf '%s %s\n' "$device" "$size"
    +}
    +
    +collect_disk_checks() {
    +  local zebra_info zebra_device zebra_size
    +  local gib required
    +
    +  gib=$((1024 * 1024 * 1024))
    +  required=$((300 * gib))
    +
    +  if ! zebra_info="$(disk_device_and_size "$ZEBRA_STATE_DIR")"; then
    +    add_error "failed to inspect filesystem for zebra state path: $ZEBRA_STATE_DIR"
    +    return
    +  fi
    +
    +  read -r zebra_device zebra_size <<<"$zebra_info"
    +  if ((zebra_size < required)); then
    +    add_low_spec_error "zebra state mount (path: $ZEBRA_STATE_DIR) has provisioned capacity $(human_gib "$zebra_size"), minimum required is $(human_gib "$required")"
    +  fi
    +
    +  _="$zebra_device"
    +}
    +
    +collect_source_checks() {
    +  if [[ "$MODE" != "build-from-source" ]]; then
    +    return
    +  fi
    +
    +  if [[ ! -d "$REPO_ROOT" ]]; then
    +    add_error "Zebra source tree is missing: $REPO_ROOT"
    +  elif [[ ! -f "$REPO_ROOT/Cargo.toml" ]]; then
    +    add_error "Zebra source tree is missing Cargo.toml: $REPO_ROOT"
    +  fi
    +
    +  ZEBRAD_PATH="${ZEBRAD_PATH:-$REPO_ROOT/target/release/zebrad}"
    +
    +  if [[ -e "$ZEBRAD_PATH" && ! -x "$ZEBRAD_PATH" ]]; then
    +    add_error "zebrad binary $ZEBRAD_PATH exists but is not executable by the current user"
    +  fi
    +}
    +
    +collect_checks() {
    +  collect_platform_checks
    +  collect_tool_checks
    +  collect_permission_checks
    +  collect_cpu_checks
    +  collect_memory_checks
    +  collect_disk_checks
    +  collect_source_checks
    +}
    +
    +download_and_extract() {
    +  local name="$1"
    +  local url="$2"
    +  local member="$3"
    +  local archive_name="$4"
    +  local destination="$5"
    +  local archive_path extract_dir source_path
    +
    +  archive_path="$CACHE_DIR/$archive_name"
    +  extract_dir="$CACHE_DIR/${archive_name%.tar.gz}"
    +
    +  if ((DRY_RUN)); then
    +    if ((USE_ANSI)); then
    +      printf '%s %s\n' "$(style "$CYAN" "[down]")" "$(style "$DIM" "Dry run: would download $name from $url")"
    +      printf '%s %s\n' "$(style "$CYAN" "[file]")" "$(style "$DIM" "Dry run: would extract $member to $destination")"
    +    else
    +      printf 'Dry run: would download %s from %s\n' "$name" "$url"
    +      printf 'Dry run: would extract %s to %s\n' "$member" "$destination"
    +    fi
    +    return
    +  fi
    +
    +  mkdir -p "$CACHE_DIR" "$extract_dir" "$(dirname "$destination")"
    +  if ((USE_ANSI)); then
    +    printf '%s Downloading %s from %s\n' "$(style "$CYAN" "[down]")" "$(style "$BOLD" "$name")" "$url"
    +  else
    +    printf 'Downloading %s from %s\n' "$name" "$url"
    +  fi
    +  curl -fsSL "$url" -o "$archive_path"
    +
    +  rm -rf "$extract_dir"
    +  mkdir -p "$extract_dir"
    +  tar -xzf "$archive_path" -C "$extract_dir"
    +
    +  source_path="$extract_dir/${member#./}"
    +  if [[ ! -x "$source_path" ]]; then
    +    add_error "expected executable missing from $name archive: $member"
    +    finalize_checks
    +  fi
    +
    +  install -D -m 0755 "$source_path" "$destination"
    +}
    +
    +prepare_binary_path() {
    +  ZEBRAD_PATH="${ZEBRAD_PATH:-$INSTALL_DIR/zebrad/bin/zebrad}"
    +
    +  if ((DOWNLOAD_BINARIES == 0)); then
    +    if ((USE_ANSI)); then
    +      printf '%s Skipping binary downloads. You must provision the right Zebra version yourself.\n' "$(style "$YELLOW" "[!]")"
    +    else
    +      printf 'Skipping binary downloads. You must provision the right Zebra version yourself.\n'
    +    fi
    +    printf '\nDownload Zebra:\n%s\n\n' "$ZEBRA_URL"
    +    if ((!DRY_RUN)); then
    +      [[ -x "$ZEBRAD_PATH" ]] || add_error "zebrad binary $ZEBRAD_PATH does not exist or is not executable by the current user"
    +      finalize_checks
    +    fi
    +    return
    +  fi
    +
    +  download_and_extract "zebrad" "$ZEBRA_URL" "$ZEBRA_MEMBER" "$ZEBRA_ARCHIVE" "$ZEBRAD_PATH"
    +
    +  if ((!DRY_RUN)); then
    +    [[ -x "$ZEBRAD_PATH" ]] || add_error "zebrad binary $ZEBRAD_PATH does not exist or is not executable by the current user"
    +    finalize_checks
    +  fi
    +}
    +
    +data_detection_message() {
    +  if ((USE_ANSI)); then
    +    print_section "[*]" "Snapshot data"
    +  fi
    +
    +  if [[ -d "$ZEBRA_STATE_DIR" && -n "$(find "$ZEBRA_STATE_DIR" -mindepth 1 -maxdepth 1 -print -quit 2>/dev/null)" ]]; then
    +    if ((USE_ANSI)); then
    +      printf '%s You already have Zebra data configured but feel free to redownload a fresh snapshot\n' "$(style "$GREEN" "[ok]")"
    +    else
    +      printf 'You already have Zebra data configured but feel free to redownload a fresh snapshot\n'
    +    fi
    +  else
    +    if ((USE_ANSI)); then
    +      printf '%s Please download the Zebra snapshot from the location below if you want a faster sync\n' "$(style "$CYAN" "[down]")"
    +    else
    +      printf 'Please download the Zebra snapshot from the location below if you want a faster sync\n'
    +    fi
    +  fi
    +  printf '\nhttps://zebra.valargroup.org/\n\n'
    +}
    +
    +docker_image_available_or_pull() {
    +  local image="$1"
    +
    +  if ((DRY_RUN)); then
    +    if ((USE_ANSI)); then
    +      printf '%s %s\n' "$(style "$CYAN" "[down]")" "$(style "$DIM" "Dry run: would inspect or pull Docker image $image")"
    +    else
    +      printf 'Dry run: would inspect or pull Docker image %s\n' "$image"
    +    fi
    +    return 0
    +  fi
    +
    +  if docker image inspect "$image" >/dev/null 2>&1; then
    +    return 0
    +  fi
    +
    +  docker pull "$image" >/dev/null 2>&1
    +}
    +
    +prepare_docker_image() {
    +  docker_image_available_or_pull "$ZEBRA_DOCKER_IMAGE" ||
    +    add_error "Docker image is missing or could not be pulled: $ZEBRA_DOCKER_IMAGE"
    +
    +  finalize_checks
    +}
    +
    +print_native_command() {
    +  cat <&2
    +          usage >&2
    +          exit 2
    +          ;;
    +      esac
    +      shift 2
    +      ;;
    +    --dry-run)
    +      DRY_RUN=1
    +      NON_INTERACTIVE=1
    +      shift
    +      ;;
    +    --unsafe-low-specs)
    +      UNSAFE_LOW_SPECS=1
    +      shift
    +      ;;
    +    -y | --yes | --non-interactive)
    +      NON_INTERACTIVE=1
    +      shift
    +      ;;
    +    -h | --help)
    +      usage
    +      exit 0
    +      ;;
    +    *)
    +      echo "unknown argument: $1" >&2
    +      usage >&2
    +      exit 2
    +      ;;
    +  esac
    +done
    +
    +normalize_inputs
    +collect_checks
    +finalize_checks
    +data_detection_message
    +
    +case "$MODE" in
    +  native)
    +    prepare_binary_path
    +    ;;
    +  docker)
    +    prepare_docker_image
    +    ;;
    +  build-from-source)
    +    ;;
    +esac
    +
    +print_ready_commands
    diff --git a/scripts/install-zcashd-compat.sh b/scripts/install-zcashd-compat.sh
    index cf4ba3e2896..f647af317f8 100755
    --- a/scripts/install-zcashd-compat.sh
    +++ b/scripts/install-zcashd-compat.sh
    @@ -11,18 +11,19 @@ fi
     REPO_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
     UNITY_ROOT="$(cd "$REPO_ROOT/.." && pwd)"
     
    -ZEBRA_RELEASE_TAG="v5.0.0-test.1"
    +ZEBRA_RELEASE_TAG="v5.0.0-test.4"
     ZEBRA_ARCHIVE="zebrad-${ZEBRA_RELEASE_TAG}-linux-x86_64.tar.gz"
     ZEBRA_URL="https://github.com/valargroup/zebra/releases/download/${ZEBRA_RELEASE_TAG}/${ZEBRA_ARCHIVE}"
     ZEBRA_MEMBER="./bin/zebrad"
    -ZEBRA_DOCKER_IMAGE="valaroman/zebra:5.0.0-test.1"
    -ZEBRA_COMPAT_DOCKER_IMAGE="valaroman/zebra:zcashd-compat-5.0.0-test.1"
    +ZEBRA_DOCKER_IMAGE="valaroman/zebra:5.0.0-test.4"
    +ZEBRA_COMPAT_DOCKER_IMAGE="valaroman/zebra:zcashd-compat-5.0.0-test.4"
     ZEBRA_COMPAT_DOCKER_FALLBACK_IMAGE="valaroman/zebra:zcashd-compat-latest"
    +ZEBRA_DEFAULT_CACHE_DIR="${XDG_CACHE_HOME:-${HOME}/.cache}/zebra"
     
     MANIFEST_PATH="$REPO_ROOT/zebrad/zcashd-compat-manifest.json"
     TARGET_TRIPLE="x86_64-pc-linux-gnu"
    -ZCASHD_RUNTIME_ARCHIVE_URL="https://github.com/valargroup/zcashd/releases/download/v6.2.1-alpha.7/zcashd-zebra-compat-v6.2.1-alpha.7-linux-x86_64.tar.gz"
    -ZCASHD_RUNTIME_ARCHIVE_SHA256="6a1c4dc673646bc514f289b306944b4081a5f44dfd3206e5b22336f95c3ad6c6"
    +ZCASHD_RUNTIME_ARCHIVE_URL="https://github.com/valargroup/zcashd/releases/download/v6.2.1-alpha.9/zcashd-zebra-compat-v6.2.1-alpha.9-linux-x86_64.tar.gz"
    +ZCASHD_RUNTIME_ARCHIVE_SHA256="5f899869a0e5fdb8d5179e291d764f37b9e4e6db04163a07fc2439806c8f0097"
     ZCASHD_RUNTIME_ARCHIVE_MEMBER_BINARY_PATH="./bin/zcashd"
     
     MODE=""
    @@ -365,11 +366,6 @@ normalize_inputs() {
       ZEBRA_STATE_DIR="$(prompt_value "Zebra state directory" "$ZEBRA_STATE_DIR")"
       ZCASHD_DATADIR="$(prompt_value "zcashd datadir" "$ZCASHD_DATADIR")"
       INSTALL_DIR="$(prompt_value "Install directory" "$INSTALL_DIR")"
    -  CACHE_DIR="$(prompt_value "Download/cache directory" "$CACHE_DIR")"
    -
    -  if [[ -z "$COOKIE_DIR" ]]; then
    -    COOKIE_DIR="$ZEBRA_STATE_DIR"
    -  fi
     
       if [[ -z "$ZCASHD_CONF" ]]; then
         ZCASHD_CONF="$ZCASHD_DATADIR/zcash.conf"
    @@ -379,6 +375,9 @@ normalize_inputs() {
       ZCASHD_DATADIR="$(printf '%s' "$ZCASHD_DATADIR" | sanitize_terminal_input)"
       INSTALL_DIR="$(printf '%s' "$INSTALL_DIR" | sanitize_terminal_input)"
       CACHE_DIR="$(printf '%s' "$CACHE_DIR" | sanitize_terminal_input)"
    +  if [[ -z "$COOKIE_DIR" ]]; then
    +    COOKIE_DIR="$ZEBRA_STATE_DIR"
    +  fi
       COOKIE_DIR="$(printf '%s' "$COOKIE_DIR" | sanitize_terminal_input)"
       ZCASHD_CONF="$(printf '%s' "$ZCASHD_CONF" | sanitize_terminal_input)"
       ZEBRAD_PATH="$(printf '%s' "$ZEBRAD_PATH" | sanitize_terminal_input)"
    @@ -860,7 +859,7 @@ prepare_docker_images() {
             add_error "Docker image is missing or could not be pulled: $ZEBRA_DOCKER_IMAGE"
     
           if [[ -z "$ZCASHD_DOCKER_IMAGE" ]]; then
    -        ZCASHD_DOCKER_IMAGE="valargroup/zcashd:zcashd-compat-v6.2.1-alpha.7"
    +        ZCASHD_DOCKER_IMAGE="valaroman/zcashd:v6.2.1-alpha.9@sha256:ca1878cc78f2794105c7b903a8a3e4839fc4f63e2ff2964f4d3721a32b53e766"
           fi
     
           docker_image_available_or_pull "$ZCASHD_DOCKER_IMAGE" ||
    @@ -893,6 +892,7 @@ print_supervised_command() {
       cat <
    Date: Sat, 13 Jun 2026 17:28:47 -0300
    Subject: [PATCH 316/431] docs: streamline Zakura install section
    
    Replace the long Zakura binary and snapshot setup block with the supported installer command so Valarbook no longer shows stale release-specific node startup instructions.
    ---
     book/valarbook/index.html | 35 +----------------------------------
     1 file changed, 1 insertion(+), 34 deletions(-)
    
    diff --git a/book/valarbook/index.html b/book/valarbook/index.html
    index 4f65274f26c..967a481591b 100644
    --- a/book/valarbook/index.html
    +++ b/book/valarbook/index.html
    @@ -529,7 +529,7 @@ 

    Overview

    -

    Install Zakura

    +

    Install and Run Zakura

    Download and run the standalone Zakura installer from the latest Zebra GitHub release. It guides native, Docker, and build-from-source setup modes and downloads only Zebra binaries.

    @@ -544,39 +544,6 @@

    Install Zakura

    -

    - The installer defaults to normal zebrad start operation and does not enable zcashd compatibility mode. -

    -
    -
    -

    Run the node

    -
    -
    # 1. download the prebuilt zebrad release binary (Linux x86_64)
    -wget https://github.com/valargroup/zebra/releases/download/v5.0.0-test.4/zebrad-v5.0.0-test.4-linux-x86_64.tar.gz
    -tar -xzf zebrad-v5.0.0-test.4-linux-x86_64.tar.gz
    -
    -# 2. download the latest snapshot (resumable, parallel, checksum-verified)
    -aria2c -x16 -s16 --checksum=sha-256=1154d81bafca6cccbd6a53888284a5c941523be693d33eaddec2b05f029d7ba2 \
    -  -o zebra-mainnet-20260612T013145Z-3374802.tar.zst \
    -  https://zebra-snapshots.nyc3.cdn.digitaloceanspaces.com/mainnet/zebra-mainnet-20260612T013145Z-3374802.tar.zst \
    -  https://zebra-snapshots-ams3.ams3.digitaloceanspaces.com/mainnet/zebra-mainnet-20260612T013145Z-3374802.tar.zst \
    -  https://zebra-snapshots-sgp1.sgp1.digitaloceanspaces.com/mainnet/zebra-mainnet-20260612T013145Z-3374802.tar.zst
    -
    -# 3. extract into the cache dir
    -mkdir -p ~/.cache/zebra && zstd -dc zebra-mainnet-20260612T013145Z-3374802.tar.zst | tar -x -C ~/.cache/zebra
    -
    -# 4. start at snapshot height, not genesis
    -export ZEBRA_RPC__LISTEN_ADDR=127.0.0.1:8232 ZEBRA_RPC__ENABLE_COOKIE_AUTH=false
    -./bin/zebrad start
    - -
    From f752c9cb69a782dced0e87aced8c4198deef6efa Mon Sep 17 00:00:00 2001 From: Roman Akhtariev Date: Sat, 13 Jun 2026 17:29:50 -0300 Subject: [PATCH 317/431] ci: installer and dev ops edits (#80) * fix(zcashd-compat): allow readiness hysteresis in reorg tests * ci: isntaller improvements * docs(valarbook): streamline Zakura install section From 88347ac67a177b29c251300d8a3370511695a583 Mon Sep 17 00:00:00 2001 From: Roman Akhtariev Date: Sat, 13 Jun 2026 17:35:27 -0300 Subject: [PATCH 318/431] fix(zcashd-compat): repair split-container zcashd startup (#82) --- scripts/install-zcashd-compat.sh | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/scripts/install-zcashd-compat.sh b/scripts/install-zcashd-compat.sh index f647af317f8..7fd33eccb50 100755 --- a/scripts/install-zcashd-compat.sh +++ b/scripts/install-zcashd-compat.sh @@ -859,7 +859,7 @@ prepare_docker_images() { add_error "Docker image is missing or could not be pulled: $ZEBRA_DOCKER_IMAGE" if [[ -z "$ZCASHD_DOCKER_IMAGE" ]]; then - ZCASHD_DOCKER_IMAGE="valaroman/zcashd:v6.2.1-alpha.9@sha256:ca1878cc78f2794105c7b903a8a3e4839fc4f63e2ff2964f4d3721a32b53e766" + ZCASHD_DOCKER_IMAGE="valaroman/zcashd:v6.2.1-alpha.9.1@sha256:77f7a000c47248aef00a7f7b14b45ae49aa98f8c9d86c86767714ff76034698c" fi docker_image_available_or_pull "$ZCASHD_DOCKER_IMAGE" || @@ -937,13 +937,12 @@ docker run --rm -it --name zebra-compat-zcashd --network host \\ --mount type=bind,src=$(shell_quote "$ZCASHD_DATADIR"),dst=/home/zcashd/.zcash \\ --mount type=bind,src=$(shell_quote "$ZEBRA_STATE_DIR"),dst=/zebra-state,readonly \\ $(shell_quote "$ZCASHD_DOCKER_IMAGE") \\ - zcashd \\ - -zebra-compat \\ - -zebra-compat-url=http://127.0.0.1:28232 \\ - -zebra-compat-cookiefile=$(shell_quote "$cookie_file") \\ - -datadir=/home/zcashd/.zcash \\ - -conf=/home/zcashd/.zcash/zcash.conf \\ - -printtoconsole + -zebra-compat \\ + -zebra-compat-url=http://127.0.0.1:28232 \\ + -zebra-compat-cookiefile=$(shell_quote "$cookie_file") \\ + -datadir=/home/zcashd/.zcash \\ + -conf=/home/zcashd/.zcash/zcash.conf \\ + -printtoconsole EOF } From 21cc835223132ba8850af98c65229cc533514d5d Mon Sep 17 00:00:00 2001 From: Roman Akhtariev Date: Sat, 13 Jun 2026 17:59:11 -0300 Subject: [PATCH 319/431] ci: devops fixes (#83) --- docker/entrypoint.sh | 1 + make/zcashd-compat.mk | 2 +- scripts/install-zcashd-compat.sh | 8 ++++++-- 3 files changed, 8 insertions(+), 3 deletions(-) diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh index 8c651fe8a24..b24f3c4a50e 100755 --- a/docker/entrypoint.sh +++ b/docker/entrypoint.sh @@ -72,6 +72,7 @@ create_owned_directory() { # Create and own cache and config directories based on ZEBRA_* environment variables [[ -n ${ZEBRA_STATE__CACHE_DIR} ]] && create_owned_directory "${ZEBRA_STATE__CACHE_DIR}" [[ -n ${ZEBRA_RPC__COOKIE_DIR} ]] && create_owned_directory "${ZEBRA_RPC__COOKIE_DIR}" +[[ -n ${ZEBRA_ZCASHD_COMPAT__ZCASHD_DATADIR:-} ]] && create_owned_directory "${ZEBRA_ZCASHD_COMPAT__ZCASHD_DATADIR}" [[ -n ${ZEBRA_TRACING__LOG_FILE} ]] && create_owned_directory "$(dirname "${ZEBRA_TRACING__LOG_FILE}")" # --- Optional config file support --- diff --git a/make/zcashd-compat.mk b/make/zcashd-compat.mk index fa03b24e91e..00dae2861f6 100644 --- a/make/zcashd-compat.mk +++ b/make/zcashd-compat.mk @@ -25,7 +25,7 @@ ZCASHD_CONF ?= $(ZCASHD_DATADIR)/zcash.conf ZCASHD_EXTRA_ARGS ?= -printtoconsole ZCASHD_ZEBRA_RPC_URL ?= http://127.0.0.1:28232 -ZEBRA_COOKIE_DIR ?= $(HOME)/.cache/zebra +ZEBRA_COOKIE_DIR ?= $(ZEBRA_STATE_CACHE_DIR) ZEBRA_COOKIE_FILE ?= $(ZEBRA_COOKIE_DIR)/.zcashd-compat.cookie HEIGHT_MAX_DRIFT ?= 10 diff --git a/scripts/install-zcashd-compat.sh b/scripts/install-zcashd-compat.sh index 7fd33eccb50..907ab5071c4 100755 --- a/scripts/install-zcashd-compat.sh +++ b/scripts/install-zcashd-compat.sh @@ -908,11 +908,13 @@ docker run --rm -it \\ -e ZEBRA_STATE__CACHE_DIR=/home/zebra/.cache/zebra \\ -e ZEBRA_ZCASHD_COMPAT__MANAGE_ZCASHD=true \\ -e ZEBRA_ZCASHD_COMPAT__ZCASHD_DATADIR=/home/zebra/.cache/zcashd \\ - -e ZEBRA_ZCASHD_COMPAT__LISTEN_ADDR=127.0.0.1:28232 \\ + -e ZEBRA_ZCASHD_COMPAT__LISTEN_ADDR=0.0.0.0:28232 \\ + -e ZEBRA_ZCASHD_COMPAT__UNSAFE_ALLOW_REMOTE_HTTP=true \\ -e ZEBRA_ZCASHD_COMPAT__ZCASHD_EXTRA_ARGS='["-rpcbind=0.0.0.0","-rpcallowip=0.0.0.0/0"]' \\ --mount type=bind,src=$(shell_quote "$ZEBRA_STATE_DIR"),dst=/home/zebra/.cache/zebra \\ --mount type=bind,src=$(shell_quote "$ZCASHD_DATADIR"),dst=/home/zebra/.cache/zcashd \\ -p 8233:8233 \\ + -p 127.0.0.1:28232:28232 \\ -p 127.0.0.1:8232:8232 \\ $(shell_quote "$image") \\ zebrad start --zcashd-compat @@ -926,7 +928,9 @@ $(style "$GREEN$BOLD" "Start Zebra container in terminal 1:") docker run --rm -it --name zebra-compat-zebrad \\ -e ZEBRA_NETWORK__LISTEN_ADDR='[::]:8233' \\ -e ZEBRA_STATE__CACHE_DIR=/home/zebra/.cache/zebra \\ - --mount type=bind,src=$(shell_quote "$ZEBRA_STATE_DIR"),dst=/home/zebra/.cache/zebra \\ + -e ZEBRA_ZCASHD_COMPAT__LISTEN_ADDR=0.0.0.0:28232 \\ + -e ZEBRA_ZCASHD_COMPAT__UNSAFE_ALLOW_REMOTE_HTTP=true \\ + --mount type=bind,src=$(shell_quote "$ZEBRA_STATE_DIR"),dst=/home/zebra/.cache/zebra \\ -p 8233:8233 \\ -p 127.0.0.1:28232:28232 \\ $(shell_quote "$ZEBRA_DOCKER_IMAGE") \\ From 0822884c1bd9b33dab024c4e55878048392c7054 Mon Sep 17 00:00:00 2001 From: Roman Akhtariev Date: Sat, 13 Jun 2026 18:14:07 -0300 Subject: [PATCH 320/431] fix(zcashd-compat): clarify docker supervised paths (#84) --- scripts/install-zcashd-compat.sh | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/scripts/install-zcashd-compat.sh b/scripts/install-zcashd-compat.sh index 907ab5071c4..46afca6f4bb 100755 --- a/scripts/install-zcashd-compat.sh +++ b/scripts/install-zcashd-compat.sh @@ -901,18 +901,21 @@ EOF print_docker_supervised_command() { local image="${ZEBRA_COMPAT_DOCKER_SELECTED:-$ZEBRA_COMPAT_DOCKER_IMAGE}" + local container_zebra_state_dir="/home/zebra/.cache/zebra" + local container_zcashd_datadir="/home/zebra/.cache/zcashd" cat < Date: Sun, 14 Jun 2026 00:22:30 -0300 Subject: [PATCH 321/431] ci: docker permissioning edits (#86) --- scripts/install-zcashd-compat.sh | 59 ++++++++++++++++++++++++++++++++ 1 file changed, 59 insertions(+) diff --git a/scripts/install-zcashd-compat.sh b/scripts/install-zcashd-compat.sh index 46afca6f4bb..b580fdbe147 100755 --- a/scripts/install-zcashd-compat.sh +++ b/scripts/install-zcashd-compat.sh @@ -19,6 +19,8 @@ ZEBRA_DOCKER_IMAGE="valaroman/zebra:5.0.0-test.4" ZEBRA_COMPAT_DOCKER_IMAGE="valaroman/zebra:zcashd-compat-5.0.0-test.4" ZEBRA_COMPAT_DOCKER_FALLBACK_IMAGE="valaroman/zebra:zcashd-compat-latest" ZEBRA_DEFAULT_CACHE_DIR="${XDG_CACHE_HOME:-${HOME}/.cache}/zebra" +ZEBRA_DOCKER_RUNTIME_UID=10001 +ZEBRA_DOCKER_RUNTIME_GID=10001 MANIFEST_PATH="$REPO_ROOT/zebrad/zcashd-compat-manifest.json" TARGET_TRIPLE="x86_64-pc-linux-gnu" @@ -843,6 +845,54 @@ docker_image_available_or_pull() { docker pull "$image" >/dev/null 2>&1 } +run_privileged_or_current_user() { + if ((EUID == 0)); then + "$@" + return + fi + + if command_exists sudo && sudo -n true 2>/dev/null; then + sudo "$@" + return + fi + + "$@" +} + +prepare_docker_owned_directory() { + local label="$1" + local dir="$2" + local owner="${ZEBRA_DOCKER_RUNTIME_UID}:${ZEBRA_DOCKER_RUNTIME_GID}" + + if ((DRY_RUN)); then + if ((USE_ANSI)); then + printf '%s %s\n' "$(style "$CYAN" "[file]")" "$(style "$DIM" "Dry run: would create $label at $dir and chown it to $owner")" + else + printf 'Dry run: would create %s at %s and chown it to %s\n' "$label" "$dir" "$owner" + fi + return + fi + + if ! mkdir -p "$dir"; then + add_error "failed to create $label for Docker mount: $dir" + return + fi + + if ! run_privileged_or_current_user chown -R "$owner" "$dir"; then + add_error "failed to chown $label $dir to Docker runtime user $owner; run: sudo chown -R $owner $(shell_quote "$dir")" + fi +} + +prepare_docker_supervised_mounts() { + if [[ "$MODE" != "docker-supervised" ]]; then + return + fi + + prepare_docker_owned_directory "Zebra state directory" "$ZEBRA_STATE_DIR" + prepare_docker_owned_directory "zcashd datadir" "$ZCASHD_DATADIR" + finalize_checks +} + prepare_docker_images() { case "$MODE" in docker-supervised) @@ -993,6 +1043,14 @@ print_ready_commands() { esac print_command_block_end + + case "$MODE" in + docker-supervised | docker-split-containers) + printf '\n%s ZEBRA_ZCASHD_COMPAT__UNSAFE_ALLOW_REMOTE_HTTP=true is set because the compat RPC is published on 0.0.0.0 over plain HTTP,\n' "$(style "$YELLOW" "[!]")" + printf ' so the cookie crosses the network in cleartext and is safe behind the local container/private-network boundary.\n' + printf ' To remove it, enable TLS via ZEBRA_ZCASHD_COMPAT__TLS_CERT_FILE/_TLS_KEY_FILE/_TLS_CA_FILE.\n' + ;; + esac } while (($#)); do @@ -1106,6 +1164,7 @@ case "$MODE" in prepare_binary_paths ;; docker-split-containers | docker-supervised) + prepare_docker_supervised_mounts prepare_docker_images ;; build-from-source) From 6878fbf6846cd23f2c050e7ff5dabe61d215a316 Mon Sep 17 00:00:00 2001 From: Roman Akhtariev Date: Sun, 14 Jun 2026 11:41:17 -0300 Subject: [PATCH 322/431] fix: recommend portable datadirs (#87) * fix(zcashd-compat): recommend portable datadirs * chore: remove datadir changelog entry * chore: keep compat make defaults unchanged * chore: restore compat makefile comment --- scripts/install-zcashd-compat.sh | 317 ++++++++++++++++++++++++++++++- 1 file changed, 315 insertions(+), 2 deletions(-) diff --git a/scripts/install-zcashd-compat.sh b/scripts/install-zcashd-compat.sh index b580fdbe147..0d84bb16ac6 100755 --- a/scripts/install-zcashd-compat.sh +++ b/scripts/install-zcashd-compat.sh @@ -30,8 +30,9 @@ ZCASHD_RUNTIME_ARCHIVE_MEMBER_BINARY_PATH="./bin/zcashd" MODE="" NETWORK="Mainnet" -ZEBRA_STATE_DIR="/mnt/data/zebra-state" -ZCASHD_DATADIR="/mnt/data/zcashd-mainnet" +ZCASHD_DEFAULT_DATADIR="${HOME}/.zcash" +ZEBRA_STATE_DIR="$ZEBRA_DEFAULT_CACHE_DIR" +ZCASHD_DATADIR="$ZCASHD_DEFAULT_DATADIR" INSTALL_DIR="${HOME}/.local/zcashd-compat" CACHE_DIR="${HOME}/.cache/zcashd-compat" COOKIE_DIR="" @@ -41,6 +42,8 @@ ZCASHD_PATH="" ZCASHD_DOCKER_IMAGE="" DOWNLOAD_BINARIES=1 DOWNLOAD_BINARIES_SET=0 +ZEBRA_STATE_DIR_SET=0 +ZCASHD_DATADIR_SET=0 DRY_RUN=0 NON_INTERACTIVE=0 UNSAFE_LOW_SPECS=0 @@ -301,6 +304,312 @@ prompt_yes_no() { printf '%s\n' "${sanitized_reply:-$default}" } +network_name_lowercase() { + local network="$NETWORK" + network="${network,,}" + + case "$network" in + main | mainnet) printf 'mainnet\n' ;; + test | testnet) printf 'testnet\n' ;; + regtest) printf 'regtest\n' ;; + *) printf '%s\n' "$network" ;; + esac +} + +zcashd_network_datadir() { + local datadir="$1" + + case "$(network_name_lowercase)" in + testnet) printf '%s/testnet3\n' "$datadir" ;; + regtest) printf '%s/regtest\n' "$datadir" ;; + *) printf '%s\n' "$datadir" ;; + esac +} + +path_capacity_bytes() { + local path="$1" + local ancestor size + + command_exists df || return 1 + command_exists awk || return 1 + + ancestor="$(nearest_existing_ancestor "$path")" || return 1 + size="$(df -PB1 "$ancestor" 2>/dev/null | awk 'NR == 2 { gsub(/B$/, "", $2); print $2 }')" + + [[ "$size" =~ ^[0-9]+$ ]] || return 1 + printf '%s\n' "$size" +} + +path_has_min_capacity() { + local path="$1" + local min_bytes="$2" + local size + + size="$(path_capacity_bytes "$path")" || return 1 + ((size >= min_bytes)) +} + +zebra_state_has_expected_files() { + local dir="$1" + local net_dir matches + + [[ -d "$dir" ]] || return 1 + net_dir="$(network_name_lowercase)" + + matches=("$dir"/state/v*/"$net_dir"/version "$dir"/state/v*/"$net_dir"/CURRENT "$dir"/state/v*/"$net_dir"/MANIFEST-*) + for match in "${matches[@]}"; do + if [[ -e "$match" ]]; then + return 0 + fi + done + + return 1 +} + +zcashd_datadir_has_expected_files() { + local datadir="$1" + local effective_datadir + + [[ -d "$datadir" ]] || return 1 + + for effective_datadir in "$datadir" "$(zcashd_network_datadir "$datadir")"; do + [[ -f "$effective_datadir/zcash.conf" ]] && return 0 + [[ -d "$effective_datadir/blocks" ]] && return 0 + [[ -d "$effective_datadir/blocks/index" ]] && return 0 + [[ -d "$effective_datadir/chainstate" ]] && return 0 + done + + return 1 +} + +candidate_search_roots() { + local root seen + local roots=("$HOME" "${XDG_CACHE_HOME:-$HOME/.cache}" /mnt /media /srv /var/lib /data) + seen="" + + if command_exists df && command_exists awk; then + while IFS= read -r root; do + roots+=("$root") + done < <(df -P 2>/dev/null | awk 'NR > 1 { print $6 }') + fi + + for root in "${roots[@]}"; do + [[ -n "$root" ]] || continue + case $'\n'"$seen" in + *$'\n'"$root"$'\n'*) continue ;; + esac + seen+="$root"$'\n' + printf '%s\n' "$root" + done +} + +check_candidate_datadir() { + local candidate="$1" + local min_bytes="$2" + local expected_files_check="$3" + + [[ -d "$candidate" ]] || return 1 + path_has_min_capacity "$candidate" "$min_bytes" || return 1 + "$expected_files_check" "$candidate" +} + +candidate_size_kib() { + local candidate="$1" + + if command_exists du && command_exists awk; then + du -sk "$candidate" 2>/dev/null | awk 'NR == 1 { print $1 }' + else + printf '0\n' + fi +} + +candidate_score() { + local candidate="$1" + local size_kib + + size_kib="$(candidate_size_kib "$candidate")" + if [[ "$size_kib" =~ ^[0-9]+$ ]]; then + printf '%s\n' "$size_kib" + else + printf '0\n' + fi +} + +maybe_select_better_candidate() { + local candidate="$1" + local min_bytes="$2" + local expected_files_check="$3" + local score + + check_candidate_datadir "$candidate" "$min_bytes" "$expected_files_check" || return 0 + score="$(candidate_score "$candidate")" + + if [[ -z "${BEST_CANDIDATE:-}" || "$score" -gt "${BEST_CANDIDATE_SCORE:-0}" ]]; then + BEST_CANDIDATE="$candidate" + BEST_CANDIDATE_SCORE="$score" + fi +} + +maybe_select_better_install_root() { + local root="$1" + local min_bytes="$2" + local score + + [[ -n "$root" && "$root" != "/" && -d "$root" ]] || return 0 + score="$(path_capacity_bytes "$root" 2>/dev/null || printf '0')" + [[ "$score" =~ ^[0-9]+$ ]] || return 0 + ((score >= min_bytes)) || return 0 + + if [[ -z "${BEST_INSTALL_ROOT:-}" || "$score" -gt "${BEST_INSTALL_ROOT_SCORE:-0}" ]]; then + BEST_INSTALL_ROOT="$root" + BEST_INSTALL_ROOT_SCORE="$score" + fi +} + +recommend_install_root() { + local min_bytes="$1" + local root + BEST_INSTALL_ROOT="" + BEST_INSTALL_ROOT_SCORE=0 + + while IFS= read -r root; do + maybe_select_better_install_root "$root" "$min_bytes" + done < <(candidate_search_roots) + + if [[ -n "$BEST_INSTALL_ROOT" ]]; then + printf '%s\n' "$BEST_INSTALL_ROOT" + return 0 + fi + + return 1 +} + +search_named_candidates() { + local min_bytes="$1" + local expected_files_check="$2" + shift 2 + + local root name candidate + while IFS= read -r root; do + [[ -n "$root" && "$root" != "/" && -d "$root" ]] || continue + + for name in "$@"; do + candidate="$root/$name" + maybe_select_better_candidate "$candidate" "$min_bytes" "$expected_files_check" + done + done < <(candidate_search_roots) +} + +search_zebra_state_candidates() { + local min_bytes="$1" + local root candidate + + search_named_candidates "$min_bytes" zebra_state_has_expected_files \ + ".cache/zebra" "zebra" "zebra-state" "data/zebra" "data/zebra-state" \ + "mnt/data/zebra" "mnt/data/zebra-state" + + command_exists find || return 0 + while IFS= read -r root; do + [[ -n "$root" && "$root" != "/" && -d "$root" ]] || continue + path_has_min_capacity "$root" "$min_bytes" || continue + + while IFS= read -r candidate; do + maybe_select_better_candidate "$candidate" "$min_bytes" zebra_state_has_expected_files + done < <(find "$root" -xdev -maxdepth 5 -type d \( -name zebra -o -name zebra-state \) -print 2>/dev/null) + done < <(candidate_search_roots) +} + +search_zcashd_datadir_candidates() { + local min_bytes="$1" + local root candidate + + search_named_candidates "$min_bytes" zcashd_datadir_has_expected_files \ + ".zcash" "zcash" "zcashd" "zcashd-mainnet" "data/.zcash" "data/zcashd" "data/zcashd-mainnet" \ + "mnt/data/.zcash" "mnt/data/zcashd" "mnt/data/zcashd-mainnet" + + command_exists find || return 0 + while IFS= read -r root; do + [[ -n "$root" && "$root" != "/" && -d "$root" ]] || continue + path_has_min_capacity "$root" "$min_bytes" || continue + + while IFS= read -r candidate; do + maybe_select_better_candidate "$candidate" "$min_bytes" zcashd_datadir_has_expected_files + done < <(find "$root" -xdev -maxdepth 5 -type d \( -name .zcash -o -name zcash -o -name zcashd -o -name zcashd-mainnet \) -print 2>/dev/null) + done < <(candidate_search_roots) +} + +recommend_zebra_state_dir() { + local binary_default="$1" + local min_bytes=$((300 * 1024 * 1024 * 1024)) + local synthetic_min_bytes="${SYNTHETIC_INSTALL_MIN_BYTES:-$min_bytes}" + local install_root + BEST_CANDIDATE="" + BEST_CANDIDATE_SCORE=0 + + maybe_select_better_candidate "$binary_default" "$min_bytes" zebra_state_has_expected_files + search_zebra_state_candidates "$min_bytes" + + if [[ -n "$BEST_CANDIDATE" ]]; then + printf '%s\n' "$BEST_CANDIDATE" + return + fi + + if ! path_has_min_capacity "$binary_default" "$min_bytes"; then + if install_root="$(recommend_install_root "$synthetic_min_bytes")"; then + printf '%s/.cache/zebra\n' "$install_root" + return + fi + fi + + printf '%s\n' "$binary_default" +} + +recommend_zcashd_datadir() { + local binary_default="$1" + local min_bytes=$((300 * 1024 * 1024 * 1024)) + local synthetic_min_bytes="${SYNTHETIC_INSTALL_MIN_BYTES:-$min_bytes}" + local install_root + BEST_CANDIDATE="" + BEST_CANDIDATE_SCORE=0 + + maybe_select_better_candidate "$binary_default" "$min_bytes" zcashd_datadir_has_expected_files + search_zcashd_datadir_candidates "$min_bytes" + + if [[ -n "$BEST_CANDIDATE" ]]; then + printf '%s\n' "$BEST_CANDIDATE" + return + fi + + if ! path_has_min_capacity "$binary_default" "$min_bytes"; then + if install_root="$(recommend_install_root "$synthetic_min_bytes")"; then + printf '%s/.zcash\n' "$install_root" + return + fi + fi + + printf '%s\n' "$binary_default" +} + +recommend_datadir_defaults() { + # Empty fallback locations share a filesystem, so size them for both datadirs + # when both prompt defaults are being selected together. + if ((ZEBRA_STATE_DIR_SET == 0 && ZCASHD_DATADIR_SET == 0)); then + SYNTHETIC_INSTALL_MIN_BYTES=$((600 * 1024 * 1024 * 1024)) + else + SYNTHETIC_INSTALL_MIN_BYTES=$((300 * 1024 * 1024 * 1024)) + fi + + if ((ZEBRA_STATE_DIR_SET == 0)); then + ZEBRA_STATE_DIR="$(recommend_zebra_state_dir "$ZEBRA_DEFAULT_CACHE_DIR")" + fi + + if ((ZCASHD_DATADIR_SET == 0)); then + ZCASHD_DATADIR="$(recommend_zcashd_datadir "$ZCASHD_DEFAULT_DATADIR")" + fi + + unset SYNTHETIC_INSTALL_MIN_BYTES +} + prompt_mode() { local reply @@ -365,6 +674,8 @@ normalize_inputs() { fi fi + recommend_datadir_defaults + ZEBRA_STATE_DIR="$(prompt_value "Zebra state directory" "$ZEBRA_STATE_DIR")" ZCASHD_DATADIR="$(prompt_value "zcashd datadir" "$ZCASHD_DATADIR")" INSTALL_DIR="$(prompt_value "Install directory" "$INSTALL_DIR")" @@ -1068,11 +1379,13 @@ while (($#)); do --zebra-state-dir) require_value "$1" "${2:-}" ZEBRA_STATE_DIR="$2" + ZEBRA_STATE_DIR_SET=1 shift 2 ;; --zcashd-datadir) require_value "$1" "${2:-}" ZCASHD_DATADIR="$2" + ZCASHD_DATADIR_SET=1 shift 2 ;; --install-dir) From 7167bc4bcc190f6f52795b1ca5cf45a2917dda4c Mon Sep 17 00:00:00 2001 From: Roman Akhtariev Date: Tue, 16 Jun 2026 10:09:04 -0300 Subject: [PATCH 323/431] fix(network,zebrad): avoid genesis-to-tip sync stalls (#92) --- zebra-network/src/peer/client.rs | 54 +++++++++++++++----------------- zebrad/src/components/inbound.rs | 1 + zebrad/src/components/sync.rs | 13 ++++++-- 3 files changed, 37 insertions(+), 31 deletions(-) diff --git a/zebra-network/src/peer/client.rs b/zebra-network/src/peer/client.rs index 5b6e3d2843f..5db74efb0db 100644 --- a/zebra-network/src/peer/client.rs +++ b/zebra-network/src/peer/client.rs @@ -359,13 +359,12 @@ impl MissingInventoryCollector { } } - /// Forwards any missing inventory to the registry. + /// Forwards explicitly observed missing inventory to the registry. /// - /// `zcashd` doesn't send `notfound` messages for blocks, - /// so we need to track missing blocks ourselves. - /// - /// This can sometimes send duplicate missing inventory, - /// but the registry ignores duplicates anyway. + /// Only explicit `notfound` responses update the registry. Transport + /// failures and timeouts do not prove the peer lacks the inventory, + /// so they are not forwarded (prevents registry poisoning under + /// high sync concurrency). pub fn send(self, response: &Result) { let missing_inv: HashSet = match (self.request, response) { // Missing block hashes from partial responses. @@ -385,31 +384,28 @@ impl MissingInventoryCollector { // Other response types never contain missing inventory. (_, Ok(_)) => iter::empty().collect(), - // We don't forward NotFoundRegistry errors, - // because the errors are generated locally from the registry, - // so those statuses are already in the registry. - // - // Unfortunately, we can't access the inner error variant here, - // due to TracedError. + // Registry-generated errors are already tracked — don't re-forward. (_, Err(e)) if e.inner_debug().contains("NotFoundRegistry") => iter::empty().collect(), - // Missing inventory from other errors, including NotFoundResponse, timeouts, - // and dropped connections. - (request, Err(_)) => { - // The request either contains blocks or transactions, - // but this is a convenient way to collect them both. - let missing_blocks = request - .block_hash_inventory() - .into_iter() - .map(InventoryHash::Block); - - let missing_txs = request - .transaction_id_inventory() - .into_iter() - .map(InventoryHash::from); - - missing_blocks.chain(missing_txs).collect() - } + // Only mark inventory as missing for explicit notfound responses. + // Transport failures (timeouts, dropped connections) do not prove + // the peer lacks the inventory — they only prove the request failed. + // Treating them as missing poisons the routing registry under high + // sync concurrency. + (ref request, Err(e)) if e.inner_debug().contains("NotFoundResponse") => request + .block_hash_inventory() + .into_iter() + .map(InventoryHash::Block) + .chain( + request + .transaction_id_inventory() + .into_iter() + .map(InventoryHash::from), + ) + .collect(), + + // All other errors (timeouts, drops, overload): don't poison. + (_, Err(_)) => iter::empty().collect(), }; if let Some(missing_inv) = diff --git a/zebrad/src/components/inbound.rs b/zebrad/src/components/inbound.rs index 5211bd4a648..465bd9e6ced 100644 --- a/zebrad/src/components/inbound.rs +++ b/zebrad/src/components/inbound.rs @@ -113,6 +113,7 @@ pub struct InboundSetupData { } /// Tracks the internal state of the [`Inbound`] service during setup. +#[allow(clippy::large_enum_variant)] pub enum Setup { /// Waiting for service setup to complete. /// diff --git a/zebrad/src/components/sync.rs b/zebrad/src/components/sync.rs index 07aa9541ea8..cf5dc9b1be0 100644 --- a/zebrad/src/components/sync.rs +++ b/zebrad/src/components/sync.rs @@ -1100,8 +1100,18 @@ where IndexSet::new() }; + // Dispatch blocks with duplicate-tolerant error handling. + // DuplicateBlockQueuedForDownload is caught and skipped instead of + // propagating — this prevents dropping unprocessed hashes from the + // batch, which would create frontier gaps and stalls (#5709). for hash in hashes.into_iter() { - self.downloads.download_and_verify(hash).await?; + match self.downloads.download_and_verify(hash).await { + Ok(()) => {} + Err(BlockDownloadVerifyError::DuplicateBlockQueuedForDownload { .. }) => { + debug!("block request was already queued, continuing"); + } + Err(error) => return Err(error), + } } Ok(extra_hashes) @@ -1149,7 +1159,6 @@ where match response { Ok((height, hash)) => { trace!(?height, ?hash, "verified and committed block to state"); - return Ok(()); } From 3dbdc6d21d75b385cd04f47ce7351697e8d20dad Mon Sep 17 00:00:00 2001 From: Roman Akhtariev Date: Tue, 16 Jun 2026 10:19:57 -0300 Subject: [PATCH 324/431] docs: update installer wording (#100) --- book/valarbook/index.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/book/valarbook/index.html b/book/valarbook/index.html index 967a481591b..24cb3c3ea88 100644 --- a/book/valarbook/index.html +++ b/book/valarbook/index.html @@ -519,7 +519,7 @@

    Zakura

    Overview

    - Zakura is a Valar Group's implementation of a Zcash node. It is an evolution of Zebra. Zakura's network stack is interoperable with zcashd. Zakura implements all the features required to reach Zcash network consensus including the validation of all the consensus rules. + Zakura is a Valar Group's implementation of a Zcash node. Zakura's network stack is interoperable with zcashd. Zakura implements all the features required to reach Zcash network consensus including the validation of all the consensus rules.