Add optional EIP-8025 execution proofs

.github/workflows/kurtosis-eip8025.yml

@@ -0,0 +1,139 @@
+# Test that the EIP-8025 Kurtosis testnet starts and the proof engine integrates
+# correctly with real zkboost-server backends.
+name: kurtosis eip8025
+
+on:
+  push:
+    branches:
+      - unstable
+  pull_request:
+  merge_group:
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  run-eip8025-testnet:
+    runs-on: ${{ github.repository == 'sigp/lighthouse' && 'warp-ubuntu-latest-x64-8x' || 'ubuntu-latest' }}
+    steps:
+      - uses: actions/checkout@v5
+
+      - name: Install Kurtosis
+        run: |
+          echo "deb [trusted=yes] https://sdk.kurtosis.com/kurtosis-cli-release-artifacts/ /" | sudo tee /etc/apt/sources.list.d/kurtosis.list
+          sudo apt update
+          sudo apt install -y kurtosis-cli
+          kurtosis analytics disable
+
+      - name: Install yq
+        run: |
+          sudo wget -qO /usr/local/bin/yq https://github.com/mikefarah/yq/releases/latest/download/yq_linux_amd64
+          sudo chmod +x /usr/local/bin/yq
+
+      - name: Start EIP-8025 zkboost testnet
+        run: ./start_eip8025_testnet.sh -e eip8025-zkboost -n network_params_eip8025_zkboost.yaml
+        working-directory: scripts/local_testnet
+
+      - name: Wait for chain liveness
+        run: |
+          BEACON_URL=$(kurtosis port print eip8025-zkboost cl-1-lighthouse-reth http)
+          echo "Polling $BEACON_URL for head slot > 10..."
+          for i in $(seq 1 20); do
+            SLOT=$(curl -sf "$BEACON_URL/eth/v1/beacon/headers/head" \
+              | jq -r '.data.header.message.slot' 2>/dev/null || echo 0)
+            echo "  attempt $i: head slot = $SLOT"
+            if [ "$SLOT" -gt 10 ]; then
+              echo "Chain is live at slot $SLOT"
+              break
+            fi
+            if [ "$i" -eq 20 ]; then
+              echo "Timed out waiting for head slot > 10"
+              exit 1
+            fi
+            sleep 30
+          done
+
+      - name: Inspect beacon node state
+        run: |
+          set -euo pipefail
+          ENCLAVE=eip8025-zkboost
+          SERVICES=(cl-1-lighthouse-reth cl-2-lighthouse-reth cl-3-lighthouse-reth cl-4-lighthouse-reth)
+          FAILED=0
+
+          for SVC in "${SERVICES[@]}"; do
+            URL=$(kurtosis port print "$ENCLAVE" "$SVC" http)
+            echo "=== $SVC ($URL) ==="
+
+            # Syncing status — must not be syncing
+            SYNCING=$(curl -sf "$URL/eth/v1/node/syncing" | jq -r '.data.is_syncing')
+            echo "  is_syncing: $SYNCING"
+            if [ "$SYNCING" != "false" ]; then
+              echo "  FAIL: $SVC is still syncing"
+              FAILED=1
+            fi
+
+            # Peer count — must have at least one connected peer
+            PEERS=$(curl -sf "$URL/eth/v1/node/peer_count" | jq -r '.data.connected')
+            echo "  connected peers: $PEERS"
+            if [ "${PEERS:-0}" -lt 1 ]; then
+              echo "  FAIL: $SVC has no connected peers"
+              FAILED=1
+            fi
+
+            # Head slot — must be non-zero
+            SLOT=$(curl -sf "$URL/eth/v1/beacon/headers/head" | jq -r '.data.header.message.slot')
+            echo "  head slot: $SLOT"
+            if [ "${SLOT:-0}" -lt 1 ]; then
+              echo "  FAIL: $SVC head slot is 0"
+              FAILED=1
+            fi
+
+            # Finality checkpoints — informational, log the finalized epoch
+            FINALIZED=$(curl -sf "$URL/eth/v1/beacon/states/head/finality_checkpoints" \
+              | jq -r '.data.finalized.epoch')
+            echo "  finalized epoch: $FINALIZED"
+          done
+
+          exit "$FAILED"
+
+      - name: Check zkboost sidecars are running and generating proofs
+        run: |
+          ENCLAVE=eip8025-zkboost
+
+          # Both zkboost services must be in RUNNING state
+          kurtosis enclave inspect "$ENCLAVE" | grep -E "zkboost-[12].*RUNNING" \
+            || { echo "FAIL: one or more zkboost services not in RUNNING state"; exit 1; }
+
+          # Each zkboost sidecar must have generated at least one proof.
+          # Check via the Prometheus metrics endpoint (zkboost_prove_total) rather than
+          # log scraping — kurtosis service logs may not be available in all CI environments.
+          for SVC in zkboost-1 zkboost-2; do
+            URL=$(kurtosis port print "$ENCLAVE" "$SVC" http)
+            COUNT=$(curl -sf "$URL/metrics" \
+              | awk '/^zkboost_prove_total\{/ {sum += $2} END {print int(sum)}')
+            echo "$SVC: $COUNT proofs generated"
+            if [ "${COUNT:-0}" -lt 1 ]; then
+              echo "FAIL: $SVC has not generated any proofs"
+              exit 1
+            fi
+          done
+
+      - name: Stop testnet and collect logs
+        if: always()
+        run: |
+          mkdir -p scripts/local_testnet/logs
+          ENCLAVE=eip8025-zkboost
+          for SVC in cl-1-lighthouse-reth cl-2-lighthouse-reth cl-3-lighthouse-reth cl-4-lighthouse-reth \
+                     zkboost-1 zkboost-2; do
+            kurtosis service logs "$ENCLAVE" "$SVC" > "scripts/local_testnet/logs/${SVC}.log" 2>&1 || true
+          done
+          kurtosis enclave rm -f "$ENCLAVE" || true
+
+      - name: Upload logs artifact
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: logs-kurtosis-eip8025
+          path: scripts/local_testnet/logs
+          retention-days: 3

.github/workflows/nightly-tests.yml

@@ -24,6 +24,8 @@ env:
   LIGHTHOUSE_GITHUB_TOKEN: ${{ secrets.LIGHTHOUSE_GITHUB_TOKEN }}
   # Disable incremental compilation
   CARGO_INCREMENTAL: 0
+  # Keep optional-proofs branch-family caches separate while their dependency graphs diverge.
+  RUST_CACHE_EXTRA_IDENTIFIER: ${{ contains(github.head_ref || github.ref_name, 'optional-proofs-unstable') && 'optional-proofs-unstable' || contains(github.head_ref || github.ref_name, 'optional-proofs') && 'optional-proofs' || '' }}
   # Enable portable to prevent issues with caching `blst` for the wrong CPU type
   TEST_FEATURES: portable
 
@@ -58,6 +60,7 @@ jobs:
       uses: moonrepo/setup-rust@v1
       with:
           channel: stable
+          cache-extra-identifier: ${{ env.RUST_CACHE_EXTRA_IDENTIFIER }}
           cache-target: release
           bins: cargo-nextest
     - name: Run beacon_chain tests for ${{ matrix.fork }}
@@ -82,6 +85,7 @@ jobs:
       uses: moonrepo/setup-rust@v1
       with:
           channel: stable
+          cache-extra-identifier: ${{ env.RUST_CACHE_EXTRA_IDENTIFIER }}
           cache-target: release
           bins: cargo-nextest
     - name: Run operation_pool tests for ${{ matrix.fork }}
@@ -106,6 +110,7 @@ jobs:
       uses: moonrepo/setup-rust@v1
       with:
         channel: stable
+        cache-extra-identifier: ${{ env.RUST_CACHE_EXTRA_IDENTIFIER }}
         cache-target: release
         bins: cargo-nextest
     - name: Create CI logger dir

.github/workflows/test-suite.yml

@@ -26,6 +26,8 @@ env:
   CARGO_INCREMENTAL: 0
   # Enable portable to prevent issues with caching `blst` for the wrong CPU type
   TEST_FEATURES: portable
+  # Keep optional-proofs branch-family caches separate while their dependency graphs diverge.
+  RUST_CACHE_EXTRA_IDENTIFIER: ${{ contains(github.head_ref || github.ref_name, 'optional-proofs-unstable') && 'optional-proofs-unstable' || contains(github.head_ref || github.ref_name, 'optional-proofs') && 'optional-proofs' || '' }}
 jobs:
   check-labels:
     runs-on: ubuntu-latest
@@ -112,6 +114,7 @@ jobs:
       uses: moonrepo/setup-rust@v1
       with:
           channel: stable
+          cache-extra-identifier: ${{ env.RUST_CACHE_EXTRA_IDENTIFIER }}
           cache-target: release
           bins: cargo-nextest
       env:
@@ -120,12 +123,37 @@ jobs:
       uses: Swatinem/rust-cache@v2
       with:
           cache-provider: warpbuild
+          key: ${{ env.RUST_CACHE_EXTRA_IDENTIFIER }}
     - name: Run tests in release
       run: make test-release
     - name: Show cache stats
       if: env.SELF_HOSTED_RUNNERS == 'true'
       continue-on-error: true
       run: sccache --show-stats
+  proof-engine-tests:
+    name: proof-engine-tests
+    needs: [check-labels]
+    if: needs.check-labels.outputs.skip_ci != 'true'
+    runs-on: ${{ github.repository == 'sigp/lighthouse' && 'warp-ubuntu-latest-x64-4x;snapshot.key=lighthouse-ubuntu-latest-v1' || 'ubuntu-latest' }}
+    steps:
+    - uses: actions/checkout@v5
+    - if: github.repository != 'sigp/lighthouse'
+      name: Get latest version of stable Rust
+      uses: moonrepo/setup-rust@v1
+      with:
+          channel: stable
+          cache-extra-identifier: ${{ env.RUST_CACHE_EXTRA_IDENTIFIER }}
+          cache-target: release
+          bins: cargo-nextest
+      env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+    - if: github.repository == 'sigp/lighthouse'
+      uses: Swatinem/rust-cache@v2
+      with:
+          cache-provider: warpbuild
+          key: ${{ env.RUST_CACHE_EXTRA_IDENTIFIER }}
+    - name: Run proof engine tests sequentially
+      run: make test-proof-engine
   beacon-chain-tests:
     name: beacon-chain-tests
     needs: [check-labels]
@@ -140,12 +168,14 @@ jobs:
       uses: moonrepo/setup-rust@v1
       with:
           channel: stable
+          cache-extra-identifier: ${{ env.RUST_CACHE_EXTRA_IDENTIFIER }}
           cache-target: release
           bins: cargo-nextest
     - if: github.repository == 'sigp/lighthouse'
       uses: Swatinem/rust-cache@v2
       with:
           cache-provider: warpbuild
+          key: ${{ env.RUST_CACHE_EXTRA_IDENTIFIER }}
     - name: Run beacon_chain tests for all known forks
       run: make test-beacon-chain
   http-api-tests:
@@ -162,12 +192,14 @@ jobs:
       uses: moonrepo/setup-rust@v1
       with:
           channel: stable
+          cache-extra-identifier: ${{ env.RUST_CACHE_EXTRA_IDENTIFIER }}
           cache-target: release
           bins: cargo-nextest
     - if: github.repository == 'sigp/lighthouse'
       uses: Swatinem/rust-cache@v2
       with:
           cache-provider: warpbuild
+          key: ${{ env.RUST_CACHE_EXTRA_IDENTIFIER }}
     - name: Run http_api tests for all recent forks
       run: make test-http-api
   op-pool-tests:
@@ -183,6 +215,7 @@ jobs:
       uses: moonrepo/setup-rust@v1
       with:
           channel: stable
+          cache-extra-identifier: ${{ env.RUST_CACHE_EXTRA_IDENTIFIER }}
           cache-target: release
           bins: cargo-nextest
     - name: Run operation_pool tests for all known forks
@@ -200,6 +233,7 @@ jobs:
       uses: moonrepo/setup-rust@v1
       with:
         channel: stable
+        cache-extra-identifier: ${{ env.RUST_CACHE_EXTRA_IDENTIFIER }}
         cache-target: release
         bins: cargo-nextest
     - name: Create CI logger dir
@@ -229,6 +263,7 @@ jobs:
       uses: moonrepo/setup-rust@v1
       with:
           channel: stable
+          cache-extra-identifier: ${{ env.RUST_CACHE_EXTRA_IDENTIFIER }}
           cache-target: release
           bins: cargo-nextest
     - name: Run slasher tests for all supported backends
@@ -247,11 +282,13 @@ jobs:
       uses: moonrepo/setup-rust@v1
       with:
           channel: stable
+          cache-extra-identifier: ${{ env.RUST_CACHE_EXTRA_IDENTIFIER }}
           bins: cargo-nextest
     - if: github.repository == 'sigp/lighthouse'
       uses: Swatinem/rust-cache@v2
       with:
           cache-provider: warpbuild
+          key: ${{ env.RUST_CACHE_EXTRA_IDENTIFIER }}
     - name: Run tests in debug
       run: make test-debug
   state-transition-vectors-ubuntu:
@@ -265,6 +302,7 @@ jobs:
       uses: moonrepo/setup-rust@v1
       with:
           channel: stable
+          cache-extra-identifier: ${{ env.RUST_CACHE_EXTRA_IDENTIFIER }}
           cache-target: release
     - name: Run state_transition_vectors in release.
       run: make run-state-transition-tests
@@ -282,12 +320,14 @@ jobs:
       uses: moonrepo/setup-rust@v1
       with:
           channel: stable
+          cache-extra-identifier: ${{ env.RUST_CACHE_EXTRA_IDENTIFIER }}
           cache-target: release
           bins: cargo-nextest
     - if: github.repository == 'sigp/lighthouse'
       uses: Swatinem/rust-cache@v2
       with:
           cache-provider: warpbuild
+          key: ${{ env.RUST_CACHE_EXTRA_IDENTIFIER }}
     - name: Run consensus-spec-tests with blst and fake_crypto
       run: make test-ef
   basic-simulator-ubuntu:
@@ -301,10 +341,11 @@ jobs:
       uses: moonrepo/setup-rust@v1
       with:
           channel: stable
+          cache-extra-identifier: ${{ env.RUST_CACHE_EXTRA_IDENTIFIER }}
           cache-target: release
     - name: Create log dir
       run: mkdir ${{ runner.temp }}/basic_simulator_logs
-    - name: Run a basic beacon chain sim that starts from Deneb
+    - name: Run a basic beacon chain sim
       run: cargo run --release --bin simulator basic-sim --disable-stdout-logging --log-dir ${{ runner.temp }}/basic_simulator_logs
     - name: Upload logs
       if: always()
@@ -323,6 +364,7 @@ jobs:
       uses: moonrepo/setup-rust@v1
       with:
           channel: stable
+          cache-extra-identifier: ${{ env.RUST_CACHE_EXTRA_IDENTIFIER }}
           cache-target: release
     - name: Create log dir
       run: mkdir ${{ runner.temp }}/fallback_simulator_logs
@@ -346,6 +388,7 @@ jobs:
       uses: moonrepo/setup-rust@v1
       with:
           channel: stable
+          cache-extra-identifier: ${{ env.RUST_CACHE_EXTRA_IDENTIFIER }}
           cache: false
       env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
@@ -362,6 +405,7 @@ jobs:
       uses: moonrepo/setup-rust@v1
       with:
           channel: stable
+          cache-extra-identifier: ${{ env.RUST_CACHE_EXTRA_IDENTIFIER }}
           cache-target: release
           components: rustfmt,clippy
           bins: cargo-audit,cargo-deny
@@ -410,6 +454,7 @@ jobs:
       uses: moonrepo/setup-rust@v1
       with:
           channel: nightly
+          cache-extra-identifier: ${{ env.RUST_CACHE_EXTRA_IDENTIFIER }}
           bins: cargo-udeps
           cache: false
       env:
@@ -447,6 +492,7 @@ jobs:
       uses: moonrepo/setup-rust@v1
       with:
         channel: stable
+        cache-extra-identifier: ${{ env.RUST_CACHE_EXTRA_IDENTIFIER }}
         cache-target: release
     - name: Run Makefile to trigger the bash script
       run: make cli-local
@@ -461,6 +507,7 @@ jobs:
       uses: moonrepo/setup-rust@v1
       with:
         channel: stable
+        cache-extra-identifier: ${{ env.RUST_CACHE_EXTRA_IDENTIFIER }}
     - uses: taiki-e/install-action@cargo-hack
     - name: Check types feature powerset
       run: cargo hack check -p types --feature-powerset --no-dev-deps --exclude-features arbitrary-fuzz,portable
@@ -477,6 +524,7 @@ jobs:
       uses: moonrepo/setup-rust@v1
       with:
         channel: stable
+        cache-extra-identifier: ${{ env.RUST_CACHE_EXTRA_IDENTIFIER }}
         cache-target: release
         bins: cargo-sort
     - name: Run cargo sort to check if Cargo.toml files are sorted