From 733dfe868d1ee3c6b1eb2bbf2c5d46c8b435734b Mon Sep 17 00:00:00 2001 From: Stefan Date: Thu, 3 Sep 2026 14:46:48 +0200 Subject: [PATCH] devnet-9: cut buildoor memory instead of resizing the hosts The four buildoors are the only 16 GB hosts in a 1002x32 GB fleet and are OOM-killed 7-27 times per hour. Measured per-container, the beacon node is 62-72% of RAM on three of them and every sidecar combined is ~1-2 GiB, so the fix belongs in the beacon config, not the instance size. Buildoors carry no validator range (only builder_index), so they never attest or aggregate, yet buildoor.yaml explicitly subscribed all five clients to every attestation subnet and additionally imported all attestations on lighthouse. On a 1M-validator network that pulls ~31k attestations per slot across 64 gossip meshes into a node that has no use for any of them. Drop the subnet-subscription flags and --import-all-attestations; keep the payload-preparation flags, which are the actual builder requirement. Also override the teku heap for this group: teku.yaml sets -Xmx12g for the validator fleet's 32 GB hosts, which cannot fit on a 16 GB buildoor alongside the EL and sidecars. Claude-Session: https://claude.ai/code/session_01LgfRTaHjnv57ASe51yw1D6 --- .../devnet-9/group_vars/buildoor.yaml | 28 +++++++++++++++---- 1 file changed, 22 insertions(+), 6 deletions(-) diff --git a/ansible/inventories/devnet-9/group_vars/buildoor.yaml b/ansible/inventories/devnet-9/group_vars/buildoor.yaml index 82536d10..95243cba 100644 --- a/ansible/inventories/devnet-9/group_vars/buildoor.yaml +++ b/ansible/inventories/devnet-9/group_vars/buildoor.yaml @@ -34,23 +34,39 @@ buildoor_container_env: BUILDOOR_INJECT_HEAD_HTML: -# Re-apply the prysm.yaml merge so we don't lose the bootnode args while adding --prepare-all-payloads and --subscribe-all-subnets +# Builders carry no validators (only `builder_index`), so they never attest or +# aggregate and have no use for the 64 attestation subnets. Subscribing to all of +# them on a 1M-validator network pulls in every attestation each slot (~31k) +# instead of the backbone a non-validating node needs, and the resulting gossip +# buffers and per-mesh peer state are the bulk of the beacon container's RSS. +# Payload-preparation flags are the actual builder requirement and are kept. +# Re-apply the prysm.yaml merge so we don't lose the bootnode args. prysm_container_command_extra_args: >- {{ prysm_container_command_extra_simple_args + prysm_container_command_extra_bootnode_args - + ['--prepare-all-payloads', '--subscribe-all-subnets'] }} + + ['--prepare-all-payloads'] }} lighthouse_container_command_extra_args: >- {{ lighthouse_container_command_extra_simple_args - + ['--always-prepare-payload', '--subscribe-all-subnets', '--import-all-attestations'] }} + + ['--always-prepare-payload'] }} lodestar_container_command_extra_args: >- {{ lodestar_container_command_extra_simple_args - + ['--emitPayloadAttributes', '--subscribeAllSubnets'] }} + + ['--emitPayloadAttributes'] }} teku_container_command_extra_args: >- {{ teku_container_command_extra_simple_args - + ['--Xfork-choice-updated-always-send-payload-attributes=true', '--p2p-subscribe-all-subnets-enabled'] }} + + ['--Xfork-choice-updated-always-send-payload-attributes=true'] }} grandine_container_command_extra_args: >- {{ grandine_container_command_extra_simple_args - + ['--features=AlwaysPrepareExecutionPayload', '--subscribe-all-subnets'] }} + + ['--features=AlwaysPrepareExecutionPayload'] }} + +# teku.yaml sets -Xmx12g for the validator fleet's 32 GB hosts. A buildoor has +# 16 GB and also runs the EL (~1.1 GiB) plus sidecars (~0.9 GiB); a 12 GiB heap +# plus JVM off-heap overhead does not fit, which is why this host has the highest +# restart count in the fleet. `ansible_group_priority=100` lets us override it here. +teku_container_env: + JAVA_OPTS: "-Xmx8g" + VIRTUAL_HOST: "{{ ethereum_node_beacon_hostname }}" + VIRTUAL_PORT: "{{ ethereum_node_cl_ports_http_beacon | string }}" + LETSENCRYPT_HOST: "{{ ethereum_node_beacon_hostname }}" ethereum_node_docker_watchtower_containers_list: