From 480c26a894ab59b48b13c70749e18e7aa2e3ba94 Mon Sep 17 00:00:00 2001 From: Daniel Noland Date: Fri, 14 Aug 2026 21:36:28 -0600 Subject: [PATCH 01/10] build(nix): Keep debug outputs lean Debug binaries retained the complete Rust toolchain through their standard-library source paths, adding roughly 2.4 GB to the closure. They also carried a sizable DWARF index that neither packaged debugger consumes. Point those paths at the much smaller rust-src component and remove .debug_names. Source browsing and symbols remain available while the resulting diagnostic images become practical to store and transfer. Signed-off-by: Daniel Noland Co-Authored-By: Claude Opus 5 (1M context) --- default.nix | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/default.nix b/default.nix index f99c89405e..31990b3f9e 100644 --- a/default.nix +++ b/default.nix @@ -461,6 +461,8 @@ let # Keep debug paths stable across revisions. Source readers # must resolve this relative prefix from the workspace root. "--remap-path-prefix==${src-prefix}" + # Keep debug outputs from retaining the complete Rust toolchain. + "--remap-path-prefix=${pkgs.rust-toolchain}/lib/rustlib/src/rust=${pkgs.rust-toolchain.passthru.availableComponents.rust-src}/lib/rustlib/src/rust" ] ) else @@ -500,6 +502,13 @@ let mkdir -p $debug/bin for f in $out/bin/*; do mv "$f" "$debug/bin/$(basename "$f")" + # Trade index for size. gdb has consumed `.debug_names` + # as a real DWARF-5 index since 14, and this ships 17.2, + # so dropping it is not free -- gdb rebuilds an index on + # each start instead. The section is large enough on + # these binaries that the image is worth more than the + # startup, and bugstalker does not read it at all. + ${objcopy} --remove-section=.debug_names "$debug/bin/$(basename "$f")" ${strip} --strip-debug "$debug/bin/$(basename "$f")" -o "$f" ${objcopy} --add-gnu-debuglink="$debug/bin/$(basename "$f")" "$f" done From cdc0b7ea19edfa3db5598dc184b4dc1a92305a1f Mon Sep 17 00:00:00 2001 From: Daniel Noland Date: Fri, 14 Aug 2026 21:37:45 -0600 Subject: [PATCH 02/10] feat(debug): Add a version-matched core viewer A core collected from the lab is useful only with the exact unstripped binaries and sources that produced it. A general debugging toolbox cannot reconstruct that relationship after the release has moved on. Provide a purpose-built gdb image alongside each build and teach it Rust's standard-library types without retaining rustc. This keeps post-mortem debugging reproducible while avoiding unrelated live-debugging tools. Signed-off-by: Daniel Noland Co-Authored-By: Claude Opus 5 (1M context) --- .github/workflows/README.md | 15 ++++++- .github/workflows/dev.yml | 2 + default.nix | 82 ++++++++++++++++++++++++++----------- justfile | 17 ++++---- 4 files changed, 84 insertions(+), 32 deletions(-) diff --git a/.github/workflows/README.md b/.github/workflows/README.md index d28097881f..0c1ef549ef 100644 --- a/.github/workflows/README.md +++ b/.github/workflows/README.md @@ -103,7 +103,8 @@ If those queue failures stop being rare, the phasing is worth revisiting. - Checks: `debug` by default; `release` and `fuzz` on deep runs - Coverage: `debug` by default; `fuzz` on deep runs - Miri: required on deep runs; opt-in on pull requests with `ci:+miri` -- Containers: debug/release for dataplane and FRR; release for validator +- Containers: debug/release for dataplane, its debugger, and FRR; release for + validator - VLAB configurations: spine-leaf fabric mode, L2VNI/L3VNI VPC modes, with gateway enabled @@ -111,6 +112,18 @@ If those queue failures stop being rare, the phasing is worth revisiting. - Container images pushed to GitHub Container Registry (GHCR) - Release containers published on tag pushes via `just push` +- `ghcr.io/githedgehog/dataplane/core-viewer` opens a core file from the lab. + It carries gdb plus the unstripped binaries and sources for the matching + `ghcr.io/githedgehog/dataplane` build. + Pull the tag matching the build the core came from; symbols only line up with + the exact version and profile that produced it. + The entrypoint takes the core as its only argument: + + ```console + docker run --rm -it -v /path/to/cores:/cores \ + ghcr.io/githedgehog/dataplane/core-viewer:TAG /cores/core.1234 + ``` + - Coverage reports from each `coverage/` job, kept for 7 days: - `coverage-html-.tar.gz` - `llvm-cov` HTML report, including the per-branch counts that Codecov does not render. Unpack and open diff --git a/.github/workflows/dev.yml b/.github/workflows/dev.yml index e368da318b..5c3792613c 100644 --- a/.github/workflows/dev.yml +++ b/.github/workflows/dev.yml @@ -457,6 +457,8 @@ jobs: nix-target: - frr.dataplane - dataplane + # Must match the build that produced the core. + - dataplane-core-viewer - validator # TODO: enable cfi and safe-stack on release when possible profile: "${{ fromJSON(needs.plan.outputs.container_profiles) }}" diff --git a/default.nix b/default.nix index 31990b3f9e..6c3d7e9f7f 100644 --- a/default.nix +++ b/default.nix @@ -1006,32 +1006,68 @@ let }).overrideAttrs source-volatile; - containers.dataplane-debugger = + # Shared runtime and unstripped binaries for the debugger images. + debug-image-paths = [ + pkgs.pkgsBuildHost.coreutils + pkgs.pkgsBuildHost.bashInteractive + pkgs.pkgsHostHost.dockerTools.usrBinEnv + + pkgs.pkgsHostHost.libc.debug + workspace.cli.debug + workspace.dataplane.debug + workspace.init.debug + ]; + + # Copy Rust's gdb helpers without retaining rustc as a runtime dependency. + rust-gdb-printers = pkgs.runCommand "rust-gdb-printers" { } '' + mkdir -p "$out/lib/rustlib/etc" + for f in gdb_load_rust_pretty_printers.py gdb_lookup.py gdb_providers.py rust_types.py; do + cp -L "${pkgs.rust-toolchain}/lib/rustlib/etc/$f" "$out/lib/rustlib/etc/$f" + done + ''; + + # Opens dataplane core files with matching symbols and sources. + containers.dataplane-core-viewer = (pkgs.dockerTools.buildLayeredImage { - name = "ghcr.io/githedgehog/dataplane/debugger"; + name = "ghcr.io/githedgehog/dataplane/core-viewer"; inherit tag; - contents = pkgs.buildEnv { - name = "dataplane-debugger-env"; - pathsToLink = [ - "/bin" - "/etc" - "/var" - "/lib" - ]; - paths = [ - pkgs.pkgsBuildHost.gdb - pkgs.pkgsBuildHost.rr - pkgs.pkgsBuildHost.coreutils - pkgs.pkgsBuildHost.bashInteractive - pkgs.pkgsBuildHost.iproute2 - pkgs.pkgsBuildHost.ethtool - pkgs.pkgsHostHost.dockerTools.usrBinEnv - - pkgs.pkgsHostHost.libc.debug - workspace.cli.debug - workspace.dataplane.debug - workspace.init.debug + contents = + (pkgs.buildEnv { + name = "dataplane-core-viewer-env"; + pathsToLink = [ + "/bin" + "/etc" + "/var" + "/lib" + ]; + paths = [ + pkgs.pkgsBuildHost.gdb + rust-gdb-printers + ] + ++ debug-image-paths; + }).overrideAttrs + source-volatile; + # gdb needs a writable HOME for logs and its index cache. + extraCommands = '' + # Point `src-prefix` at the sources this image ships. Referencing ${src} + # here is also what keeps it in the image closure: with the remap no + # longer naming a store path, nothing else retains it. + mkdir -p ".$(dirname "${src-prefix}")" + ln -s "${src}" ".${src-prefix}" + mkdir -p tmp + chmod 1777 tmp + ''; + config = { + Entrypoint = [ + "/bin/gdb" + "--directory=/lib/rustlib/etc" + "-iex" + "add-auto-load-safe-path /lib/rustlib/etc" + "-iex" + "source /lib/rustlib/etc/gdb_load_rust_pretty_printers.py" + "/bin/dataplane" ]; + Env = [ "HOME=/tmp" ]; }; }).overrideAttrs source-volatile; diff --git a/justfile b/justfile index 25185d14d5..c4b6f9af40 100644 --- a/justfile +++ b/justfile @@ -182,7 +182,7 @@ oci_insecure := "" oci_name := "githedgehog/dataplane" oci_frr_prefix := "githedgehog/dataplane/frr" oci_image_dataplane := oci_repo + "/" + oci_name + ":" + version -oci_image_dataplane_debugger := oci_repo + "/" + oci_name + "/debugger:" + version +oci_image_dataplane_core_viewer := oci_repo + "/" + oci_name + "/core-viewer:" + version oci_image_dataplane_validator := oci_repo + "/" + oci_name + "/validator:" + version oci_image_frr_dataplane := oci_repo + "/" + oci_frr_prefix + ":" + version oci_image_frr_host := oci_repo + "/" + oci_frr_prefix + "-host:" + version @@ -550,10 +550,10 @@ build-container target="dataplane" *args: _refuse-instrumented-artifact (build ( docker tag "${img}" "{{oci_image_dataplane}}" echo "imported {{ oci_image_dataplane }} (${docker_platform})" ;; - "dataplane-debugger") - docker load < ./results/containers.dataplane-debugger - docker tag "ghcr.io/githedgehog/dataplane/debugger:{{version}}" "{{oci_image_dataplane_debugger}}" - echo "imported {{ oci_image_dataplane_debugger }}" + "dataplane-core-viewer") + docker load < ./results/containers.dataplane-core-viewer + docker tag "ghcr.io/githedgehog/dataplane/core-viewer:{{version}}" "{{oci_image_dataplane_core_viewer}}" + echo "imported {{ oci_image_dataplane_core_viewer }}" ;; "debug-tools") # Uses nix only to produce a base image with the runtime closure (glibc, bash, etc.) @@ -657,8 +657,8 @@ push-container target="dataplane" *args: (build-container target args) && versio "dataplane") push_image "{{ oci_image_dataplane }}" ;; - "dataplane-debugger") - push_image "{{ oci_image_dataplane_debugger }}" + "dataplane-core-viewer") + push_image "{{ oci_image_dataplane_core_viewer }}" ;; "debug-tools") >&2 echo "do not push the debug tools!" @@ -692,7 +692,8 @@ push-container target="dataplane" *args: (build-container target args) && versio [script] push: {{ _just_debuggable_ }} - for container in dataplane frr.dataplane validator; do + # The core viewer must match the release it inspects. + for container in dataplane dataplane-core-viewer frr.dataplane validator; do if [ "${container}" = "validator" ]; then platform="wasm32-wasip1" else From ae94234e8fa3dc30a67c43bba2a1fc2591ea0f75 Mon Sep 17 00:00:00 2001 From: Daniel Noland Date: Fri, 14 Aug 2026 21:38:11 -0600 Subject: [PATCH 03/10] feat(debug): Add a live DAP debugger Post-mortem inspection and live debugging need different tools. The core viewer cannot offer an editor-driven session, while bugstalker understands Rust layouts and can expose the running dataplane through the Debug Adapter Protocol. Track bugstalker upstream for its current remote DAP support and package it separately with the matching binaries and sources. Keeping the image single-purpose avoids making every diagnostic artifact carry every debugger. Signed-off-by: Daniel Noland Co-Authored-By: Claude Opus 5 (1M context) --- .github/workflows/README.md | 49 ++++++++++++++++++++++++++++++++-- .github/workflows/dev.yml | 2 ++ default.nix | 48 +++++++++++++++++++++++++++++++++ justfile | 9 +++++++ nix/overlays/dataplane-dev.nix | 9 +++++++ npins/sources.json | 16 +++++++++++ 6 files changed, 131 insertions(+), 2 deletions(-) diff --git a/.github/workflows/README.md b/.github/workflows/README.md index 0c1ef549ef..9fcce2c5cf 100644 --- a/.github/workflows/README.md +++ b/.github/workflows/README.md @@ -103,8 +103,8 @@ If those queue failures stop being rare, the phasing is worth revisiting. - Checks: `debug` by default; `release` and `fuzz` on deep runs - Coverage: `debug` by default; `fuzz` on deep runs - Miri: required on deep runs; opt-in on pull requests with `ci:+miri` -- Containers: debug/release for dataplane, its debugger, and FRR; release for - validator +- Containers: debug/release for dataplane, its two debug images, and FRR; + release for validator - VLAB configurations: spine-leaf fabric mode, L2VNI/L3VNI VPC modes, with gateway enabled @@ -124,6 +124,51 @@ If those queue failures stop being rare, the phasing is worth revisiting. ghcr.io/githedgehog/dataplane/core-viewer:TAG /cores/core.1234 ``` +- `ghcr.io/githedgehog/dataplane/dev-debugger` debugs a live dataplane from an + editor. It carries bugstalker, which understands Rust's std collections and + enum layouts, and listens for a Debug Adapter Protocol client on port 4711. + Publish the port and point the editor's DAP client at it: + + ```console + docker run --rm -p 127.0.0.1:4711:4711 ghcr.io/githedgehog/dataplane/dev-debugger:TAG + ``` + + Connecting does not by itself start anything. In remote-DAP mode bugstalker + waits for the client's `launch` request to name the program, so the editor + has to send `program`, and any dataplane arguments as `args`. A request + without `program` is rejected with `launch: missing arguments.program`. + For VS Code, in `.vscode/launch.json`. `type` has to match whatever debug + type the BugStalker extension you installed registers -- it is not a name we + choose, and it differs between extensions, so check the one you have rather + than copying this field blind: + + ```json + { + "type": "bs", + "request": "launch", + "name": "dataplane (container)", + "debugServer": 4711, + "program": "/bin/dataplane", + "args": [] + } + ``` + + For `nvim-dap`, where the first line names the adapter itself, so `type = "bs"` + below is our own label rather than an extension's: + + ```lua + dap.adapters.bs = { type = "server", host = "127.0.0.1", port = 4711 } + dap.configurations.rust = { + { + type = "bs", + request = "launch", + name = "dataplane (container)", + program = "/bin/dataplane", + args = {}, + }, + } + ``` + - Coverage reports from each `coverage/` job, kept for 7 days: - `coverage-html-.tar.gz` - `llvm-cov` HTML report, including the per-branch counts that Codecov does not render. Unpack and open diff --git a/.github/workflows/dev.yml b/.github/workflows/dev.yml index 5c3792613c..6bd886a8af 100644 --- a/.github/workflows/dev.yml +++ b/.github/workflows/dev.yml @@ -459,6 +459,8 @@ jobs: - dataplane # Must match the build that produced the core. - dataplane-core-viewer + # Live debugging of the same build, driven from an editor over DAP. + - dataplane-dev-debugger - validator # TODO: enable cfi and safe-stack on release when possible profile: "${{ fromJSON(needs.plan.outputs.container_profiles) }}" diff --git a/default.nix b/default.nix index 6c3d7e9f7f..e196b652c1 100644 --- a/default.nix +++ b/default.nix @@ -1072,6 +1072,54 @@ let }).overrideAttrs source-volatile; + # Exposes bugstalker's DAP server for live debugging. + containers.dataplane-dev-debugger = + (pkgs.dockerTools.buildLayeredImage { + name = "ghcr.io/githedgehog/dataplane/dev-debugger"; + inherit tag; + contents = + (pkgs.buildEnv { + name = "dataplane-dev-debugger-env"; + pathsToLink = [ + "/bin" + "/etc" + "/var" + "/lib" + ]; + paths = [ pkgs.pkgsBuildHost.bugstalker ] ++ debug-image-paths; + }).overrideAttrs + source-volatile; + # bugstalker needs a writable HOME for its keymap and history. + extraCommands = '' + # Point `src-prefix` at the sources this image ships. Referencing ${src} + # here is also what keeps it in the image closure: with the remap no + # longer naming a store path, nothing else retains it. + mkdir -p ".$(dirname "${src-prefix}")" + ln -s "${src}" ".${src-prefix}" + mkdir -p tmp + chmod 1777 tmp + ''; + config = { + Entrypoint = [ + "/bin/bs" + # Bind the published interface rather than container-local loopback. + "--dap-remote=0.0.0.0:4711" + # rustc is absent, so bugstalker cannot infer this path. + "--std-lib-path=${pkgs.rust-toolchain.passthru.availableComponents.rust-src}/lib/rustlib/src/rust" + # No debuggee here on purpose. In `--dap-remote` mode bugstalker + # ignores the CLI debuggee and waits for the client's `launch` request + # to name one, so a path here would be silently dead and would imply + # that connecting alone starts the dataplane. The editor supplies + # `program` instead; see .github/workflows/README.md. + ]; + Env = [ "HOME=/tmp" ]; + ExposedPorts = { + "4711/tcp" = { }; + }; + }; + }).overrideAttrs + source-volatile; + debug-tools = pkgs: [ diff --git a/justfile b/justfile index c4b6f9af40..c99f6d77f8 100644 --- a/justfile +++ b/justfile @@ -183,6 +183,7 @@ oci_name := "githedgehog/dataplane" oci_frr_prefix := "githedgehog/dataplane/frr" oci_image_dataplane := oci_repo + "/" + oci_name + ":" + version oci_image_dataplane_core_viewer := oci_repo + "/" + oci_name + "/core-viewer:" + version +oci_image_dataplane_dev_debugger := oci_repo + "/" + oci_name + "/dev-debugger:" + version oci_image_dataplane_validator := oci_repo + "/" + oci_name + "/validator:" + version oci_image_frr_dataplane := oci_repo + "/" + oci_frr_prefix + ":" + version oci_image_frr_host := oci_repo + "/" + oci_frr_prefix + "-host:" + version @@ -555,6 +556,11 @@ build-container target="dataplane" *args: _refuse-instrumented-artifact (build ( docker tag "ghcr.io/githedgehog/dataplane/core-viewer:{{version}}" "{{oci_image_dataplane_core_viewer}}" echo "imported {{ oci_image_dataplane_core_viewer }}" ;; + "dataplane-dev-debugger") + docker load < ./results/containers.dataplane-dev-debugger + docker tag "ghcr.io/githedgehog/dataplane/dev-debugger:{{version}}" "{{oci_image_dataplane_dev_debugger}}" + echo "imported {{ oci_image_dataplane_dev_debugger }}" + ;; "debug-tools") # Uses nix only to produce a base image with the runtime closure (glibc, bash, etc.) # then layers locally-compiled cargo binaries on top via Dockerfile. @@ -660,6 +666,9 @@ push-container target="dataplane" *args: (build-container target args) && versio "dataplane-core-viewer") push_image "{{ oci_image_dataplane_core_viewer }}" ;; + "dataplane-dev-debugger") + push_image "{{ oci_image_dataplane_dev_debugger }}" + ;; "debug-tools") >&2 echo "do not push the debug tools!" exit 1 diff --git a/nix/overlays/dataplane-dev.nix b/nix/overlays/dataplane-dev.nix index 881cccf0f6..8ed43ba09f 100644 --- a/nix/overlays/dataplane-dev.nix +++ b/nix/overlays/dataplane-dev.nix @@ -30,6 +30,15 @@ in inherit (override-packages) rustPlatform; version = "0.16.1"; }; + # cargoDeps must be fetched from the overridden source too. + bugstalker = prev.bugstalker.overrideAttrs (orig: { + version = final.lib.removePrefix "v" sources.bugstalker.version; + src = sources.bugstalker; + cargoDeps = prev.rustPlatform.fetchCargoVendor { + src = sources.bugstalker; + hash = "sha256-GGi5hnrK5WpvnXHNckpsBch/SJ4lDvH7peSlrCdk218="; + }; + }); cargo-bolero = prev.cargo-bolero.override { inherit (override-packages) rustPlatform; }; cargo-deny = prev.cargo-deny.override { inherit (override-packages) rustPlatform; }; cargo-edit = prev.cargo-edit.override { inherit (override-packages) rustPlatform; }; diff --git a/npins/sources.json b/npins/sources.json index 4c6020af08..2f5ce3ac94 100644 --- a/npins/sources.json +++ b/npins/sources.json @@ -16,6 +16,22 @@ "url": "https://api.github.com/repos/KaTeX/KaTeX/tarball/refs/tags/v0.18.4", "hash": "sha256-Z458Crgd7o68C1pKE9qf5nJhMAs5D+uyK4nbbtJO/lg=" }, + "bugstalker": { + "type": "GitRelease", + "repository": { + "type": "GitHub", + "owner": "godzie44", + "repo": "BugStalker" + }, + "pre_releases": false, + "version_upper_bound": null, + "release_prefix": null, + "submodules": false, + "version": "v0.4.7", + "revision": "9c18e546eca6a1d68ef69da340a6fb0f2bc1bab7", + "url": "https://api.github.com/repos/godzie44/BugStalker/tarball/refs/tags/v0.4.7", + "hash": "sha256-AAeSvy/rvyylPH2jTVBGN95QIc6gumREQYuruhRo2ZI=" + }, "crane": { "type": "GitRelease", "repository": { From 752c2d43055cfe308a0173868391a99fb2b871cb Mon Sep 17 00:00:00 2001 From: Daniel Noland Date: Fri, 14 Aug 2026 21:38:36 -0600 Subject: [PATCH 04/10] feat(debug): Add a syscall tracer image Some failures need a record of the dataplane's kernel interactions rather than an interactive debugger. A small, repeatable tracing environment is easier to deploy and feed into existing log analysis than a general-purpose toolbox. Package lurk around the matching release binaries and follow the worker threads where the dataplane does its work. Because syscall tracing needs no symbols, this image can stay much smaller than the debugger images. Signed-off-by: Daniel Noland Co-Authored-By: Claude Opus 5 (1M context) --- .github/workflows/README.md | 22 +++++++++++++++- .github/workflows/dev.yml | 2 ++ default.nix | 47 ++++++++++++++++++++++++++++++++++ justfile | 9 +++++++ nix/overlays/dataplane-dev.nix | 27 +++++++++++++++++++ 5 files changed, 106 insertions(+), 1 deletion(-) diff --git a/.github/workflows/README.md b/.github/workflows/README.md index 9fcce2c5cf..a036099f08 100644 --- a/.github/workflows/README.md +++ b/.github/workflows/README.md @@ -103,7 +103,7 @@ If those queue failures stop being rare, the phasing is worth revisiting. - Checks: `debug` by default; `release` and `fuzz` on deep runs - Coverage: `debug` by default; `fuzz` on deep runs - Miri: required on deep runs; opt-in on pull requests with `ci:+miri` -- Containers: debug/release for dataplane, its two debug images, and FRR; +- Containers: debug/release for dataplane, its three debug images, and FRR; release for validator - VLAB configurations: spine-leaf fabric mode, L2VNI/L3VNI VPC modes, with gateway enabled @@ -169,6 +169,26 @@ If those queue failures stop being rare, the phasing is worth revisiting. } ``` +- `ghcr.io/githedgehog/dataplane/syscall-tracer` records what the dataplane + asks the kernel for, as JSON, using lurk. + It carries the same stripped binaries the release image ships, since nothing + here symbolizes, so it is smaller than the other two -- though not by as much + as that suggests: like them it ships the source tree, which the entrypoint + makes the working directory. Only the debug symbols and the debuggers + themselves are absent. + + ```console + docker run --rm ghcr.io/githedgehog/dataplane/syscall-tracer:TAG > trace.jsonl + ``` + + The stream is one JSON object per line, except that tracing child threads + makes lurk announce each one with a bare `Attaching to child ` line. + Filter those out if the consumer needs strict JSONL: + + ```console + jq -R 'fromjson? // empty' < trace.jsonl + ``` + - Coverage reports from each `coverage/` job, kept for 7 days: - `coverage-html-.tar.gz` - `llvm-cov` HTML report, including the per-branch counts that Codecov does not render. Unpack and open diff --git a/.github/workflows/dev.yml b/.github/workflows/dev.yml index 6bd886a8af..80ef38adb4 100644 --- a/.github/workflows/dev.yml +++ b/.github/workflows/dev.yml @@ -461,6 +461,8 @@ jobs: - dataplane-core-viewer # Live debugging of the same build, driven from an editor over DAP. - dataplane-dev-debugger + # Syscall trace of the same build, as JSON. + - dataplane-syscall-tracer - validator # TODO: enable cfi and safe-stack on release when possible profile: "${{ fromJSON(needs.plan.outputs.container_profiles) }}" diff --git a/default.nix b/default.nix index e196b652c1..5575db8daf 100644 --- a/default.nix +++ b/default.nix @@ -1120,6 +1120,53 @@ let }).overrideAttrs source-volatile; + # Traces the release binaries' syscalls as JSON with lurk. + containers.dataplane-syscall-tracer = + (pkgs.dockerTools.buildLayeredImage { + name = "ghcr.io/githedgehog/dataplane/syscall-tracer"; + inherit tag; + contents = + (pkgs.buildEnv { + name = "dataplane-syscall-tracer-env"; + pathsToLink = [ + "/bin" + "/etc" + "/var" + "/lib" + ]; + paths = [ + pkgs.pkgsBuildHost.lurk + pkgs.pkgsHostHost.dockerTools.fakeNss + pkgs.pkgsHostHost.busybox + pkgs.pkgsHostHost.dockerTools.usrBinEnv + workspace.cli + workspace.dataplane + workspace.init + ]; + }).overrideAttrs + source-volatile; + extraCommands = '' + # Point `src-prefix` at the sources this image ships. Referencing ${src} + # here is also what keeps it in the image closure: with the remap no + # longer naming a store path, nothing else retains it. + mkdir -p ".$(dirname "${src-prefix}")" + ln -s "${src}" ".${src-prefix}" + mkdir -p tmp + chmod 1777 tmp + ''; + config = { + Entrypoint = [ + "/bin/lurk" + "--json" + # Include the worker threads where the dataplane does its work. + "--follow-forks" + "/bin/dataplane" + ]; + Env = [ "HOME=/tmp" ]; + }; + }).overrideAttrs + source-volatile; + debug-tools = pkgs: [ diff --git a/justfile b/justfile index c99f6d77f8..6b883b179f 100644 --- a/justfile +++ b/justfile @@ -184,6 +184,7 @@ oci_frr_prefix := "githedgehog/dataplane/frr" oci_image_dataplane := oci_repo + "/" + oci_name + ":" + version oci_image_dataplane_core_viewer := oci_repo + "/" + oci_name + "/core-viewer:" + version oci_image_dataplane_dev_debugger := oci_repo + "/" + oci_name + "/dev-debugger:" + version +oci_image_dataplane_syscall_tracer := oci_repo + "/" + oci_name + "/syscall-tracer:" + version oci_image_dataplane_validator := oci_repo + "/" + oci_name + "/validator:" + version oci_image_frr_dataplane := oci_repo + "/" + oci_frr_prefix + ":" + version oci_image_frr_host := oci_repo + "/" + oci_frr_prefix + "-host:" + version @@ -561,6 +562,11 @@ build-container target="dataplane" *args: _refuse-instrumented-artifact (build ( docker tag "ghcr.io/githedgehog/dataplane/dev-debugger:{{version}}" "{{oci_image_dataplane_dev_debugger}}" echo "imported {{ oci_image_dataplane_dev_debugger }}" ;; + "dataplane-syscall-tracer") + docker load < ./results/containers.dataplane-syscall-tracer + docker tag "ghcr.io/githedgehog/dataplane/syscall-tracer:{{version}}" "{{oci_image_dataplane_syscall_tracer}}" + echo "imported {{ oci_image_dataplane_syscall_tracer }}" + ;; "debug-tools") # Uses nix only to produce a base image with the runtime closure (glibc, bash, etc.) # then layers locally-compiled cargo binaries on top via Dockerfile. @@ -669,6 +675,9 @@ push-container target="dataplane" *args: (build-container target args) && versio "dataplane-dev-debugger") push_image "{{ oci_image_dataplane_dev_debugger }}" ;; + "dataplane-syscall-tracer") + push_image "{{ oci_image_dataplane_syscall_tracer }}" + ;; "debug-tools") >&2 echo "do not push the debug tools!" exit 1 diff --git a/nix/overlays/dataplane-dev.nix b/nix/overlays/dataplane-dev.nix index 8ed43ba09f..6b7f47b929 100644 --- a/nix/overlays/dataplane-dev.nix +++ b/nix/overlays/dataplane-dev.nix @@ -39,6 +39,33 @@ in hash = "sha256-GGi5hnrK5WpvnXHNckpsBch/SJ4lDvH7peSlrCdk218="; }; }); + # lurk disables ASLR in the tracee before exec, and treats failure as fatal. + # Docker's default seccomp profile answers personality(ADDR_NO_RANDOMIZE) with + # EPERM, so under a plain `docker run` the traced program never starts -- and + # lurk still exits 0 after emitting a well-formed JSON trace of its own child + # failing, which the `jq -R 'fromjson? // empty'` filter we document accepts + # without complaint. + # + # Nothing in the tracer image symbolizes an address, so a fixed layout buys us + # nothing. Make it advisory rather than telling users to pass + # `--security-opt seccomp=unconfined`, which drops confinement on a container + # whose whole job is ptracing a process. + # + # Two single-line substitutions rather than one spanning both: nix strips the + # common indentation from an indented string, so a multi-line search pattern + # would not match the source's own indentation. + lurk = prev.lurk.overrideAttrs (orig: { + postPatch = (orig.postPatch or "") + '' + substituteInPlace src/lib.rs \ + --replace-fail \ + 'personality::set(Persona::ADDR_NO_RANDOMIZE)' \ + 'let _ = personality::set(Persona::ADDR_NO_RANDOMIZE);' \ + --replace-fail \ + '.map_err(|_| anyhow!("Unable to set ADDR_NO_RANDOMIZE"))?;' \ + "" + ''; + }); + cargo-bolero = prev.cargo-bolero.override { inherit (override-packages) rustPlatform; }; cargo-deny = prev.cargo-deny.override { inherit (override-packages) rustPlatform; }; cargo-edit = prev.cargo-edit.override { inherit (override-packages) rustPlatform; }; From afa2d5cf51556a9dcad850efd2358af43f1d7897 Mon Sep 17 00:00:00 2001 From: Daniel Noland Date: Fri, 14 Aug 2026 21:39:43 -0600 Subject: [PATCH 05/10] ci: Publish debug images on a deliberate cadence The diagnostic images are useful only when they match the build being investigated, but building roughly 850 MB of extra images for every pull request would undermine the runner-load reduction this CI rework is meant to achieve. Build them automatically for pushes, the merge queue, and manual runs, with an explicit label available for debugging a pull request. Publish all three beside tagged releases so the matching tools remain available when a deployed build needs investigation. Signed-off-by: Daniel Noland Co-Authored-By: Claude Opus 5 (1M context) --- .github/workflows/README.md | 10 ++++++++-- .github/workflows/dev.yml | 22 ++++++++++++---------- justfile | 10 ++++++++-- 3 files changed, 28 insertions(+), 14 deletions(-) diff --git a/.github/workflows/README.md b/.github/workflows/README.md index a036099f08..4f8fd6d86d 100644 --- a/.github/workflows/README.md +++ b/.github/workflows/README.md @@ -63,6 +63,11 @@ Production artifacts are produced via nix builds in a separate CI workflow. - `ci:+miri` - Run Miri checks - `ci:+wasm` - Run the WASM build check - `ci:+concurrency` - Run Shuttle and Loom tests +- `ci:+debug-images` - Also build and push the core viewer, DAP debugger, and + syscall tracer images. They are built on main, in the merge queue, and on + dispatch regardless, and `ci:+merge-ready` turns them on too, since + `ci-gate` treats that label as enabling every gate; this is for when the + build itself needs debugging - `ci:+cross` - Build all cross-platform containers - `ci:+cross/full` - Also run the workspace test suite under qemu-user, on the two aarch64 musl legs. Gated like every other job, so the merge queue and @@ -103,8 +108,9 @@ If those queue failures stop being rare, the phasing is worth revisiting. - Checks: `debug` by default; `release` and `fuzz` on deep runs - Coverage: `debug` by default; `fuzz` on deep runs - Miri: required on deep runs; opt-in on pull requests with `ci:+miri` -- Containers: debug/release for dataplane, its three debug images, and FRR; - release for validator +- Containers: debug/release for dataplane and FRR; release for validator +- Debug images (core viewer, DAP debugger, syscall tracer): deep runs only, + or on a pull request with `ci:+debug-images` - VLAB configurations: spine-leaf fabric mode, L2VNI/L3VNI VPC modes, with gateway enabled diff --git a/.github/workflows/dev.yml b/.github/workflows/dev.yml index 80ef38adb4..a11cf06647 100644 --- a/.github/workflows/dev.yml +++ b/.github/workflows/dev.yml @@ -68,6 +68,7 @@ jobs: outputs: container_profiles: "${{ steps.container-profiles.outputs.value }}" parallel: "${{ steps.parallel.outputs.value }}" + container_targets: "${{ steps.container-targets.outputs.value }}" profiles: "${{ steps.profiles.outputs.value }}" concurrency: "${{ steps.concurrency.outputs.value }}" cross: "${{ steps.cross.outputs.value }}" @@ -132,6 +133,16 @@ jobs: on-value: '["debug", "release", "fuzz"]' off-value: '["debug"]' + # Debug images are opt-in on pull requests because of their size. + - id: "container-targets" + uses: *gate + with: + labels: "debug-images" + # Keep this on one line: `ci-gate` writes the value to GITHUB_OUTPUT + # with a plain printf, which a multi-line value would corrupt. + on-value: '["frr.dataplane", "dataplane", "dataplane-core-viewer", "dataplane-dev-debugger", "dataplane-syscall-tracer", "validator"]' + off-value: '["frr.dataplane", "dataplane", "validator"]' + # Lab jobs require release images but not other release/fuzz checks. - id: "container-profiles" uses: *gate @@ -454,16 +465,7 @@ jobs: fail-fast: false max-parallel: ${{ fromJSON(needs.plan.outputs.parallel) }} matrix: - nix-target: - - frr.dataplane - - dataplane - # Must match the build that produced the core. - - dataplane-core-viewer - # Live debugging of the same build, driven from an editor over DAP. - - dataplane-dev-debugger - # Syscall trace of the same build, as JSON. - - dataplane-syscall-tracer - - validator + nix-target: "${{ fromJSON(needs.plan.outputs.container_targets) }}" # TODO: enable cfi and safe-stack on release when possible profile: "${{ fromJSON(needs.plan.outputs.container_profiles) }}" exclude: diff --git a/justfile b/justfile index 6b883b179f..ddd356f517 100644 --- a/justfile +++ b/justfile @@ -710,8 +710,14 @@ push-container target="dataplane" *args: (build-container target args) && versio [script] push: {{ _just_debuggable_ }} - # The core viewer must match the release it inspects. - for container in dataplane dataplane-core-viewer frr.dataplane validator; do + # Debug images must match the release they inspect. + for container in \ + dataplane \ + dataplane-core-viewer \ + dataplane-dev-debugger \ + dataplane-syscall-tracer \ + frr.dataplane \ + validator; do if [ "${container}" = "validator" ]; then platform="wasm32-wasip1" else From 74b29d90e5cfeba0aace6a8ac208683a73deac8e Mon Sep 17 00:00:00 2001 From: Daniel Noland Date: Sat, 15 Aug 2026 21:36:02 -0600 Subject: [PATCH 06/10] test(debug): Exercise the debug images instead of only building them Building proves an image links; it says nothing about whether its entrypoint runs. `smoke-container` runs each image the way the README tells a user to. The tracer's guard is an `execve` in the trace, not merely well-formed JSON: lurk can record its own child failing to start and still exit 0, which the documented `jq -R 'fromjson? // empty'` filter accepts without complaint. The core viewer goes through its own entrypoint, since the `--directory` and `source` flags that register the printers live there. The debugger is checked only for coming up and listening -- driving a real DAP session would mean carrying a protocol client in-tree, and an editor pointed at the image exercises that contract better. The trace goes to a file rather than a shell variable: at a few megabytes it overruns the here-string limit, and grep then fails with E2BIG, which reads exactly like a failed trace. Signed-off-by: Daniel Noland Co-Authored-By: Claude Opus 5 (1M context) --- .github/workflows/dev.yml | 11 ++++ ci.just | 4 ++ justfile | 105 ++++++++++++++++++++++++++++++++++++++ 3 files changed, 120 insertions(+) diff --git a/.github/workflows/dev.yml b/.github/workflows/dev.yml index a11cf06647..e7eb88c2d2 100644 --- a/.github/workflows/dev.yml +++ b/.github/workflows/dev.yml @@ -480,6 +480,17 @@ jobs: with: recipe: "ci::push-container" recipe_args: "${{ matrix.nix-target }} ${{ matrix.profile }} ${{ needs.version.outputs.version }}" + + # A debug image that builds is not a debug image that works. Two of + # these shipped with entrypoints that could not do what the README + # documents, and building them said nothing about it. + - name: "smoke" + if: "${{ startsWith(matrix.nix-target, 'dataplane-') }}" + uses: *just + with: + recipe: "ci::smoke-container" + recipe_args: "${{ matrix.nix-target }} ${{ matrix.profile }}" + - *verify-clean-tree - *tmate diff --git a/ci.just b/ci.just index f4c5406f8f..7f659c5ecc 100644 --- a/ci.just +++ b/ci.just @@ -78,6 +78,10 @@ cross platform libc +args: cross-test platform libc: NEXTEST_PROFILE=cross-qemu just {{ _lab }} platform={{ platform }} libc={{ libc }} profile=debug test +# Verify a debug image's entrypoint actually does its job. +smoke-container target profile: + just {{ _lab }} profile={{ profile }} platform=x86-64-v3 smoke-container {{ target }} + # Publish both content-derived and discoverable per-commit tags. [script] push-container target profile version: diff --git a/justfile b/justfile index ddd356f517..5c48f91b75 100644 --- a/justfile +++ b/justfile @@ -508,6 +508,111 @@ setup-roots *args: {{ args }} done +# Check that the debug images actually do what the README says they do. +# +# Building an image proves it links; it does not prove the entrypoint runs. +# Both of these shipped broken: the tracer's documented `docker run` produced a +# well-formed JSON trace of its own child failing to start, and still exited 0. +[script] +smoke-container target: (build-container (if target == "dataplane-core-viewer" { target } else if target == "dataplane-dev-debugger" { target } else if target == "dataplane-syscall-tracer" { target } else { error("smoke-container: no smoke test for '" + target + "'; expected dataplane-core-viewer, dataplane-dev-debugger, or dataplane-syscall-tracer") })) + {{ _just_debuggable_ }} + declare -xr DOCKER_HOST="${DOCKER_HOST:-unix://{{ docker_sock }}}" + case "{{ target }}" in + "dataplane-syscall-tracer") + # A trace runs to megabytes, so keep it in a file rather than a + # variable: `declare -x` would put it in the environment, and every + # child process then fails to exec with E2BIG, which reads exactly + # like a failed trace. Here-strings are fine at this size -- bash + # backs them with a pipe or temp file -- and the checks below use + # them on captured output. + declare trace + trace="$(mktemp)" + declare -r trace + # Name the container and remove it by name. `timeout` signals the + # docker CLI, and the daemon -- not the CLI -- owns the container's + # lifetime, so on the timeout path `--rm` alone can leave a `lurk` + # tracing something for as long as the runner lives. + declare cid + cid="smoke-syscall-tracer-$$" + declare -r cid + trap 'rm -f -- "${trace}"; docker rm -f "${cid}" >/dev/null 2>&1 || true' EXIT + # No seccomp relaxation on purpose: this is the documented command. + timeout 60 docker run --rm --name "${cid}" \ + "{{ oci_image_dataplane_syscall_tracer }}" \ + >"${trace}" 2>&1 || true + if grep -q "Unable to set ADDR_NO_RANDOMIZE" "${trace}"; then + >&2 echo "::error::lurk could not disable ASLR, so the tracee never ran" + exit 1 + fi + # The tracee has to actually execute, not merely be attached to. + if ! grep -q '"syscall":"execve"' "${trace}"; then + >&2 echo "::error::no execve in the trace: the traced program never started" + >&2 head -20 "${trace}" + exit 1 + fi + printf 'syscall-tracer: traced %s syscalls\n' "$(grep -c '"type":"SYSCALL"' "${trace}")" + ;; + "dataplane-dev-debugger") + # Only that it comes up and listens. Driving a DAP session from + # CI means carrying a protocol client for a contract better + # exercised by pointing a real editor at `just debug bugstalker`. + declare cid + # Let docker pick the host port and read it back. A fixed one + # collides: these run on a shared daemon, and `container_profiles` + # can put two of these jobs on the same node at once, where the + # second bind fails and reads as "the listener never came up". + cid="$(docker run -d --rm -p 127.0.0.1::4711 "{{ oci_image_dataplane_dev_debugger }}")" + declare -r cid + trap 'docker kill "${cid}" >/dev/null 2>&1 || true' EXIT + declare -i waited=0 + declare port + port="$(docker port "${cid}" 4711/tcp | head -1)" + port="${port##*:}" + declare -r port + if [ -z "${port}" ]; then + >&2 echo "::error::docker published no host port for 4711/tcp" + exit 1 + fi + until timeout 1 bash -c "/dev/null; do + if [ "${waited}" -ge 60 ]; then + >&2 echo "::error::dev-debugger never listened on 4711" + >&2 docker logs "${cid}" 2>&1 | tail -20 + exit 1 + fi + sleep 1 + waited=$(( waited + 1 )) + done + echo "dev-debugger: DAP listener up" + ;; + "dataplane-core-viewer") + # The Rust pretty-printers are the reason this image exists, and + # what registers them is the entrypoint's own `source` flag -- so + # drive the real entrypoint rather than invoking gdb directly, + # which would only test a copy of it. + declare out + out="$(printf 'info pretty-printer\nquit\n' \ + | timeout 120 docker run --rm -i "{{ oci_image_dataplane_core_viewer }}" 2>&1)" + if grep -qiE "traceback|no module named" <<<"${out}"; then + >&2 echo "::error::gdb could not load the rust pretty-printers" + >&2 printf '%s\n' "${out}" + exit 1 + fi + # A registered printer set, not merely a clean start. + for want in StdString StdVec StdHashMap; do + if ! grep -q "${want}" <<<"${out}"; then + >&2 echo "::error::rust pretty-printer ${want} is not registered" + exit 1 + fi + done + echo "core-viewer: rust pretty-printers registered" + ;; + *) + # Unreachable: the dependency above rejects anything else first. + >&2 echo "::error::no smoke test defined for {{ target }}" + exit 1 + ;; + esac + # An instrumented binary is a measuring device, not a shippable artifact -- and unlike the platform, # profile and sanitizer axes, instrumentation appears nowhere in `version`. A coverage or fuzz image # would therefore carry the *same tag* as a clean one and quietly replace it in the registry. From 101d68302076c5a0a7eb2b568771ad8e14333e00 Mon Sep 17 00:00:00 2001 From: Daniel Noland Date: Sat, 15 Aug 2026 21:36:22 -0600 Subject: [PATCH 07/10] feat(debug): Run a binary or a single test inside the debug images The published images debug what CI built. Debugging what you are building meant rebuilding an image by hand or falling back to a system gdb, which is exactly the case where symbols do not line up. `just debug ` builds the image at the current profile, platform, instrumentation and sanitizer, and runs the target inside it. Target resolution stays unambiguous, which is what makes this safe to call from a script: exactly one match runs without asking, no match is an error, and several with no terminal to ask at is an error listing them rather than a guess. Randomization stays enabled under gdbserver. Docker's default seccomp answers personality(ADDR_NO_RANDOMIZE) with EPERM; gdbserver treats that as non-fatal, but otherwise opens with a warning that reads like a real failure. Signed-off-by: Daniel Noland Co-Authored-By: Claude Opus 5 (1M context) --- .github/workflows/README.md | 47 ++++++++ default.nix | 38 +++--- justfile | 228 ++++++++++++++++++++++++++++++++++++ 3 files changed, 298 insertions(+), 15 deletions(-) diff --git a/.github/workflows/README.md b/.github/workflows/README.md index 4f8fd6d86d..aa266bf706 100644 --- a/.github/workflows/README.md +++ b/.github/workflows/README.md @@ -205,6 +205,53 @@ If those queue failures stop being rare, the phasing is worth revisiting. Both upload unarchived, so they download as the named file rather than wrapped in a zip. +### Debugging locally + +The published images debug what CI built. To debug what you are building, `just +debug` builds the matching image and runs a workspace binary or a single test +inside it, at whatever `profile`, `platform`, `instrument`, and `sanitize` you +pass. Symbols only line up when those match the build the problem appeared in, +which is the whole reason to go through the image rather than a system gdb. + +```console +just debug # pick from a list +just debug bugstalker # pick, then wait for an editor +just debug lurk dataplane # trace syscalls, streams JSON, runs to exit +just debug gdb dataplane # gdbserver on 2345, waits for a client +just profile=checked debug gdb test_parse_interface args +just debug-list # print the same list without running anything +``` + +Name nothing and everything is offered through `skim`, which the dev shell +provides. Name a filter matching one test and it runs without asking; name one +matching several and those are offered. A filter matching nothing is an error +rather than a guess, and so is an ambiguous one when there is no terminal to +ask at, which is what makes this safe to call from a script. + +The third argument narrows which archive is searched, so +`just debug gdb some_test args` builds only `args`' tests. It is worth passing: +the default builds every test in the workspace, which is a long wait if all you +wanted was to pick from a short list. + +`gdb` and `bugstalker` block until you disconnect and interrupt them; that is +the point. `gdb` prints the `target remote` line to use. `bugstalker` prints a +`.zed/debug.json` entry ready to paste, because in remote-DAP mode it takes the +program from the client's launch request rather than from its own command line, +so connecting an editor is only half of it. The `tcp_connection` field in that +entry is what stops the editor spawning a second debugger of its own. + +A test runs with its package directory as the working directory, the way +nextest runs it, so relative paths behave the same as under `just test`. + +To open a core file: + +```console +just inspect-core /path/to/core.1234 +``` + +Pass the same build settings that produced the binary that dumped +(`just profile=release inspect-core ...`), for the same reason. + --- ## Linting and Validation Workflows for Pull Requests diff --git a/default.nix b/default.nix index 5575db8daf..b008394a2a 100644 --- a/default.nix +++ b/default.nix @@ -206,6 +206,7 @@ let qemu-user rust-toolchain shellcheck + skim # the `just debug` picker skopeo # Serves the html that `just coverage` and `just bench criterion` produce. Opening those # from file:// works for the index but breaks the sub-pages' relative fetches. @@ -1049,15 +1050,17 @@ let source-volatile; # gdb needs a writable HOME for logs and its index cache. extraCommands = '' - # Point `src-prefix` at the sources this image ships. Referencing ${src} - # here is also what keeps it in the image closure: with the remap no - # longer naming a store path, nothing else retains it. - mkdir -p ".$(dirname "${src-prefix}")" - ln -s "${src}" ".${src-prefix}" + # The remapped prefix is relative, so a debugger resolves source against + # its working directory rather than an absolute path. Ship the tree at + # /src and start there. Referencing ${src} is also what keeps it in the + # image closure: with the remap no longer naming a store path, nothing + # else retains it. + ln -s "${src}" src mkdir -p tmp chmod 1777 tmp ''; config = { + WorkingDir = "/src"; Entrypoint = [ "/bin/gdb" "--directory=/lib/rustlib/etc" @@ -1091,15 +1094,17 @@ let source-volatile; # bugstalker needs a writable HOME for its keymap and history. extraCommands = '' - # Point `src-prefix` at the sources this image ships. Referencing ${src} - # here is also what keeps it in the image closure: with the remap no - # longer naming a store path, nothing else retains it. - mkdir -p ".$(dirname "${src-prefix}")" - ln -s "${src}" ".${src-prefix}" + # The remapped prefix is relative, so a debugger resolves source against + # its working directory rather than an absolute path. Ship the tree at + # /src and start there. Referencing ${src} is also what keeps it in the + # image closure: with the remap no longer naming a store path, nothing + # else retains it. + ln -s "${src}" src mkdir -p tmp chmod 1777 tmp ''; config = { + WorkingDir = "/src"; Entrypoint = [ "/bin/bs" # Bind the published interface rather than container-local loopback. @@ -1146,15 +1151,17 @@ let }).overrideAttrs source-volatile; extraCommands = '' - # Point `src-prefix` at the sources this image ships. Referencing ${src} - # here is also what keeps it in the image closure: with the remap no - # longer naming a store path, nothing else retains it. - mkdir -p ".$(dirname "${src-prefix}")" - ln -s "${src}" ".${src-prefix}" + # The remapped prefix is relative, so a debugger resolves source against + # its working directory rather than an absolute path. Ship the tree at + # /src and start there. Referencing ${src} is also what keeps it in the + # image closure: with the remap no longer naming a store path, nothing + # else retains it. + ln -s "${src}" src mkdir -p tmp chmod 1777 tmp ''; config = { + WorkingDir = "/src"; Entrypoint = [ "/bin/lurk" "--json" @@ -1181,6 +1188,7 @@ let # pkgs.wireshark-cli pkgs.bashInteractive + pkgs.bugstalker pkgs.coreutils pkgs.curl pkgs.debianutils diff --git a/justfile b/justfile index 5c48f91b75..16a1031573 100644 --- a/justfile +++ b/justfile @@ -508,6 +508,234 @@ setup-roots *args: {{ args }} done +# Ports the debug helpers listen on. Override for a second session. +gdb_port := "2345" + +bs_port := "4711" + +# Workspace binaries the debug images already carry, spelled as the images +# spell them. The nix attribute is keyed by directory (`workspace.init`) but +# the binary it installs is `dataplane-init`, and it is the binary name that has +# to appear here -- `debug` passes it straight through as `/bin/`. +[private] +_debug_binaries := "dataplane cli dataplane-init" + +# Build settings to forward when a recipe has to re-enter `just` rather than +# depend on it. A debug session is only useful against the same build the +# problem showed up in. +[private] +_forward := "jobs=" + jobs + " cores=" + cores + " debug_justfile=" + debug_justfile \ + + " profile=" + profile + " platform=" + platform + " libc=" + libc \ + + " features=" + features + " default_features=" + default_features \ + + " instrument=" + instrument + " sanitize=" + sanitize + " nightly=" + nightly + +# List what `just debug` can run at the current profile: the workspace binaries +# the images carry, and every test in a nextest archive. +[script] +debug-list package="tests.all": (build (if package == "tests.all" { "tests.all" } else { "tests.pkg." + package })) + {{ _just_debuggable_ }} + declare -r suite="{{ if package == "tests.all" { "tests.all" } else { "tests.pkg." + package } }}" + echo "binaries:" + for b in {{ _debug_binaries }}; do printf ' %s\n' "${b}"; done + echo "tests:" + cargo nextest list --archive-file results/"${suite}"/*.tar.zst --workspace-remap "$(pwd)" + +# Run a workspace binary or a single test under a debug helper, in the image +# that carries it, built at the current profile and instrumentation. +# +# just debug # pick from a list +# just debug bugstalker # pick, then wait for an editor +# just debug lurk dataplane # trace syscalls, streams JSON +# just debug gdb dataplane # gdbserver, waits for gdb +# just profile=checked debug gdb test_parse_interface args +# +# `target` is one of the workspace binaries, or a nextest filter. Leave it out +# and everything is offered; give a filter matching one test and it is used +# without asking; give one matching several and those are offered. `package` +# narrows which archive is built to find them, so naming one is much quicker +# than the default of building every test in the workspace. gdb and bugstalker +# wait for a client instead of running to completion. +# +# The remapped source prefix is relative, so a client shows source only if its +# working directory is the root of the tree the binary was built from. This +# passes `-w` for that in every case; a local `gdb` needs you to be standing in +# the right place yourself. +[script] +debug tool="gdb" target="" package="tests.all" *args: (build-container (if tool == "gdb" { "dataplane-core-viewer" } else if tool == "bugstalker" { "dataplane-dev-debugger" } else if tool == "lurk" { "dataplane-syscall-tracer" } else { error("debug: unknown tool '" + tool + "'; expected gdb, bugstalker, or lurk") })) + {{ _just_debuggable_ }} + declare -xr DOCKER_HOST="${DOCKER_HOST:-unix://{{ docker_sock }}}" + + declare image + case "{{ tool }}" in + gdb) image="{{ oci_image_dataplane_core_viewer }}" ;; + bugstalker) image="{{ oci_image_dataplane_dev_debugger }}" ;; + lurk) image="{{ oci_image_dataplane_syscall_tracer }}" ;; + *) + # Unreachable: the dependency above rejects anything else before + # this body runs. Kept so `set -u` cannot meet an unset `image`. + >&2 echo "debug: unknown tool '{{ tool }}'" + exit 1 + ;; + esac + declare -r image + + # Resolve the target to a program, its arguments, and a working directory. + declare program workdir + declare -a program_args=() mounts=() + if [[ -n '{{ target }}' && " {{ _debug_binaries }} " == *" {{ target }} "* ]]; then + # The images carry these already, so nothing needs mounting. `/src` is + # where they ship the sources, and the remapped prefix is relative, so + # the debugger only resolves source if it starts there. + program="/bin/{{ target }}" + workdir="/src" + else + # A test, or nothing yet. Either way the archive has to exist before + # there is anything to name or to choose between. + declare -r suite="{{ if package == "tests.all" { "tests.all" } else { "tests.pkg." + package } }}" + just {{ _forward }} build "${suite}" + declare -r extract="${PWD}/results/debug-extract" + rm -rf -- "${extract}" + mkdir -p -- "${extract}" + declare errors candidates + errors="$(mktemp)" + candidates="$(mktemp)" + declare -r errors candidates + trap 'rm -f -- "${errors}" "${candidates}"' EXIT + # Keep nextest's diagnostics: it writes progress to stderr and JSON to + # stdout, and discarding the former turns "cargo is not on PATH" into + # an empty result that looks like "no such test". + declare listing + if ! listing="$(cargo nextest list --archive-file results/"${suite}"/*.tar.zst \ + --workspace-remap "$(pwd)" --extract-to "${extract}" --extract-overwrite \ + --message-format json \ + {{ if target == "" { "" } else { "-E 'test(/" + target + "/)'" } }} \ + 2>"${errors}")"; then + >&2 echo "::error::could not list the tests in ${suite}" + >&2 cat -- "${errors}" + exit 1 + fi + declare -r listing + + # label \t binary \t workdir \t test, so a picker can show the label + # and the caller can read the rest back off the same line. + if [ -z '{{ target }}' ]; then + for b in {{ _debug_binaries }}; do + printf '%s (binary)\t/bin/%s\t/\t\n' "${b}" "${b}" + done >>"${candidates}" + fi + jq -r ' + ."rust-suites" | to_entries[] | .value as $s + | ($s.testcases // {} | to_entries[] + | select(."value"."filter-match"."status" == "matches") | .key) as $t + | "\($s."binary-id") \($t)\t\($s."binary-path")\t\($s.cwd)\t\($t)" + ' <<<"${listing}" >>"${candidates}" + + declare -i found + found="$(wc -l <"${candidates}")" + declare chosen + if [ "${found}" -eq 0 ]; then + >&2 echo "::error::nothing in ${suite} matches '{{ target }}'" + exit 1 + elif [ "${found}" -eq 1 ]; then + chosen="$(cat -- "${candidates}")" + elif [ -t 0 ]; then + # Several matches and someone to ask. Showing only the label keeps + # the store paths out of the list without losing them. + if ! command -v sk >/dev/null 2>&1; then + >&2 echo "::error::sk (skim) is not on PATH; use the dev shell, or name a target exactly" + exit 1 + fi + chosen="$(sk --delimiter '\t' --with-nth 1 \ + --prompt "{{ tool }} > " --height 40% --reverse <"${candidates}")" + if [ -z "${chosen}" ]; then + >&2 echo "debug: nothing picked" + exit 1 + fi + else + if [ -n '{{ target }}' ]; then + >&2 echo "::error::'{{ target }}' matches ${found} in ${suite}; name one exactly, or pick from a terminal:" + else + >&2 echo "::error::not a terminal, so nothing to ask; name one of:" + fi + >&2 cut -f1 -- "${candidates}" + exit 1 + fi + declare -r chosen + + program="$(cut -f2 <<<"${chosen}")" + # nextest gives a test its package root as the working directory, and + # anything reading a relative path depends on that. + workdir="$(cut -f3 <<<"${chosen}")" + declare test_name + test_name="$(cut -f4 <<<"${chosen}")" + declare -r test_name + if [ -n "${test_name}" ]; then + program_args=(--exact "${test_name}" --nocapture) + # Built outside the image, so it still resolves its loader and its + # libraries through the store; the store has to come along. + mounts=(-v /nix/store:/nix/store:ro -v "${extract}":"${extract}":ro -v "${PWD}":"${PWD}":ro) + fi + fi + declare -r program workdir + + case "{{ tool }}" in + lurk) + docker run --rm -i "${mounts[@]}" -w "${workdir}" --entrypoint /bin/lurk \ + "${image}" --json --follow-forks "${program}" "${program_args[@]}" {{ args }} + ;; + gdb) + >&2 echo "gdbserver on {{ gdb_port }}. Connect with:" + >&2 echo " gdb -ex 'target remote 127.0.0.1:{{ gdb_port }}' '${program}'" + # Randomization stays on: docker's default seccomp answers + # personality(ADDR_NO_RANDOMIZE) with EPERM, and gdbserver would + # otherwise open with an alarming warning about a benign failure. + # Loopback, not `0.0.0.0`: gdbserver runs whatever a client asks + # it to, with no authentication, and a bare `-p` would offer that + # to anything that can reach this machine. + docker run --rm -i -p "127.0.0.1:{{ gdb_port }}:{{ gdb_port }}" "${mounts[@]}" -w "${workdir}" \ + --entrypoint /bin/gdbserver "${image}" --no-disable-randomization \ + ":{{ gdb_port }}" "${program}" "${program_args[@]}" {{ args }} + ;; + bugstalker) + # It takes the program from the client's launch request, not from + # here, so connecting an editor to this is only half of it. Print + # the other half ready to paste into .zed/debug.json: `tcp_connection` + # is what stops the editor spawning a `bs` of its own. + >&2 echo "bugstalker DAP on {{ bs_port }}. Give the editor:" + >&2 jq -n --arg l "container: {{ tool }} {{ target }}" --arg p "${program}" \ + --arg w "${workdir}" --argjson port "{{ bs_port }}" \ + '{label: $l, adapter: "bugstalker-dap", request: "launch", + program: $p, args: $ARGS.positional, cwd: $w, + tcp_connection: {host: "127.0.0.1", port: $port}}' \ + --args "${program_args[@]}" {{ args }} + # The container end is 4711 whatever `bs_port` says: the image's + # entrypoint hardcodes `--dap-remote=0.0.0.0:4711`, so publishing + # `bs_port:bs_port` only worked while `bs_port` was 4711. Loopback + # for the same reason as gdbserver -- a DAP `launch` request runs + # an arbitrary program. + docker run --rm -i -p "127.0.0.1:{{ bs_port }}:4711" "${mounts[@]}" -w "${workdir}" "${image}" + ;; + esac + +# Open a core file in a gdb matched to the build that produced it. +# +# Symbols only line up with the exact version and profile that dumped, so pass +# the same build settings that produced the binary. +[script] +inspect-core core *args: (build-container "dataplane-core-viewer") + {{ _just_debuggable_ }} + declare -xr DOCKER_HOST="${DOCKER_HOST:-unix://{{ docker_sock }}}" + if [ ! -r '{{ core }}' ]; then + >&2 echo "inspect-core: cannot read '{{ core }}'" + exit 1 + fi + declare core_dir core_file + core_dir="$(cd "$(dirname -- '{{ core }}')" && pwd)" + core_file="$(basename -- '{{ core }}')" + declare -r core_dir core_file + docker run --rm -it -v "${core_dir}":/cores:ro \ + "{{ oci_image_dataplane_core_viewer }}" "/cores/${core_file}" {{ args }} + # Check that the debug images actually do what the README says they do. # # Building an image proves it links; it does not prove the entrypoint runs. From 5383f501c07fce78888aaf8c8ee60a40c463dcd9 Mon Sep 17 00:00:00 2001 From: Daniel Noland Date: Tue, 25 Aug 2026 20:50:28 -0600 Subject: [PATCH 08/10] feat(debug): Build gdb and perf to run where there is no nix store Both are meant to be copied into a VM or an image that has no store to resolve an interpreter against. Why configure flags cannot get you there, and why perf needs a different route from gdb, is recorded at each derivation -- the reasoning constrains the argument lists, so it lives beside them. Signed-off-by: Daniel Noland Co-Authored-By: Claude Opus 5 (1M context) --- nix/overlays/dataplane-dev.nix | 120 ++++++++++++++++++++++++++++++--- 1 file changed, 109 insertions(+), 11 deletions(-) diff --git a/nix/overlays/dataplane-dev.nix b/nix/overlays/dataplane-dev.nix index 6b7f47b929..26595d66bf 100644 --- a/nix/overlays/dataplane-dev.nix +++ b/nix/overlays/dataplane-dev.nix @@ -91,15 +91,113 @@ in destination = "/src/fabric/${p}"; }; - gdb' = prev.gdb.overrideAttrs (orig: { - CFLAGS = "-Os -flto"; - CXXFLAGS = "-Os -flto"; - LDFLAGS = "-flto -Wl,--as-needed,--gc-sections -static-libstdc++ -static-libgcc"; - buildInputs = (orig.buildInputs or [ ]); - configureFlags = (orig.configureFlags or [ ]) ++ [ - "--enable-static" - "--disable-inprocess-agent" - "--disable-source-highlight" # breaks static compile - ]; - }); + # A gdb that can be copied into a VM or an image that has no nix store, and + # still run: musl, no interpreter, no shared libraries at all. + # + # Configure flags cannot get you here. `--enable-static` and + # `--disable-shared` -- which nixpkgs already passes -- only decide whether + # the libbfd, libopcodes, and libctf that this tree builds are archives; they + # say nothing about how the gdb executable links against readline, ncurses, + # expat, or python, and those have no static outputs in the default package + # set. Hence pkgsStatic, which rebuilds the dependencies rather than the + # link line. + # + # pkgsStatic gates python support on host == build, so this gdb configures + # `--without-python` and cannot load the Rust pretty printers. It is a + # bare-metal debugger, not a replacement for the ordinary `gdb` that + # `containers.dataplane-core-viewer` ships with `rust-gdb-printers`. + gdb' = final.pkgsStatic.gdb.override { + # dejagnu is a buildInput only so that gdb's own test suite can run, and we + # never run it. It also cannot be built here: expect resolves `tclStubsPtr` + # from tcl's stub library, which exists to be filled in by a dynamic loader, + # so a static link leaves it undefined. + dejagnu = final.pkgsStatic.emptyDirectory; + }; + + # A perf that can be copied into a VM or an image with no nix store, for the + # same reason as `gdb'`. + # + # Unlike gdb this cannot come from pkgsStatic: elfutils carries + # `badPlatforms = isStatic` because its Makefile builds libelf.so + # unconditionally, and a static toolchain cannot emit a shared object at all + # (`crtbeginT.o: relocation R_X86_64_32 against hidden symbol __TMC_END__`). + # perf without libelf/libdw is not worth shipping, so build against ordinary + # glibc packages -- whose elfutils already installs libelf.a and libdw.a -- + # and make only the final link static. + perf' = + let + # Every dependency below is either unusable in a static binary or not + # worth its transitive static closure. Dropping them at the argument + # layer keeps them out of the build; the NO_* flags tell perf's own + # configure-equivalent the same thing, so the two cannot disagree. + none = final.emptyDirectory; + in + (final.perf.override { + stdenv = final.stdenvAdapters.makeStaticBinaries final.stdenv; + withPython = false; + withLibcap = false; + newt = none; + slang = none; + babeltrace = none; + libunwind = none; + libpfm = none; + numactl = none; + openssl = none; + libopcodes = none; + libtraceevent = none; + systemtap-unwrapped = none; + }).overrideAttrs + (orig: { + # dlfilters are dlopen-ed plugins, which a static perf could not load + # even if the toolchain could build them. + postPatch = orig.postPatch + '' + substituteInPlace Makefile.perf \ + --replace-fail \ + 'DLFILTERS := dlfilter-test-api-v0.so dlfilter-test-api-v2.so dlfilter-show-cycles.so' \ + 'DLFILTERS :=' \ + --replace-fail '$(INSTALL) $(DLFILTERS) ' 'true ' + ''; + # perf keys off -static in LDFLAGS to add `-lelf -lz -llzma -lbz2 -ldl` + # to the libdw link. Without it the libdw probe fails and perf builds + # with DWARF support silently off -- it still links, so this is only + # visible in `perf version --build-options`. Pass it in the + # environment, not via makeFlags, so Makefile.config's own `LDFLAGS +=` + # still appends. + env = orig.env // { + LDFLAGS = "-static"; + }; + # A static link does not follow a library's own dependencies, so + # libelf.a and libdw.a's compression backends have to be named here, + # as archives rather than the shared objects the normal outputs carry. + buildInputs = orig.buildInputs ++ [ + final.zlib.static + (final.zstd.override { static = true; }) + (final.xz.override { enableStatic = true; }) + (final.bzip2.override { enableStatic = true; }) + ]; + makeFlags = orig.makeFlags ++ [ + "NO_LIBPYTHON=1" + "NO_LIBPERL=1" + "NO_SLANG=1" + "NO_NEWT=1" + "NO_LIBBABELTRACE=1" + "NO_LIBNUMA=1" + "NO_LIBAUDIT=1" + "NO_LIBBPF=1" + "NO_LIBPFM4=1" + "NO_LIBCRYPTO=1" + "NO_JVMTI=1" + "NO_LIBUNWIND=1" + "NO_LIBDEBUGINFOD=1" + "NO_LIBTRACEEVENT=1" + "NO_SDT=1" + # Only C++ demangling. perf's Rust v0 demangler is built in and + # unaffected, which is what matters for a Rust dataplane. + "NO_DEMANGLE=1" + ]; + # wrapProgram would replace the binary with a shell script, defeating + # the point of a static build. It only put objdump on PATH for + # `perf annotate`, which needs a toolchain on the target regardless. + preFixup = ""; + }); } From 6165cd5c14ea3a7da218985b3b11a5932610f940 Mon Sep 17 00:00:00 2001 From: Daniel Noland Date: Tue, 25 Aug 2026 20:50:28 -0600 Subject: [PATCH 09/10] feat(debug): Dump a core from a running process without ending it Signed-off-by: Daniel Noland Co-Authored-By: Claude Opus 5 (1M context) --- scripts/dump-core.sh | 82 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 82 insertions(+) create mode 100755 scripts/dump-core.sh diff --git a/scripts/dump-core.sh b/scripts/dump-core.sh new file mode 100755 index 0000000000..e9c398a362 --- /dev/null +++ b/scripts/dump-core.sh @@ -0,0 +1,82 @@ +#!/usr/bin/env bash +# SPDX-License-Identifier: Apache-2.0 +# Copyright Open Network Fabric Authors + +# Dump a core from a running process without ending it. +# +# Attaching stops the process, so the window between attach and detach is time +# the dataplane is not forwarding. Keep the command list to the dump itself. + +set -euo pipefail + +if [ "$#" -lt 1 ] || [ "$#" -gt 2 ]; then + >&2 echo "usage: ${0##*/} [core-file]" + exit 2 +fi + +declare -r pid="$1" +declare -r core="${2:-/tmp/dataplane.core}" + +if [[ ! "${pid}" =~ ^[0-9]+$ ]]; then + >&2 echo "${0##*/}: '${pid}' is not a pid" + exit 2 +fi + +if [ ! -d "/proc/${pid}" ]; then + >&2 echo "${0##*/}: no process ${pid}" + exit 1 +fi + +# ptrace across uids needs CAP_SYS_PTRACE, and a dataplane does not run as you. +# Checking here turns a confusing gdb error into a clear one. +if [ "$(id -u)" -ne 0 ]; then + >&2 echo "${0##*/}: must run as root to attach to ${pid}" + exit 1 +fi + +# The static gdb is the point of this script: prefer one sitting beside it, so +# that scp'ing the pair to a machine with no nix store is enough. +declare gdb="${GDB:-}" +if [ -z "${gdb}" ]; then + if [ -x "$(dirname -- "$(readlink -f -- "$0")")/gdb" ]; then + gdb="$(dirname -- "$(readlink -f -- "$0")")/gdb" + else + gdb="$(command -v gdb || true)" + fi +fi +declare -r gdb +if [ -z "${gdb}" ]; then + >&2 echo "${0##*/}: no gdb; set GDB, or put one next to this script" + exit 1 +fi + +# `--nx` because an operator's ~/.gdbinit must not decide what a core contains, +# and `auto-load off` because the only thing gdb would auto-load here is +# libthread_db, which it does not need: threads are enumerated from /proc, and +# a static gdb cannot dlopen it anyway. Without this it prints a paragraph of +# safe-path advice on every run. +"${gdb}" --nx --quiet --batch \ + -iex 'set auto-load off' \ + -ex "generate-core-file ${core}" \ + -ex detach \ + -p "${pid}" + +if [ ! -s "${core}" ]; then + >&2 echo "${0##*/}: gdb wrote no core to ${core}" + exit 1 +fi + +# `detach` resumes, but say so from the process's own state rather than from +# gdb's exit status: a core you can open is worthless if the dataplane is still +# sitting in ptrace-stop. 't' is TASK_TRACED. +# The comm field is parenthesised and may contain spaces, so cut past the last +# ')' rather than counting whitespace-separated fields. +declare state +state="$(sed -e 's/^.*) //' -e 's/ .*//' "/proc/${pid}/stat" 2>/dev/null || echo gone)" +declare -r state +case "${state}" in + t) >&2 echo "${0##*/}: ${pid} is still stopped; detach it by hand"; exit 1 ;; + gone) >&2 echo "${0##*/}: ${pid} did not survive"; exit 1 ;; +esac + +echo "${0##*/}: ${core} ($(stat -c %s -- "${core}") bytes), ${pid} running (${state})" From bc4dda80db0dd8cc0105c8e4103904930f26511a Mon Sep 17 00:00:00 2001 From: Daniel Noland Date: Thu, 27 Aug 2026 01:56:21 -0600 Subject: [PATCH 10/10] fix(debug): Keep a dumped core out of reach The script runs as root by construction -- it checks and says why -- and wrote to a fixed name in a world-writable directory. `generate-core-file` does not open with O_NOFOLLOW, so an unprivileged user who plants /tmp/dataplane.core as a symlink first gets root's gdb to write through it; and a core is the dataplane's entire address space, left readable by whoever else is on the box. A private `mktemp -d` closes both. An explicit path is still the caller's business, but a symlink there is refused, since it is never what was meant. Also report TASK_STOPPED. 't' was checked because it is what a failed detach leaves behind, but 'T' -- stopped on a signal -- printed "running (T)" to an operator asking whether the dataplane is forwarding again. Co-Authored-By: Claude Opus 5 (1M context) Signed-off-by: Daniel Noland --- scripts/dump-core.sh | 29 +++++++++++++++++++++++++++-- 1 file changed, 27 insertions(+), 2 deletions(-) diff --git a/scripts/dump-core.sh b/scripts/dump-core.sh index e9c398a362..8cdae14c35 100755 --- a/scripts/dump-core.sh +++ b/scripts/dump-core.sh @@ -15,7 +15,28 @@ if [ "$#" -lt 1 ] || [ "$#" -gt 2 ]; then fi declare -r pid="$1" -declare -r core="${2:-/tmp/dataplane.core}" + +# A core is the whole address space -- configuration, keys, packets in flight -- +# and this runs as root. Two things follow for the default path. +# +# It must not be a fixed name in a world-writable directory: `generate-core-file` +# does not open with O_NOFOLLOW, so an unprivileged user who plants a symlink +# there first gets root's gdb to write through it. `mktemp -d` gives a fresh +# directory nobody else can have pre-created. +# +# And it must not be readable by anyone who happens to be on the box. The umask +# covers both the directory and the file gdb creates inside it. +umask 077 +declare core="${2:-}" +if [ -z "${core}" ]; then + core="$(mktemp -d -t dataplane-core-XXXXXX)/dataplane.core" +elif [ -L "${core}" ]; then + # An explicit path is the caller's business, but a symlink is never what was + # meant and is how the attack above is written. + >&2 echo "${0##*/}: refusing to write a core through the symlink '${core}'" + exit 2 +fi +declare -r core if [[ ! "${pid}" =~ ^[0-9]+$ ]]; then >&2 echo "${0##*/}: '${pid}' is not a pid" @@ -74,8 +95,12 @@ fi declare state state="$(sed -e 's/^.*) //' -e 's/ .*//' "/proc/${pid}/stat" 2>/dev/null || echo gone)" declare -r state +# 't' is TASK_TRACED -- gdb did not let go. 'T' is TASK_STOPPED: it did, but the +# process is sitting on a signal, which for an operator asking "is it forwarding +# again" is the same answer. case "${state}" in - t) >&2 echo "${0##*/}: ${pid} is still stopped; detach it by hand"; exit 1 ;; + t) >&2 echo "${0##*/}: ${pid} is still in ptrace-stop; detach it by hand"; exit 1 ;; + T) >&2 echo "${0##*/}: ${pid} is stopped on a signal; SIGCONT it"; exit 1 ;; gone) >&2 echo "${0##*/}: ${pid} did not survive"; exit 1 ;; esac