From 5fbd67161583ac0f473b5bc580df99686d1cf31a Mon Sep 17 00:00:00 2001 From: Daniel Noland Date: Fri, 14 Aug 2026 21:36:28 -0600 Subject: [PATCH 01/33] build(nix): keep debug outputs lean Debug binaries retained the complete Rust toolchain through their standard-library source paths, adding roughly 2.4 GB to the closure. They also carried a sizable DWARF index that neither packaged debugger consumes. Point those paths at the much smaller rust-src component, omit unused prebuilt documentation, and remove .debug_names. Source browsing and symbols remain available while the resulting diagnostic images become practical to store and transfer. Co-authored-by: Claude Opus 5 (1M context) Signed-off-by: Daniel Noland --- default.nix | 9 +++++++++ nix/overlays/llvm.nix | 2 +- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/default.nix b/default.nix index 301dba5408..c1f0f0e509 100644 --- a/default.nix +++ b/default.nix @@ -431,6 +431,8 @@ let # Keep debug paths stable across revisions. Source readers # must resolve this relative prefix from the workspace root. "--remap-path-prefix==${src-prefix}" + # Keep debug outputs from retaining the complete Rust toolchain. + "--remap-path-prefix=${pkgs.rust-toolchain}/lib/rustlib/src/rust=${pkgs.rust-toolchain.passthru.availableComponents.rust-src}/lib/rustlib/src/rust" ] ) else @@ -470,6 +472,13 @@ let mkdir -p $debug/bin for f in $out/bin/*; do mv "$f" "$debug/bin/$(basename "$f")" + # Trade index for size. gdb has consumed `.debug_names` + # as a real DWARF-5 index since 14, and this ships 17.2, + # so dropping it is not free -- gdb rebuilds an index on + # each start instead. The section is large enough on + # these binaries that the image is worth more than the + # startup, and bugstalker does not read it at all. + ${objcopy} --remove-section=.debug_names "$debug/bin/$(basename "$f")" ${strip} --strip-debug "$debug/bin/$(basename "$f")" -o "$f" ${objcopy} --add-gnu-debuglink="$debug/bin/$(basename "$f")" "$f" done diff --git a/nix/overlays/llvm.nix b/nix/overlays/llvm.nix index 334be96d21..9b7c46d39c 100644 --- a/nix/overlays/llvm.nix +++ b/nix/overlays/llvm.nix @@ -36,7 +36,7 @@ let "clippy" "llvm-tools" "rust-analyzer" - "rust-docs" + # Avoid retaining 680M of unused prebuilt documentation. "rust-src" "rust-std" "rustc" From 2c9340d3a7b9a99aa43017485496cc155bf67fb5 Mon Sep 17 00:00:00 2001 From: Daniel Noland Date: Fri, 14 Aug 2026 21:37:45 -0600 Subject: [PATCH 02/33] feat(debug): add a version-matched core viewer A core collected from the lab is useful only with the exact unstripped binaries and sources that produced it. A general debugging toolbox cannot reconstruct that relationship after the release has moved on. Provide a purpose-built gdb image alongside each build and teach it Rust's standard-library types without retaining rustc. This keeps post-mortem debugging reproducible while avoiding unrelated live-debugging tools. Co-authored-by: Claude Opus 5 (1M context) Signed-off-by: Daniel Noland --- .github/workflows/README.md | 15 ++++++- .github/workflows/dev.yml | 2 + default.nix | 82 ++++++++++++++++++++++++++----------- justfile | 17 ++++---- 4 files changed, 84 insertions(+), 32 deletions(-) diff --git a/.github/workflows/README.md b/.github/workflows/README.md index d28097881f..0c1ef549ef 100644 --- a/.github/workflows/README.md +++ b/.github/workflows/README.md @@ -103,7 +103,8 @@ If those queue failures stop being rare, the phasing is worth revisiting. - Checks: `debug` by default; `release` and `fuzz` on deep runs - Coverage: `debug` by default; `fuzz` on deep runs - Miri: required on deep runs; opt-in on pull requests with `ci:+miri` -- Containers: debug/release for dataplane and FRR; release for validator +- Containers: debug/release for dataplane, its debugger, and FRR; release for + validator - VLAB configurations: spine-leaf fabric mode, L2VNI/L3VNI VPC modes, with gateway enabled @@ -111,6 +112,18 @@ If those queue failures stop being rare, the phasing is worth revisiting. - Container images pushed to GitHub Container Registry (GHCR) - Release containers published on tag pushes via `just push` +- `ghcr.io/githedgehog/dataplane/core-viewer` opens a core file from the lab. + It carries gdb plus the unstripped binaries and sources for the matching + `ghcr.io/githedgehog/dataplane` build. + Pull the tag matching the build the core came from; symbols only line up with + the exact version and profile that produced it. + The entrypoint takes the core as its only argument: + + ```console + docker run --rm -it -v /path/to/cores:/cores \ + ghcr.io/githedgehog/dataplane/core-viewer:TAG /cores/core.1234 + ``` + - Coverage reports from each `coverage/` job, kept for 7 days: - `coverage-html-.tar.gz` - `llvm-cov` HTML report, including the per-branch counts that Codecov does not render. Unpack and open diff --git a/.github/workflows/dev.yml b/.github/workflows/dev.yml index 5cb2a33d5c..171e694cfd 100644 --- a/.github/workflows/dev.yml +++ b/.github/workflows/dev.yml @@ -428,6 +428,8 @@ jobs: nix-target: - frr.dataplane - dataplane + # Must match the build that produced the core. + - dataplane-core-viewer - validator # TODO: enable cfi and safe-stack on release when possible profile: "${{ fromJSON(needs.plan.outputs.container_profiles) }}" diff --git a/default.nix b/default.nix index c1f0f0e509..3313c11f20 100644 --- a/default.nix +++ b/default.nix @@ -971,32 +971,68 @@ let }).overrideAttrs source-volatile; - containers.dataplane-debugger = + # Shared runtime and unstripped binaries for the debugger images. + debug-image-paths = [ + pkgs.pkgsBuildHost.coreutils + pkgs.pkgsBuildHost.bashInteractive + pkgs.pkgsHostHost.dockerTools.usrBinEnv + + pkgs.pkgsHostHost.libc.debug + workspace.cli.debug + workspace.dataplane.debug + workspace.init.debug + ]; + + # Copy Rust's gdb helpers without retaining rustc as a runtime dependency. + rust-gdb-printers = pkgs.runCommand "rust-gdb-printers" { } '' + mkdir -p "$out/lib/rustlib/etc" + for f in gdb_load_rust_pretty_printers.py gdb_lookup.py gdb_providers.py rust_types.py; do + cp -L "${pkgs.rust-toolchain}/lib/rustlib/etc/$f" "$out/lib/rustlib/etc/$f" + done + ''; + + # Opens dataplane core files with matching symbols and sources. + containers.dataplane-core-viewer = (pkgs.dockerTools.buildLayeredImage { - name = "ghcr.io/githedgehog/dataplane/debugger"; + name = "ghcr.io/githedgehog/dataplane/core-viewer"; inherit tag; - contents = pkgs.buildEnv { - name = "dataplane-debugger-env"; - pathsToLink = [ - "/bin" - "/etc" - "/var" - "/lib" - ]; - paths = [ - pkgs.pkgsBuildHost.gdb - pkgs.pkgsBuildHost.rr - pkgs.pkgsBuildHost.coreutils - pkgs.pkgsBuildHost.bashInteractive - pkgs.pkgsBuildHost.iproute2 - pkgs.pkgsBuildHost.ethtool - pkgs.pkgsHostHost.dockerTools.usrBinEnv - - pkgs.pkgsHostHost.libc.debug - workspace.cli.debug - workspace.dataplane.debug - workspace.init.debug + contents = + (pkgs.buildEnv { + name = "dataplane-core-viewer-env"; + pathsToLink = [ + "/bin" + "/etc" + "/var" + "/lib" + ]; + paths = [ + pkgs.pkgsBuildHost.gdb + rust-gdb-printers + ] + ++ debug-image-paths; + }).overrideAttrs + source-volatile; + # gdb needs a writable HOME for logs and its index cache. + extraCommands = '' + # Point `src-prefix` at the sources this image ships. Referencing ${src} + # here is also what keeps it in the image closure: with the remap no + # longer naming a store path, nothing else retains it. + mkdir -p ".$(dirname "${src-prefix}")" + ln -s "${src}" ".${src-prefix}" + mkdir -p tmp + chmod 1777 tmp + ''; + config = { + Entrypoint = [ + "/bin/gdb" + "--directory=/lib/rustlib/etc" + "-iex" + "add-auto-load-safe-path /lib/rustlib/etc" + "-iex" + "source /lib/rustlib/etc/gdb_load_rust_pretty_printers.py" + "/bin/dataplane" ]; + Env = [ "HOME=/tmp" ]; }; }).overrideAttrs source-volatile; diff --git a/justfile b/justfile index 6eed433ff8..408eb0be5f 100644 --- a/justfile +++ b/justfile @@ -114,7 +114,7 @@ oci_insecure := "" oci_name := "githedgehog/dataplane" oci_frr_prefix := "githedgehog/dataplane/frr" oci_image_dataplane := oci_repo + "/" + oci_name + ":" + version -oci_image_dataplane_debugger := oci_repo + "/" + oci_name + "/debugger:" + version +oci_image_dataplane_core_viewer := oci_repo + "/" + oci_name + "/core-viewer:" + version oci_image_dataplane_validator := oci_repo + "/" + oci_name + "/validator:" + version oci_image_frr_dataplane := oci_repo + "/" + oci_frr_prefix + ":" + version oci_image_frr_host := oci_repo + "/" + oci_frr_prefix + "-host:" + version @@ -279,10 +279,10 @@ build-container target="dataplane" *args: (build (if target == "dataplane" { "da docker tag "${img}" "{{oci_image_dataplane}}" echo "imported {{ oci_image_dataplane }} (${docker_platform})" ;; - "dataplane-debugger") - docker load < ./results/containers.dataplane-debugger - docker tag "ghcr.io/githedgehog/dataplane/debugger:{{version}}" "{{oci_image_dataplane_debugger}}" - echo "imported {{ oci_image_dataplane_debugger }}" + "dataplane-core-viewer") + docker load < ./results/containers.dataplane-core-viewer + docker tag "ghcr.io/githedgehog/dataplane/core-viewer:{{version}}" "{{oci_image_dataplane_core_viewer}}" + echo "imported {{ oci_image_dataplane_core_viewer }}" ;; "debug-tools") # Uses nix only to produce a base image with the runtime closure (glibc, bash, etc.) @@ -386,8 +386,8 @@ push-container target="dataplane" *args: (build-container target args) && versio "dataplane") push_image "{{ oci_image_dataplane }}" ;; - "dataplane-debugger") - push_image "{{ oci_image_dataplane_debugger }}" + "dataplane-core-viewer") + push_image "{{ oci_image_dataplane_core_viewer }}" ;; "debug-tools") >&2 echo "do not push the debug tools!" @@ -421,7 +421,8 @@ push-container target="dataplane" *args: (build-container target args) && versio [script] push: {{ _just_debuggable_ }} - for container in dataplane frr.dataplane validator; do + # The core viewer must match the release it inspects. + for container in dataplane dataplane-core-viewer frr.dataplane validator; do if [ "${container}" = "validator" ]; then platform="wasm32-wasip1" else From 9817fc84d8a4d38509191d2d94c7f6d1add4e0e5 Mon Sep 17 00:00:00 2001 From: Daniel Noland Date: Fri, 14 Aug 2026 21:38:11 -0600 Subject: [PATCH 03/33] feat(debug): add a live DAP debugger Post-mortem inspection and live debugging need different tools. The core viewer cannot offer an editor-driven session, while bugstalker understands Rust layouts and can expose the running dataplane through the Debug Adapter Protocol. Track bugstalker upstream for its current remote DAP support and package it separately with the matching binaries and sources. Keeping the image single-purpose avoids making every diagnostic artifact carry every debugger. Co-authored-by: Claude Opus 5 (1M context) Signed-off-by: Daniel Noland --- .github/workflows/README.md | 49 ++++++++++++++++++++++++++++++++-- .github/workflows/dev.yml | 2 ++ default.nix | 48 +++++++++++++++++++++++++++++++++ justfile | 9 +++++++ nix/overlays/dataplane-dev.nix | 9 +++++++ npins/sources.json | 16 +++++++++++ 6 files changed, 131 insertions(+), 2 deletions(-) diff --git a/.github/workflows/README.md b/.github/workflows/README.md index 0c1ef549ef..9fcce2c5cf 100644 --- a/.github/workflows/README.md +++ b/.github/workflows/README.md @@ -103,8 +103,8 @@ If those queue failures stop being rare, the phasing is worth revisiting. - Checks: `debug` by default; `release` and `fuzz` on deep runs - Coverage: `debug` by default; `fuzz` on deep runs - Miri: required on deep runs; opt-in on pull requests with `ci:+miri` -- Containers: debug/release for dataplane, its debugger, and FRR; release for - validator +- Containers: debug/release for dataplane, its two debug images, and FRR; + release for validator - VLAB configurations: spine-leaf fabric mode, L2VNI/L3VNI VPC modes, with gateway enabled @@ -124,6 +124,51 @@ If those queue failures stop being rare, the phasing is worth revisiting. ghcr.io/githedgehog/dataplane/core-viewer:TAG /cores/core.1234 ``` +- `ghcr.io/githedgehog/dataplane/dev-debugger` debugs a live dataplane from an + editor. It carries bugstalker, which understands Rust's std collections and + enum layouts, and listens for a Debug Adapter Protocol client on port 4711. + Publish the port and point the editor's DAP client at it: + + ```console + docker run --rm -p 127.0.0.1:4711:4711 ghcr.io/githedgehog/dataplane/dev-debugger:TAG + ``` + + Connecting does not by itself start anything. In remote-DAP mode bugstalker + waits for the client's `launch` request to name the program, so the editor + has to send `program`, and any dataplane arguments as `args`. A request + without `program` is rejected with `launch: missing arguments.program`. + For VS Code, in `.vscode/launch.json`. `type` has to match whatever debug + type the BugStalker extension you installed registers -- it is not a name we + choose, and it differs between extensions, so check the one you have rather + than copying this field blind: + + ```json + { + "type": "bs", + "request": "launch", + "name": "dataplane (container)", + "debugServer": 4711, + "program": "/bin/dataplane", + "args": [] + } + ``` + + For `nvim-dap`, where the first line names the adapter itself, so `type = "bs"` + below is our own label rather than an extension's: + + ```lua + dap.adapters.bs = { type = "server", host = "127.0.0.1", port = 4711 } + dap.configurations.rust = { + { + type = "bs", + request = "launch", + name = "dataplane (container)", + program = "/bin/dataplane", + args = {}, + }, + } + ``` + - Coverage reports from each `coverage/` job, kept for 7 days: - `coverage-html-.tar.gz` - `llvm-cov` HTML report, including the per-branch counts that Codecov does not render. Unpack and open diff --git a/.github/workflows/dev.yml b/.github/workflows/dev.yml index 171e694cfd..5d4f972c06 100644 --- a/.github/workflows/dev.yml +++ b/.github/workflows/dev.yml @@ -430,6 +430,8 @@ jobs: - dataplane # Must match the build that produced the core. - dataplane-core-viewer + # Live debugging of the same build, driven from an editor over DAP. + - dataplane-dev-debugger - validator # TODO: enable cfi and safe-stack on release when possible profile: "${{ fromJSON(needs.plan.outputs.container_profiles) }}" diff --git a/default.nix b/default.nix index 3313c11f20..74ad7468d3 100644 --- a/default.nix +++ b/default.nix @@ -1037,6 +1037,54 @@ let }).overrideAttrs source-volatile; + # Exposes bugstalker's DAP server for live debugging. + containers.dataplane-dev-debugger = + (pkgs.dockerTools.buildLayeredImage { + name = "ghcr.io/githedgehog/dataplane/dev-debugger"; + inherit tag; + contents = + (pkgs.buildEnv { + name = "dataplane-dev-debugger-env"; + pathsToLink = [ + "/bin" + "/etc" + "/var" + "/lib" + ]; + paths = [ pkgs.pkgsBuildHost.bugstalker ] ++ debug-image-paths; + }).overrideAttrs + source-volatile; + # bugstalker needs a writable HOME for its keymap and history. + extraCommands = '' + # Point `src-prefix` at the sources this image ships. Referencing ${src} + # here is also what keeps it in the image closure: with the remap no + # longer naming a store path, nothing else retains it. + mkdir -p ".$(dirname "${src-prefix}")" + ln -s "${src}" ".${src-prefix}" + mkdir -p tmp + chmod 1777 tmp + ''; + config = { + Entrypoint = [ + "/bin/bs" + # Bind the published interface rather than container-local loopback. + "--dap-remote=0.0.0.0:4711" + # rustc is absent, so bugstalker cannot infer this path. + "--std-lib-path=${pkgs.rust-toolchain.passthru.availableComponents.rust-src}/lib/rustlib/src/rust" + # No debuggee here on purpose. In `--dap-remote` mode bugstalker + # ignores the CLI debuggee and waits for the client's `launch` request + # to name one, so a path here would be silently dead and would imply + # that connecting alone starts the dataplane. The editor supplies + # `program` instead; see .github/workflows/README.md. + ]; + Env = [ "HOME=/tmp" ]; + ExposedPorts = { + "4711/tcp" = { }; + }; + }; + }).overrideAttrs + source-volatile; + debug-tools = pkgs: [ diff --git a/justfile b/justfile index 408eb0be5f..07f78247ec 100644 --- a/justfile +++ b/justfile @@ -115,6 +115,7 @@ oci_name := "githedgehog/dataplane" oci_frr_prefix := "githedgehog/dataplane/frr" oci_image_dataplane := oci_repo + "/" + oci_name + ":" + version oci_image_dataplane_core_viewer := oci_repo + "/" + oci_name + "/core-viewer:" + version +oci_image_dataplane_dev_debugger := oci_repo + "/" + oci_name + "/dev-debugger:" + version oci_image_dataplane_validator := oci_repo + "/" + oci_name + "/validator:" + version oci_image_frr_dataplane := oci_repo + "/" + oci_frr_prefix + ":" + version oci_image_frr_host := oci_repo + "/" + oci_frr_prefix + "-host:" + version @@ -284,6 +285,11 @@ build-container target="dataplane" *args: (build (if target == "dataplane" { "da docker tag "ghcr.io/githedgehog/dataplane/core-viewer:{{version}}" "{{oci_image_dataplane_core_viewer}}" echo "imported {{ oci_image_dataplane_core_viewer }}" ;; + "dataplane-dev-debugger") + docker load < ./results/containers.dataplane-dev-debugger + docker tag "ghcr.io/githedgehog/dataplane/dev-debugger:{{version}}" "{{oci_image_dataplane_dev_debugger}}" + echo "imported {{ oci_image_dataplane_dev_debugger }}" + ;; "debug-tools") # Uses nix only to produce a base image with the runtime closure (glibc, bash, etc.) # then layers locally-compiled cargo binaries on top via Dockerfile. @@ -389,6 +395,9 @@ push-container target="dataplane" *args: (build-container target args) && versio "dataplane-core-viewer") push_image "{{ oci_image_dataplane_core_viewer }}" ;; + "dataplane-dev-debugger") + push_image "{{ oci_image_dataplane_dev_debugger }}" + ;; "debug-tools") >&2 echo "do not push the debug tools!" exit 1 diff --git a/nix/overlays/dataplane-dev.nix b/nix/overlays/dataplane-dev.nix index c8829018f2..7b0074591a 100644 --- a/nix/overlays/dataplane-dev.nix +++ b/nix/overlays/dataplane-dev.nix @@ -21,6 +21,15 @@ in opengrep = final.callPackage ../pkgs/opengrep { src = sources.opengrep; }; + # cargoDeps must be fetched from the overridden source too. + bugstalker = prev.bugstalker.overrideAttrs (orig: { + version = final.lib.removePrefix "v" sources.bugstalker.version; + src = sources.bugstalker; + cargoDeps = prev.rustPlatform.fetchCargoVendor { + src = sources.bugstalker; + hash = "sha256-GGi5hnrK5WpvnXHNckpsBch/SJ4lDvH7peSlrCdk218="; + }; + }); cargo-bolero = prev.cargo-bolero.override { inherit (override-packages) rustPlatform; }; cargo-deny = prev.cargo-deny.override { inherit (override-packages) rustPlatform; }; cargo-edit = prev.cargo-edit.override { inherit (override-packages) rustPlatform; }; diff --git a/npins/sources.json b/npins/sources.json index 369e701a56..fde579c3b7 100644 --- a/npins/sources.json +++ b/npins/sources.json @@ -16,6 +16,22 @@ "url": "https://api.github.com/repos/KaTeX/KaTeX/tarball/refs/tags/v0.18.3", "hash": "sha256-FZpiUhKFI2GAZmr667gA5yHFezAoRcAADoS0+NrNsJQ=" }, + "bugstalker": { + "type": "GitRelease", + "repository": { + "type": "GitHub", + "owner": "godzie44", + "repo": "BugStalker" + }, + "pre_releases": false, + "version_upper_bound": null, + "release_prefix": null, + "submodules": false, + "version": "v0.4.7", + "revision": "9c18e546eca6a1d68ef69da340a6fb0f2bc1bab7", + "url": "https://api.github.com/repos/godzie44/BugStalker/tarball/refs/tags/v0.4.7", + "hash": "sha256-AAeSvy/rvyylPH2jTVBGN95QIc6gumREQYuruhRo2ZI=" + }, "crane": { "type": "GitRelease", "repository": { From e303595dffd97069a07684815cafa13abee1f2e7 Mon Sep 17 00:00:00 2001 From: Daniel Noland Date: Fri, 14 Aug 2026 21:38:36 -0600 Subject: [PATCH 04/33] feat(debug): add a syscall tracer image Some failures need a record of the dataplane's kernel interactions rather than an interactive debugger. A small, repeatable tracing environment is easier to deploy and feed into existing log analysis than a general-purpose toolbox. Package lurk around the matching release binaries and follow the worker threads where the dataplane does its work. Because syscall tracing needs no symbols, this image can stay much smaller than the debugger images. Co-authored-by: Claude Opus 5 (1M context) Signed-off-by: Daniel Noland --- .github/workflows/README.md | 22 +++++++++++++++- .github/workflows/dev.yml | 2 ++ default.nix | 47 ++++++++++++++++++++++++++++++++++ justfile | 9 +++++++ nix/overlays/dataplane-dev.nix | 27 +++++++++++++++++++ 5 files changed, 106 insertions(+), 1 deletion(-) diff --git a/.github/workflows/README.md b/.github/workflows/README.md index 9fcce2c5cf..a036099f08 100644 --- a/.github/workflows/README.md +++ b/.github/workflows/README.md @@ -103,7 +103,7 @@ If those queue failures stop being rare, the phasing is worth revisiting. - Checks: `debug` by default; `release` and `fuzz` on deep runs - Coverage: `debug` by default; `fuzz` on deep runs - Miri: required on deep runs; opt-in on pull requests with `ci:+miri` -- Containers: debug/release for dataplane, its two debug images, and FRR; +- Containers: debug/release for dataplane, its three debug images, and FRR; release for validator - VLAB configurations: spine-leaf fabric mode, L2VNI/L3VNI VPC modes, with gateway enabled @@ -169,6 +169,26 @@ If those queue failures stop being rare, the phasing is worth revisiting. } ``` +- `ghcr.io/githedgehog/dataplane/syscall-tracer` records what the dataplane + asks the kernel for, as JSON, using lurk. + It carries the same stripped binaries the release image ships, since nothing + here symbolizes, so it is smaller than the other two -- though not by as much + as that suggests: like them it ships the source tree, which the entrypoint + makes the working directory. Only the debug symbols and the debuggers + themselves are absent. + + ```console + docker run --rm ghcr.io/githedgehog/dataplane/syscall-tracer:TAG > trace.jsonl + ``` + + The stream is one JSON object per line, except that tracing child threads + makes lurk announce each one with a bare `Attaching to child ` line. + Filter those out if the consumer needs strict JSONL: + + ```console + jq -R 'fromjson? // empty' < trace.jsonl + ``` + - Coverage reports from each `coverage/` job, kept for 7 days: - `coverage-html-.tar.gz` - `llvm-cov` HTML report, including the per-branch counts that Codecov does not render. Unpack and open diff --git a/.github/workflows/dev.yml b/.github/workflows/dev.yml index 5d4f972c06..915f57be6a 100644 --- a/.github/workflows/dev.yml +++ b/.github/workflows/dev.yml @@ -432,6 +432,8 @@ jobs: - dataplane-core-viewer # Live debugging of the same build, driven from an editor over DAP. - dataplane-dev-debugger + # Syscall trace of the same build, as JSON. + - dataplane-syscall-tracer - validator # TODO: enable cfi and safe-stack on release when possible profile: "${{ fromJSON(needs.plan.outputs.container_profiles) }}" diff --git a/default.nix b/default.nix index 74ad7468d3..6f6345a8f3 100644 --- a/default.nix +++ b/default.nix @@ -1085,6 +1085,53 @@ let }).overrideAttrs source-volatile; + # Traces the release binaries' syscalls as JSON with lurk. + containers.dataplane-syscall-tracer = + (pkgs.dockerTools.buildLayeredImage { + name = "ghcr.io/githedgehog/dataplane/syscall-tracer"; + inherit tag; + contents = + (pkgs.buildEnv { + name = "dataplane-syscall-tracer-env"; + pathsToLink = [ + "/bin" + "/etc" + "/var" + "/lib" + ]; + paths = [ + pkgs.pkgsBuildHost.lurk + pkgs.pkgsHostHost.dockerTools.fakeNss + pkgs.pkgsHostHost.busybox + pkgs.pkgsHostHost.dockerTools.usrBinEnv + workspace.cli + workspace.dataplane + workspace.init + ]; + }).overrideAttrs + source-volatile; + extraCommands = '' + # Point `src-prefix` at the sources this image ships. Referencing ${src} + # here is also what keeps it in the image closure: with the remap no + # longer naming a store path, nothing else retains it. + mkdir -p ".$(dirname "${src-prefix}")" + ln -s "${src}" ".${src-prefix}" + mkdir -p tmp + chmod 1777 tmp + ''; + config = { + Entrypoint = [ + "/bin/lurk" + "--json" + # Include the worker threads where the dataplane does its work. + "--follow-forks" + "/bin/dataplane" + ]; + Env = [ "HOME=/tmp" ]; + }; + }).overrideAttrs + source-volatile; + debug-tools = pkgs: [ diff --git a/justfile b/justfile index 07f78247ec..63d9ac2cd9 100644 --- a/justfile +++ b/justfile @@ -116,6 +116,7 @@ oci_frr_prefix := "githedgehog/dataplane/frr" oci_image_dataplane := oci_repo + "/" + oci_name + ":" + version oci_image_dataplane_core_viewer := oci_repo + "/" + oci_name + "/core-viewer:" + version oci_image_dataplane_dev_debugger := oci_repo + "/" + oci_name + "/dev-debugger:" + version +oci_image_dataplane_syscall_tracer := oci_repo + "/" + oci_name + "/syscall-tracer:" + version oci_image_dataplane_validator := oci_repo + "/" + oci_name + "/validator:" + version oci_image_frr_dataplane := oci_repo + "/" + oci_frr_prefix + ":" + version oci_image_frr_host := oci_repo + "/" + oci_frr_prefix + "-host:" + version @@ -290,6 +291,11 @@ build-container target="dataplane" *args: (build (if target == "dataplane" { "da docker tag "ghcr.io/githedgehog/dataplane/dev-debugger:{{version}}" "{{oci_image_dataplane_dev_debugger}}" echo "imported {{ oci_image_dataplane_dev_debugger }}" ;; + "dataplane-syscall-tracer") + docker load < ./results/containers.dataplane-syscall-tracer + docker tag "ghcr.io/githedgehog/dataplane/syscall-tracer:{{version}}" "{{oci_image_dataplane_syscall_tracer}}" + echo "imported {{ oci_image_dataplane_syscall_tracer }}" + ;; "debug-tools") # Uses nix only to produce a base image with the runtime closure (glibc, bash, etc.) # then layers locally-compiled cargo binaries on top via Dockerfile. @@ -398,6 +404,9 @@ push-container target="dataplane" *args: (build-container target args) && versio "dataplane-dev-debugger") push_image "{{ oci_image_dataplane_dev_debugger }}" ;; + "dataplane-syscall-tracer") + push_image "{{ oci_image_dataplane_syscall_tracer }}" + ;; "debug-tools") >&2 echo "do not push the debug tools!" exit 1 diff --git a/nix/overlays/dataplane-dev.nix b/nix/overlays/dataplane-dev.nix index 7b0074591a..345466c1d8 100644 --- a/nix/overlays/dataplane-dev.nix +++ b/nix/overlays/dataplane-dev.nix @@ -30,6 +30,33 @@ in hash = "sha256-GGi5hnrK5WpvnXHNckpsBch/SJ4lDvH7peSlrCdk218="; }; }); + # lurk disables ASLR in the tracee before exec, and treats failure as fatal. + # Docker's default seccomp profile answers personality(ADDR_NO_RANDOMIZE) with + # EPERM, so under a plain `docker run` the traced program never starts -- and + # lurk still exits 0 after emitting a well-formed JSON trace of its own child + # failing, which the `jq -R 'fromjson? // empty'` filter we document accepts + # without complaint. + # + # Nothing in the tracer image symbolizes an address, so a fixed layout buys us + # nothing. Make it advisory rather than telling users to pass + # `--security-opt seccomp=unconfined`, which drops confinement on a container + # whose whole job is ptracing a process. + # + # Two single-line substitutions rather than one spanning both: nix strips the + # common indentation from an indented string, so a multi-line search pattern + # would not match the source's own indentation. + lurk = prev.lurk.overrideAttrs (orig: { + postPatch = (orig.postPatch or "") + '' + substituteInPlace src/lib.rs \ + --replace-fail \ + 'personality::set(Persona::ADDR_NO_RANDOMIZE)' \ + 'let _ = personality::set(Persona::ADDR_NO_RANDOMIZE);' \ + --replace-fail \ + '.map_err(|_| anyhow!("Unable to set ADDR_NO_RANDOMIZE"))?;' \ + "" + ''; + }); + cargo-bolero = prev.cargo-bolero.override { inherit (override-packages) rustPlatform; }; cargo-deny = prev.cargo-deny.override { inherit (override-packages) rustPlatform; }; cargo-edit = prev.cargo-edit.override { inherit (override-packages) rustPlatform; }; From 417a9eab36d1fd6d6559a4a5f7deff64f5ba47db Mon Sep 17 00:00:00 2001 From: Daniel Noland Date: Fri, 14 Aug 2026 21:39:43 -0600 Subject: [PATCH 05/33] ci: publish debug images on a deliberate cadence The diagnostic images are useful only when they match the build being investigated, but building roughly 850 MB of extra images for every pull request would undermine the runner-load reduction this CI rework is meant to achieve. Build them automatically for pushes, the merge queue, and manual runs, with an explicit label available for debugging a pull request. Publish all three beside tagged releases so the matching tools remain available when a deployed build needs investigation. Co-authored-by: Claude Opus 5 (1M context) Signed-off-by: Daniel Noland --- .github/workflows/README.md | 10 ++++++++-- .github/workflows/dev.yml | 22 ++++++++++++---------- justfile | 10 ++++++++-- 3 files changed, 28 insertions(+), 14 deletions(-) diff --git a/.github/workflows/README.md b/.github/workflows/README.md index a036099f08..4f8fd6d86d 100644 --- a/.github/workflows/README.md +++ b/.github/workflows/README.md @@ -63,6 +63,11 @@ Production artifacts are produced via nix builds in a separate CI workflow. - `ci:+miri` - Run Miri checks - `ci:+wasm` - Run the WASM build check - `ci:+concurrency` - Run Shuttle and Loom tests +- `ci:+debug-images` - Also build and push the core viewer, DAP debugger, and + syscall tracer images. They are built on main, in the merge queue, and on + dispatch regardless, and `ci:+merge-ready` turns them on too, since + `ci-gate` treats that label as enabling every gate; this is for when the + build itself needs debugging - `ci:+cross` - Build all cross-platform containers - `ci:+cross/full` - Also run the workspace test suite under qemu-user, on the two aarch64 musl legs. Gated like every other job, so the merge queue and @@ -103,8 +108,9 @@ If those queue failures stop being rare, the phasing is worth revisiting. - Checks: `debug` by default; `release` and `fuzz` on deep runs - Coverage: `debug` by default; `fuzz` on deep runs - Miri: required on deep runs; opt-in on pull requests with `ci:+miri` -- Containers: debug/release for dataplane, its three debug images, and FRR; - release for validator +- Containers: debug/release for dataplane and FRR; release for validator +- Debug images (core viewer, DAP debugger, syscall tracer): deep runs only, + or on a pull request with `ci:+debug-images` - VLAB configurations: spine-leaf fabric mode, L2VNI/L3VNI VPC modes, with gateway enabled diff --git a/.github/workflows/dev.yml b/.github/workflows/dev.yml index 915f57be6a..547d3e9b33 100644 --- a/.github/workflows/dev.yml +++ b/.github/workflows/dev.yml @@ -68,6 +68,7 @@ jobs: outputs: container_profiles: "${{ steps.container-profiles.outputs.value }}" parallel: "${{ steps.parallel.outputs.value }}" + container_targets: "${{ steps.container-targets.outputs.value }}" profiles: "${{ steps.profiles.outputs.value }}" concurrency: "${{ steps.concurrency.outputs.value }}" cross: "${{ steps.cross.outputs.value }}" @@ -132,6 +133,16 @@ jobs: on-value: '["debug", "release", "fuzz"]' off-value: '["debug"]' + # Debug images are opt-in on pull requests because of their size. + - id: "container-targets" + uses: *gate + with: + labels: "debug-images" + # Keep this on one line: `ci-gate` writes the value to GITHUB_OUTPUT + # with a plain printf, which a multi-line value would corrupt. + on-value: '["frr.dataplane", "dataplane", "dataplane-core-viewer", "dataplane-dev-debugger", "dataplane-syscall-tracer", "validator"]' + off-value: '["frr.dataplane", "dataplane", "validator"]' + # Lab jobs require release images but not other release/fuzz checks. - id: "container-profiles" uses: *gate @@ -425,16 +436,7 @@ jobs: fail-fast: false max-parallel: ${{ fromJSON(needs.plan.outputs.parallel) }} matrix: - nix-target: - - frr.dataplane - - dataplane - # Must match the build that produced the core. - - dataplane-core-viewer - # Live debugging of the same build, driven from an editor over DAP. - - dataplane-dev-debugger - # Syscall trace of the same build, as JSON. - - dataplane-syscall-tracer - - validator + nix-target: "${{ fromJSON(needs.plan.outputs.container_targets) }}" # TODO: enable cfi and safe-stack on release when possible profile: "${{ fromJSON(needs.plan.outputs.container_profiles) }}" exclude: diff --git a/justfile b/justfile index 63d9ac2cd9..b2f04c564f 100644 --- a/justfile +++ b/justfile @@ -439,8 +439,14 @@ push-container target="dataplane" *args: (build-container target args) && versio [script] push: {{ _just_debuggable_ }} - # The core viewer must match the release it inspects. - for container in dataplane dataplane-core-viewer frr.dataplane validator; do + # Debug images must match the release they inspect. + for container in \ + dataplane \ + dataplane-core-viewer \ + dataplane-dev-debugger \ + dataplane-syscall-tracer \ + frr.dataplane \ + validator; do if [ "${container}" = "validator" ]; then platform="wasm32-wasip1" else From dd4e1d45424d9ab675706aa03168ea1c79c9c373 Mon Sep 17 00:00:00 2001 From: Daniel Noland Date: Sat, 15 Aug 2026 21:36:02 -0600 Subject: [PATCH 06/33] test(debug): exercise the debug images instead of only building them All three images built green while two of them could not do what the README documented. Building proves an image links; it says nothing about whether its entrypoint runs. `smoke-container` runs each one the way the README tells a user to: - the tracer under a plain `docker run`, with no seccomp relaxation, and requires an `execve` in the trace. Its failure mode is the reason this exists: lurk emitted eight well-formed JSON lines recording its own child failing to start, then exited 0, which the `jq -R 'fromjson? // empty'` filter we document accepts without complaint. Both guards fire against the pre-fix image. - the core viewer through its own entrypoint rather than by invoking gdb directly, since the `--directory` and `source` flags that register the printers live in that entrypoint. Checks that the printer set is registered, not merely that gdb started. - the debugger only for coming up and listening. Driving a real DAP session from CI means carrying a protocol client in-tree, and the contract it would pin is exercised better by pointing an editor at the image. It also would not have caught the defect on that image, which was in the documentation rather than the runtime. The trace goes to a file rather than a shell variable: at a few megabytes it overruns the here-string limit, and every grep against it then fails with E2BIG, which reads exactly like a failed trace. Co-authored-by: Claude Opus 5 (1M context) Signed-off-by: Daniel Noland --- .github/workflows/dev.yml | 11 ++++ ci.just | 4 ++ justfile | 105 ++++++++++++++++++++++++++++++++++++++ 3 files changed, 120 insertions(+) diff --git a/.github/workflows/dev.yml b/.github/workflows/dev.yml index 547d3e9b33..acd5c20e4f 100644 --- a/.github/workflows/dev.yml +++ b/.github/workflows/dev.yml @@ -451,6 +451,17 @@ jobs: with: recipe: "ci::push-container" recipe_args: "${{ matrix.nix-target }} ${{ matrix.profile }} ${{ needs.version.outputs.version }}" + + # A debug image that builds is not a debug image that works. Two of + # these shipped with entrypoints that could not do what the README + # documents, and building them said nothing about it. + - name: "smoke" + if: "${{ startsWith(matrix.nix-target, 'dataplane-') }}" + uses: *just + with: + recipe: "ci::smoke-container" + recipe_args: "${{ matrix.nix-target }} ${{ matrix.profile }}" + - *verify-clean-tree - *tmate diff --git a/ci.just b/ci.just index f4c5406f8f..7f659c5ecc 100644 --- a/ci.just +++ b/ci.just @@ -78,6 +78,10 @@ cross platform libc +args: cross-test platform libc: NEXTEST_PROFILE=cross-qemu just {{ _lab }} platform={{ platform }} libc={{ libc }} profile=debug test +# Verify a debug image's entrypoint actually does its job. +smoke-container target profile: + just {{ _lab }} profile={{ profile }} platform=x86-64-v3 smoke-container {{ target }} + # Publish both content-derived and discoverable per-commit tags. [script] push-container target profile version: diff --git a/justfile b/justfile index b2f04c564f..fb7ad50d2f 100644 --- a/justfile +++ b/justfile @@ -258,6 +258,111 @@ setup-roots *args: {{ args }} done +# Check that the debug images actually do what the README says they do. +# +# Building an image proves it links; it does not prove the entrypoint runs. +# Both of these shipped broken: the tracer's documented `docker run` produced a +# well-formed JSON trace of its own child failing to start, and still exited 0. +[script] +smoke-container target: (build-container (if target == "dataplane-core-viewer" { target } else if target == "dataplane-dev-debugger" { target } else if target == "dataplane-syscall-tracer" { target } else { error("smoke-container: no smoke test for '" + target + "'; expected dataplane-core-viewer, dataplane-dev-debugger, or dataplane-syscall-tracer") })) + {{ _just_debuggable_ }} + declare -xr DOCKER_HOST="${DOCKER_HOST:-unix://{{ docker_sock }}}" + case "{{ target }}" in + "dataplane-syscall-tracer") + # A trace runs to megabytes, so keep it in a file rather than a + # variable: `declare -x` would put it in the environment, and every + # child process then fails to exec with E2BIG, which reads exactly + # like a failed trace. Here-strings are fine at this size -- bash + # backs them with a pipe or temp file -- and the checks below use + # them on captured output. + declare trace + trace="$(mktemp)" + declare -r trace + # Name the container and remove it by name. `timeout` signals the + # docker CLI, and the daemon -- not the CLI -- owns the container's + # lifetime, so on the timeout path `--rm` alone can leave a `lurk` + # tracing something for as long as the runner lives. + declare cid + cid="smoke-syscall-tracer-$$" + declare -r cid + trap 'rm -f -- "${trace}"; docker rm -f "${cid}" >/dev/null 2>&1 || true' EXIT + # No seccomp relaxation on purpose: this is the documented command. + timeout 60 docker run --rm --name "${cid}" \ + "{{ oci_image_dataplane_syscall_tracer }}" \ + >"${trace}" 2>&1 || true + if grep -q "Unable to set ADDR_NO_RANDOMIZE" "${trace}"; then + >&2 echo "::error::lurk could not disable ASLR, so the tracee never ran" + exit 1 + fi + # The tracee has to actually execute, not merely be attached to. + if ! grep -q '"syscall":"execve"' "${trace}"; then + >&2 echo "::error::no execve in the trace: the traced program never started" + >&2 head -20 "${trace}" + exit 1 + fi + printf 'syscall-tracer: traced %s syscalls\n' "$(grep -c '"type":"SYSCALL"' "${trace}")" + ;; + "dataplane-dev-debugger") + # Only that it comes up and listens. Driving a DAP session from + # CI means carrying a protocol client for a contract better + # exercised by pointing a real editor at `just debug bugstalker`. + declare cid + # Let docker pick the host port and read it back. A fixed one + # collides: these run on a shared daemon, and `container_profiles` + # can put two of these jobs on the same node at once, where the + # second bind fails and reads as "the listener never came up". + cid="$(docker run -d --rm -p 127.0.0.1::4711 "{{ oci_image_dataplane_dev_debugger }}")" + declare -r cid + trap 'docker kill "${cid}" >/dev/null 2>&1 || true' EXIT + declare -i waited=0 + declare port + port="$(docker port "${cid}" 4711/tcp | head -1)" + port="${port##*:}" + declare -r port + if [ -z "${port}" ]; then + >&2 echo "::error::docker published no host port for 4711/tcp" + exit 1 + fi + until timeout 1 bash -c "/dev/null; do + if [ "${waited}" -ge 60 ]; then + >&2 echo "::error::dev-debugger never listened on 4711" + >&2 docker logs "${cid}" 2>&1 | tail -20 + exit 1 + fi + sleep 1 + waited=$(( waited + 1 )) + done + echo "dev-debugger: DAP listener up" + ;; + "dataplane-core-viewer") + # The Rust pretty-printers are the reason this image exists, and + # what registers them is the entrypoint's own `source` flag -- so + # drive the real entrypoint rather than invoking gdb directly, + # which would only test a copy of it. + declare out + out="$(printf 'info pretty-printer\nquit\n' \ + | timeout 120 docker run --rm -i "{{ oci_image_dataplane_core_viewer }}" 2>&1)" + if grep -qiE "traceback|no module named" <<<"${out}"; then + >&2 echo "::error::gdb could not load the rust pretty-printers" + >&2 printf '%s\n' "${out}" + exit 1 + fi + # A registered printer set, not merely a clean start. + for want in StdString StdVec StdHashMap; do + if ! grep -q "${want}" <<<"${out}"; then + >&2 echo "::error::rust pretty-printer ${want} is not registered" + exit 1 + fi + done + echo "core-viewer: rust pretty-printers registered" + ;; + *) + # Unreachable: the dependency above rejects anything else first. + >&2 echo "::error::no smoke test defined for {{ target }}" + exit 1 + ;; + esac + # Build the dataplane container image [script] build-container target="dataplane" *args: (build (if target == "dataplane" { "dataplane.tar" } else if target == "validator" { "workspace.validator" } else { "containers." + target }) args) From 9f7481ba89f7abc0cb0205045fd62acf01e9305a Mon Sep 17 00:00:00 2001 From: Daniel Noland Date: Sat, 15 Aug 2026 21:36:22 -0600 Subject: [PATCH 07/33] feat(debug): run a binary or a single test inside the debug images The published images debug what CI built. Debugging what you are building meant either rebuilding an image by hand or falling back to a system gdb, which is exactly the case where symbols do not line up. `just debug ` builds the image that carries the tool at the current profile, platform, instrumentation, and sanitizer, and runs the target inside it: - `lurk` traces syscalls and streams JSON until the program exits. - `gdb` runs gdbserver and waits, printing the `target remote` line. - `bugstalker` waits for a DAP client, printing the `program` and `args` for the launch request -- in remote-DAP mode it takes those from the client rather than from its own command line. `target` is either one of the binaries the images already carry, in which case nothing needs mounting, or a nextest filter. For a test the archive is built and unpacked, and that binary was built outside the image, so the store comes along read-only and it runs with its package directory as the working directory the way nextest runs it. Naming a target exactly is the tedious part -- test paths are long and nobody remembers them -- so leaving it out offers everything through skim, and a filter matching several offers those. Resolution stays unambiguous: exactly one match runs without asking, no match is an error, and several with no terminal to ask at is an error listing them rather than a guess. That last case is what keeps this safe to call from a script. `just debug-list` prints the same list without running anything, and `just inspect-core` opens a core file in a gdb built from the same settings. Randomization stays enabled under gdbserver. Docker's default seccomp answers personality(ADDR_NO_RANDOMIZE) with EPERM, and while gdbserver treats that as non-fatal -- unlike lurk, which is why lurk is patched -- it otherwise opens with a warning that reads like a real failure. Co-authored-by: Claude Opus 5 (1M context) Signed-off-by: Daniel Noland --- .github/workflows/README.md | 47 ++++++++ default.nix | 38 +++--- justfile | 228 ++++++++++++++++++++++++++++++++++++ 3 files changed, 298 insertions(+), 15 deletions(-) diff --git a/.github/workflows/README.md b/.github/workflows/README.md index 4f8fd6d86d..3c0dc0ba7b 100644 --- a/.github/workflows/README.md +++ b/.github/workflows/README.md @@ -205,6 +205,53 @@ If those queue failures stop being rare, the phasing is worth revisiting. Both upload unarchived, so they download as the named file rather than wrapped in a zip. +### Debugging locally + +The published images debug what CI built. To debug what you are building, `just +debug` builds the matching image and runs a workspace binary or a single test +inside it, at whatever `profile`, `platform`, `instrument`, and `sanitize` you +pass. Symbols only line up when those match the build the problem appeared in, +which is the whole reason to go through the image rather than a system gdb. + +```console +just debug # pick from a list +just debug bugstalker # pick, then wait for an editor +just debug lurk dataplane # trace syscalls, streams JSON, runs to exit +just debug gdb dataplane # gdbserver on 2345, waits for a client +just profile=fuzz debug gdb test_parse_interface args +just debug-list # print the same list without running anything +``` + +Name nothing and everything is offered through `skim`, which the dev shell +provides. Name a filter matching one test and it runs without asking; name one +matching several and those are offered. A filter matching nothing is an error +rather than a guess, and so is an ambiguous one when there is no terminal to +ask at, which is what makes this safe to call from a script. + +The third argument narrows which archive is searched, so +`just debug gdb some_test args` builds only `args`' tests. It is worth passing: +the default builds every test in the workspace, which is a long wait if all you +wanted was to pick from a short list. + +`gdb` and `bugstalker` block until you disconnect and interrupt them; that is +the point. `gdb` prints the `target remote` line to use. `bugstalker` prints a +`.zed/debug.json` entry ready to paste, because in remote-DAP mode it takes the +program from the client's launch request rather than from its own command line, +so connecting an editor is only half of it. The `tcp_connection` field in that +entry is what stops the editor spawning a second debugger of its own. + +A test runs with its package directory as the working directory, the way +nextest runs it, so relative paths behave the same as under `just test`. + +To open a core file: + +```console +just inspect-core /path/to/core.1234 +``` + +Pass the same build settings that produced the binary that dumped +(`just profile=release inspect-core ...`), for the same reason. + --- ## Linting and Validation Workflows for Pull Requests diff --git a/default.nix b/default.nix index 6f6345a8f3..1b80c72e94 100644 --- a/default.nix +++ b/default.nix @@ -182,6 +182,7 @@ let qemu-user rust-toolchain shellcheck + skim # the `just debug` picker skopeo wasmtime wget @@ -1014,15 +1015,17 @@ let source-volatile; # gdb needs a writable HOME for logs and its index cache. extraCommands = '' - # Point `src-prefix` at the sources this image ships. Referencing ${src} - # here is also what keeps it in the image closure: with the remap no - # longer naming a store path, nothing else retains it. - mkdir -p ".$(dirname "${src-prefix}")" - ln -s "${src}" ".${src-prefix}" + # The remapped prefix is relative, so a debugger resolves source against + # its working directory rather than an absolute path. Ship the tree at + # /src and start there. Referencing ${src} is also what keeps it in the + # image closure: with the remap no longer naming a store path, nothing + # else retains it. + ln -s "${src}" src mkdir -p tmp chmod 1777 tmp ''; config = { + WorkingDir = "/src"; Entrypoint = [ "/bin/gdb" "--directory=/lib/rustlib/etc" @@ -1056,15 +1059,17 @@ let source-volatile; # bugstalker needs a writable HOME for its keymap and history. extraCommands = '' - # Point `src-prefix` at the sources this image ships. Referencing ${src} - # here is also what keeps it in the image closure: with the remap no - # longer naming a store path, nothing else retains it. - mkdir -p ".$(dirname "${src-prefix}")" - ln -s "${src}" ".${src-prefix}" + # The remapped prefix is relative, so a debugger resolves source against + # its working directory rather than an absolute path. Ship the tree at + # /src and start there. Referencing ${src} is also what keeps it in the + # image closure: with the remap no longer naming a store path, nothing + # else retains it. + ln -s "${src}" src mkdir -p tmp chmod 1777 tmp ''; config = { + WorkingDir = "/src"; Entrypoint = [ "/bin/bs" # Bind the published interface rather than container-local loopback. @@ -1111,15 +1116,17 @@ let }).overrideAttrs source-volatile; extraCommands = '' - # Point `src-prefix` at the sources this image ships. Referencing ${src} - # here is also what keeps it in the image closure: with the remap no - # longer naming a store path, nothing else retains it. - mkdir -p ".$(dirname "${src-prefix}")" - ln -s "${src}" ".${src-prefix}" + # The remapped prefix is relative, so a debugger resolves source against + # its working directory rather than an absolute path. Ship the tree at + # /src and start there. Referencing ${src} is also what keeps it in the + # image closure: with the remap no longer naming a store path, nothing + # else retains it. + ln -s "${src}" src mkdir -p tmp chmod 1777 tmp ''; config = { + WorkingDir = "/src"; Entrypoint = [ "/bin/lurk" "--json" @@ -1146,6 +1153,7 @@ let # pkgs.wireshark-cli pkgs.bashInteractive + pkgs.bugstalker pkgs.coreutils pkgs.curl pkgs.debianutils diff --git a/justfile b/justfile index fb7ad50d2f..720fdec7ef 100644 --- a/justfile +++ b/justfile @@ -258,6 +258,234 @@ setup-roots *args: {{ args }} done +# Ports the debug helpers listen on. Override for a second session. +gdb_port := "2345" + +bs_port := "4711" + +# Workspace binaries the debug images already carry, spelled as the images +# spell them. The nix attribute is keyed by directory (`workspace.init`) but +# the binary it installs is `dataplane-init`, and it is the binary name that has +# to appear here -- `debug` passes it straight through as `/bin/`. +[private] +_debug_binaries := "dataplane cli dataplane-init" + +# Build settings to forward when a recipe has to re-enter `just` rather than +# depend on it. A debug session is only useful against the same build the +# problem showed up in. +[private] +_forward := "jobs=" + jobs + " cores=" + cores + " debug_justfile=" + debug_justfile \ + + " profile=" + profile + " platform=" + platform + " libc=" + libc \ + + " features=" + features + " default_features=" + default_features \ + + " instrument=" + instrument + " sanitize=" + sanitize + " nightly=" + nightly + +# List what `just debug` can run at the current profile: the workspace binaries +# the images carry, and every test in a nextest archive. +[script] +debug-list package="tests.all": (build (if package == "tests.all" { "tests.all" } else { "tests.pkg." + package })) + {{ _just_debuggable_ }} + declare -r suite="{{ if package == "tests.all" { "tests.all" } else { "tests.pkg." + package } }}" + echo "binaries:" + for b in {{ _debug_binaries }}; do printf ' %s\n' "${b}"; done + echo "tests:" + cargo nextest list --archive-file results/"${suite}"/*.tar.zst --workspace-remap "$(pwd)" + +# Run a workspace binary or a single test under a debug helper, in the image +# that carries it, built at the current profile and instrumentation. +# +# just debug # pick from a list +# just debug bugstalker # pick, then wait for an editor +# just debug lurk dataplane # trace syscalls, streams JSON +# just debug gdb dataplane # gdbserver, waits for gdb +# just profile=fuzz debug gdb test_parse_interface args +# +# `target` is one of the workspace binaries, or a nextest filter. Leave it out +# and everything is offered; give a filter matching one test and it is used +# without asking; give one matching several and those are offered. `package` +# narrows which archive is built to find them, so naming one is much quicker +# than the default of building every test in the workspace. gdb and bugstalker +# wait for a client instead of running to completion. +# +# The remapped source prefix is relative, so a client shows source only if its +# working directory is the root of the tree the binary was built from. This +# passes `-w` for that in every case; a local `gdb` needs you to be standing in +# the right place yourself. +[script] +debug tool="gdb" target="" package="tests.all" *args: (build-container (if tool == "gdb" { "dataplane-core-viewer" } else if tool == "bugstalker" { "dataplane-dev-debugger" } else if tool == "lurk" { "dataplane-syscall-tracer" } else { error("debug: unknown tool '" + tool + "'; expected gdb, bugstalker, or lurk") })) + {{ _just_debuggable_ }} + declare -xr DOCKER_HOST="${DOCKER_HOST:-unix://{{ docker_sock }}}" + + declare image + case "{{ tool }}" in + gdb) image="{{ oci_image_dataplane_core_viewer }}" ;; + bugstalker) image="{{ oci_image_dataplane_dev_debugger }}" ;; + lurk) image="{{ oci_image_dataplane_syscall_tracer }}" ;; + *) + # Unreachable: the dependency above rejects anything else before + # this body runs. Kept so `set -u` cannot meet an unset `image`. + >&2 echo "debug: unknown tool '{{ tool }}'" + exit 1 + ;; + esac + declare -r image + + # Resolve the target to a program, its arguments, and a working directory. + declare program workdir + declare -a program_args=() mounts=() + if [[ -n '{{ target }}' && " {{ _debug_binaries }} " == *" {{ target }} "* ]]; then + # The images carry these already, so nothing needs mounting. `/src` is + # where they ship the sources, and the remapped prefix is relative, so + # the debugger only resolves source if it starts there. + program="/bin/{{ target }}" + workdir="/src" + else + # A test, or nothing yet. Either way the archive has to exist before + # there is anything to name or to choose between. + declare -r suite="{{ if package == "tests.all" { "tests.all" } else { "tests.pkg." + package } }}" + just {{ _forward }} build "${suite}" + declare -r extract="${PWD}/results/debug-extract" + rm -rf -- "${extract}" + mkdir -p -- "${extract}" + declare errors candidates + errors="$(mktemp)" + candidates="$(mktemp)" + declare -r errors candidates + trap 'rm -f -- "${errors}" "${candidates}"' EXIT + # Keep nextest's diagnostics: it writes progress to stderr and JSON to + # stdout, and discarding the former turns "cargo is not on PATH" into + # an empty result that looks like "no such test". + declare listing + if ! listing="$(cargo nextest list --archive-file results/"${suite}"/*.tar.zst \ + --workspace-remap "$(pwd)" --extract-to "${extract}" --extract-overwrite \ + --message-format json \ + {{ if target == "" { "" } else { "-E 'test(/" + target + "/)'" } }} \ + 2>"${errors}")"; then + >&2 echo "::error::could not list the tests in ${suite}" + >&2 cat -- "${errors}" + exit 1 + fi + declare -r listing + + # label \t binary \t workdir \t test, so a picker can show the label + # and the caller can read the rest back off the same line. + if [ -z '{{ target }}' ]; then + for b in {{ _debug_binaries }}; do + printf '%s (binary)\t/bin/%s\t/\t\n' "${b}" "${b}" + done >>"${candidates}" + fi + jq -r ' + ."rust-suites" | to_entries[] | .value as $s + | ($s.testcases // {} | to_entries[] + | select(."value"."filter-match"."status" == "matches") | .key) as $t + | "\($s."binary-id") \($t)\t\($s."binary-path")\t\($s.cwd)\t\($t)" + ' <<<"${listing}" >>"${candidates}" + + declare -i found + found="$(wc -l <"${candidates}")" + declare chosen + if [ "${found}" -eq 0 ]; then + >&2 echo "::error::nothing in ${suite} matches '{{ target }}'" + exit 1 + elif [ "${found}" -eq 1 ]; then + chosen="$(cat -- "${candidates}")" + elif [ -t 0 ]; then + # Several matches and someone to ask. Showing only the label keeps + # the store paths out of the list without losing them. + if ! command -v sk >/dev/null 2>&1; then + >&2 echo "::error::sk (skim) is not on PATH; use the dev shell, or name a target exactly" + exit 1 + fi + chosen="$(sk --delimiter '\t' --with-nth 1 \ + --prompt "{{ tool }} > " --height 40% --reverse <"${candidates}")" + if [ -z "${chosen}" ]; then + >&2 echo "debug: nothing picked" + exit 1 + fi + else + if [ -n '{{ target }}' ]; then + >&2 echo "::error::'{{ target }}' matches ${found} in ${suite}; name one exactly, or pick from a terminal:" + else + >&2 echo "::error::not a terminal, so nothing to ask; name one of:" + fi + >&2 cut -f1 -- "${candidates}" + exit 1 + fi + declare -r chosen + + program="$(cut -f2 <<<"${chosen}")" + # nextest gives a test its package root as the working directory, and + # anything reading a relative path depends on that. + workdir="$(cut -f3 <<<"${chosen}")" + declare test_name + test_name="$(cut -f4 <<<"${chosen}")" + declare -r test_name + if [ -n "${test_name}" ]; then + program_args=(--exact "${test_name}" --nocapture) + # Built outside the image, so it still resolves its loader and its + # libraries through the store; the store has to come along. + mounts=(-v /nix/store:/nix/store:ro -v "${extract}":"${extract}":ro -v "${PWD}":"${PWD}":ro) + fi + fi + declare -r program workdir + + case "{{ tool }}" in + lurk) + docker run --rm -i "${mounts[@]}" -w "${workdir}" --entrypoint /bin/lurk \ + "${image}" --json --follow-forks "${program}" "${program_args[@]}" {{ args }} + ;; + gdb) + >&2 echo "gdbserver on {{ gdb_port }}. Connect with:" + >&2 echo " gdb -ex 'target remote 127.0.0.1:{{ gdb_port }}' '${program}'" + # Randomization stays on: docker's default seccomp answers + # personality(ADDR_NO_RANDOMIZE) with EPERM, and gdbserver would + # otherwise open with an alarming warning about a benign failure. + # Loopback, not `0.0.0.0`: gdbserver runs whatever a client asks + # it to, with no authentication, and a bare `-p` would offer that + # to anything that can reach this machine. + docker run --rm -i -p "127.0.0.1:{{ gdb_port }}:{{ gdb_port }}" "${mounts[@]}" -w "${workdir}" \ + --entrypoint /bin/gdbserver "${image}" --no-disable-randomization \ + ":{{ gdb_port }}" "${program}" "${program_args[@]}" {{ args }} + ;; + bugstalker) + # It takes the program from the client's launch request, not from + # here, so connecting an editor to this is only half of it. Print + # the other half ready to paste into .zed/debug.json: `tcp_connection` + # is what stops the editor spawning a `bs` of its own. + >&2 echo "bugstalker DAP on {{ bs_port }}. Give the editor:" + >&2 jq -n --arg l "container: {{ tool }} {{ target }}" --arg p "${program}" \ + --arg w "${workdir}" --argjson port "{{ bs_port }}" \ + '{label: $l, adapter: "bugstalker-dap", request: "launch", + program: $p, args: $ARGS.positional, cwd: $w, + tcp_connection: {host: "127.0.0.1", port: $port}}' \ + --args "${program_args[@]}" {{ args }} + # The container end is 4711 whatever `bs_port` says: the image's + # entrypoint hardcodes `--dap-remote=0.0.0.0:4711`, so publishing + # `bs_port:bs_port` only worked while `bs_port` was 4711. Loopback + # for the same reason as gdbserver -- a DAP `launch` request runs + # an arbitrary program. + docker run --rm -i -p "127.0.0.1:{{ bs_port }}:4711" "${mounts[@]}" -w "${workdir}" "${image}" + ;; + esac + +# Open a core file in a gdb matched to the build that produced it. +# +# Symbols only line up with the exact version and profile that dumped, so pass +# the same build settings that produced the binary. +[script] +inspect-core core *args: (build-container "dataplane-core-viewer") + {{ _just_debuggable_ }} + declare -xr DOCKER_HOST="${DOCKER_HOST:-unix://{{ docker_sock }}}" + if [ ! -r '{{ core }}' ]; then + >&2 echo "inspect-core: cannot read '{{ core }}'" + exit 1 + fi + declare core_dir core_file + core_dir="$(cd "$(dirname -- '{{ core }}')" && pwd)" + core_file="$(basename -- '{{ core }}')" + declare -r core_dir core_file + docker run --rm -it -v "${core_dir}":/cores:ro \ + "{{ oci_image_dataplane_core_viewer }}" "/cores/${core_file}" {{ args }} + # Check that the debug images actually do what the README says they do. # # Building an image proves it links; it does not prove the entrypoint runs. From f97fb608aab1ccc79328bcf5f03db9c0ca1ccd87 Mon Sep 17 00:00:00 2001 From: Daniel Noland Date: Fri, 7 Aug 2026 00:44:16 -0600 Subject: [PATCH 08/33] fix(net): Store the checksum an incremental update computes `Checksum::increment_update_checksum` works out the new checksum and hands it back. It does not store it, despite taking `&mut self`. `EmbeddedTransport::update_checksum` called it and dropped the answer on the floor, under a comment about ignoring errors on a truncated header -- but there is no error to ignore; the return value is the checksum. So no quoted transport header has ever had its checksum updated. NAT rewrites the ports of the packet quoted inside an ICMP error and calls this to keep the checksum in step, and the call has been doing nothing. The only correct uses of the trait method in the tree are in a test, which is presumably how the shape survived. Setting it can fail, on a header too truncated to hold a checksum -- but that is a header the caller could not have read a checksum out of either, and every caller reads one first. Hence discarding that error rather than the value, which is what the comment was reaching for. update_checksum_for_address is new, and folds a change of one of the quoted packet's addresses in. TCP, UDP and ICMPv6 are checksummed over a pseudo-header built from the source and destination addresses, so rewriting one leaves the quoted checksum describing an address that is no longer there. ICMPv4 has no pseudo-header and is left alone. Incremental for the same reason as the rest: a quote is usually truncated, so there is no payload to compute over from scratch. The tests build the one case where both routes are open -- a full header over a known payload -- and hold the fold against a computation from scratch, over two words for v4 and eight for v6. The checksum starts out correct on purpose: an RFC 1624 update is exact given a correct starting value and says nothing given a wrong one. Both fail against the discarded update; the ICMPv4 one passes either way, which is the point of it. Co-Authored-By: Claude Opus 5 (1M context) Signed-off-by: Daniel Noland --- net/src/headers/embedded.rs | 171 ++++++++++++++++++++++++++++++++++-- 1 file changed, 163 insertions(+), 8 deletions(-) diff --git a/net/src/headers/embedded.rs b/net/src/headers/embedded.rs index 8017c2231f..c98128d78b 100644 --- a/net/src/headers/embedded.rs +++ b/net/src/headers/embedded.rs @@ -22,6 +22,7 @@ use crate::udp::{TruncatedUdp, UdpChecksum, UdpPort}; use arrayvec::ArrayVec; use core::fmt::Debug; use derive_builder::Builder; +use std::net::IpAddr; use std::num::NonZero; #[cfg(any(test, feature = "bolero"))] @@ -609,42 +610,87 @@ impl EmbeddedTransport { } } + /// Fold the change of a 16-bit field into the header's checksum. + /// + /// `increment_update_checksum` computes the new checksum and hands it back; it does not store + /// it, despite taking `&mut self`. Setting it is the point of this method, and it is what makes + /// a translated quote's checksum agree with the bytes around it. + /// + /// Setting fails only on a header too truncated to hold a checksum, which is a header the + /// caller could not have read one out of either, so there is nothing to do about it here. pub fn update_checksum(&mut self, current_checksum: u16, old_value: u16, new_value: u16) { match self { EmbeddedTransport::Tcp(tcp) => { - // Silently ignore errors if transport header is truncated - let _ = tcp.increment_update_checksum( + let updated = tcp.increment_update_checksum( TcpChecksum::new(current_checksum), old_value, new_value, ); + let _ = tcp.set_checksum(updated); } EmbeddedTransport::Udp(udp) => { - // Silently ignore errors if transport header is truncated - let _ = udp.increment_update_checksum( + let updated = udp.increment_update_checksum( UdpChecksum::new(current_checksum), old_value, new_value, ); + let _ = udp.set_checksum(updated); } EmbeddedTransport::Icmp4(icmp) => { - // Silently ignore errors if transport header is truncated - let _ = icmp.increment_update_checksum( + let updated = icmp.increment_update_checksum( Icmp4Checksum::new(current_checksum), old_value, new_value, ); + let _ = icmp.set_checksum(updated); } EmbeddedTransport::Icmp6(icmp) => { - // Silently ignore errors if transport header is truncated - let _ = icmp.increment_update_checksum( + let updated = icmp.increment_update_checksum( Icmp6Checksum::new(current_checksum), old_value, new_value, ); + let _ = icmp.set_checksum(updated); } } } + + /// Fold a change to one of the quoted packet's IP addresses into this header's checksum. + /// + /// TCP, UDP and `ICMPv6` are checksummed over a pseudo-header built from the source and + /// destination addresses, so rewriting either address of a quoted packet leaves the quoted + /// transport checksum describing an address that is no longer there. `ICMPv4` has no + /// pseudo-header and is left alone. + /// + /// Incremental, because a quote is usually truncated: there is no payload to compute a + /// checksum over from scratch, only a delta to apply to the one already there. + pub fn update_checksum_for_address(&mut self, old: IpAddr, new: IpAddr) { + if matches!(self, EmbeddedTransport::Icmp4(_)) { + return; + } + // Translating between address families would change the shape of the pseudo-header rather + // than a value inside it, so there would be no delta to fold. + if old.is_ipv4() != new.is_ipv4() { + return; + } + for (old_word, new_word) in address_words(old).into_iter().zip(address_words(new)) { + let Some(current) = self.checksum() else { + return; + }; + self.update_checksum(current, old_word, new_word); + } + } +} + +// The 16-bit words of an address, in the order a checksum sums them. +fn address_words(addr: IpAddr) -> ArrayVec { + match addr { + IpAddr::V4(addr) => { + let [a, b, c, d] = addr.octets(); + ArrayVec::from_iter([u16::from_be_bytes([a, b]), u16::from_be_bytes([c, d])]) + } + IpAddr::V6(addr) => ArrayVec::from(addr.segments()), + } } impl DeParse for EmbeddedTransport { @@ -1378,6 +1424,115 @@ mod tests { // Before calling check_full_payload, should be false assert!(!headers.is_full_payload()); } + + // Checksum folding for a rewritten address in a quoted packet. + // + // A quote is usually truncated, so folding a delta into the checksum already there is the only + // option: there is no payload to compute one over. These build the one case where both routes + // are open -- a full header over a known payload -- and hold the fold against a computation + // from scratch. + // + // The checksum starts out correct on purpose. An RFC 1624 update is exact given a correct + // starting value and says nothing at all given a wrong one, so only a correct start tests + // anything. + mod address_folding { + use super::*; + use crate::icmp4::{Icmp4, Icmp4EchoRequest, Icmp4Type}; + use crate::ip::NextHeader; + use crate::ipv4::UnicastIpv4Addr; + use crate::ipv6::UnicastIpv6Addr; + use crate::tcp::{Tcp, TcpChecksumPayload}; + use crate::udp::{UdpChecksumPayload, UdpPort}; + use std::net::{Ipv4Addr, Ipv6Addr}; + + const PAYLOAD: [u8; 4] = [0xde, 0xad, 0xbe, 0xef]; + + fn v4_net(source: Ipv4Addr, next_header: NextHeader) -> Net { + let mut ip = Ipv4::default(); + ip.set_source(UnicastIpv4Addr::new(source).unwrap_or_else(|_| unreachable!())) + .set_destination(Ipv4Addr::new(192, 168, 1, 2)) + .set_next_header(next_header); + Net::Ipv4(ip) + } + + fn v6_net(source: Ipv6Addr, next_header: NextHeader) -> Net { + let mut ip = Ipv6::default(); + ip.set_source(UnicastIpv6Addr::new(source).unwrap_or_else(|_| unreachable!())) + .set_destination(Ipv6Addr::new(0xfe80, 0, 0, 0, 0, 0, 0, 2)) + .set_next_header(next_header); + Net::Ipv6(ip) + } + + /// Two words to fold, over a TCP pseudo-header. + #[test] + fn a_v4_address_change_matches_a_fresh_tcp_checksum() { + let (old, new) = (Ipv4Addr::new(192, 168, 1, 1), Ipv4Addr::new(10, 11, 12, 13)); + + let mut tcp = Tcp::new(123.try_into().unwrap(), 456.try_into().unwrap()); + tcp.update_checksum(&TcpChecksumPayload::new( + &v4_net(old, NextHeader::TCP), + &PAYLOAD, + )) + .unwrap_or_else(|()| unreachable!()); + + let mut quoted = EmbeddedTransport::Tcp(TruncatedTcp::FullHeader(tcp.clone())); + quoted.update_checksum_for_address(IpAddr::V4(old), IpAddr::V4(new)); + + let expected = tcp + .compute_checksum(&TcpChecksumPayload::new( + &v4_net(new, NextHeader::TCP), + &PAYLOAD, + )) + .unwrap_or_else(|()| unreachable!()); + assert_eq!(quoted.checksum(), Some(u16::from(expected))); + } + + /// Eight words to fold, over a UDP pseudo-header. + #[test] + fn a_v6_address_change_matches_a_fresh_udp_checksum() { + let old = Ipv6Addr::new(0xfe80, 0, 0, 0, 0, 0, 0, 1); + let new = Ipv6Addr::new(0x2001, 0xdb8, 0xdead, 0xbeef, 1, 2, 3, 4); + + let mut udp = Udp::new( + UdpPort::new_checked(123).unwrap(), + UdpPort::new_checked(456).unwrap(), + ); + udp.update_checksum(&UdpChecksumPayload::new( + &v6_net(old, NextHeader::UDP), + &PAYLOAD, + )) + .unwrap_or_else(|()| unreachable!()); + + let mut quoted = EmbeddedTransport::Udp(TruncatedUdp::FullHeader(udp.clone())); + quoted.update_checksum_for_address(IpAddr::V6(old), IpAddr::V6(new)); + + let expected = udp + .compute_checksum(&UdpChecksumPayload::new( + &v6_net(new, NextHeader::UDP), + &PAYLOAD, + )) + .unwrap_or_else(|()| unreachable!()); + assert_eq!(quoted.checksum(), Some(u16::from(expected))); + } + + /// `ICMPv4` is checksummed over itself alone, so the addresses around it are none of its + /// business and folding one must not disturb it. + #[test] + fn a_v4_address_change_leaves_an_icmpv4_quote_alone() { + let mut icmp = + Icmp4::with_type(Icmp4Type::EchoRequest(Icmp4EchoRequest { id: 18, seq: 2 })); + icmp.update_checksum(&PAYLOAD) + .unwrap_or_else(|()| unreachable!()); + + let mut quoted = EmbeddedTransport::Icmp4(TruncatedIcmp4::FullHeader(icmp)); + let before = quoted.checksum(); + quoted.update_checksum_for_address( + IpAddr::V4(Ipv4Addr::new(192, 168, 1, 1)), + IpAddr::V4(Ipv4Addr::new(10, 11, 12, 13)), + ); + assert_eq!(quoted.checksum(), before); + } + } } #[cfg(any(test, feature = "bolero"))] From 34e0b39a833700e700e7d0ca73440d6e8df3e817 Mon Sep 17 00:00:00 2001 From: Daniel Noland Date: Sat, 8 Aug 2026 14:46:06 -0600 Subject: [PATCH 09/33] test(net): Check the HeadersView unsafe boundary against the safe matcher `view.rs` was the most sensitive uncovered code in the tree, and its coverage number understated the risk. [`HeadersView`] buys zero-cost extraction with `unwrap_unchecked`: `Look::look` repeats the `ViewStep::step` chain that `sealed::Sealed::matches` already ran, and tells the compiler the `None` arms cannot happen. **Soundness rests entirely on those two chains agreeing** -- and the macro writes them out separately for every arity, eight-odd hand-written pairs, each threading the VLAN and extension cursors through by hand. That is the same shape as all six defects this campaign found in `routing`: an invariant enforced at a distance by a different function from the one relying on it. Only here a transposed cursor is not a wrong answer, it is undefined behaviour. `view.rs` sat at 42% line coverage. ## The generator was the reason, not missing tests `CommonHeaders` -- the sunny-day generator every packet test reaches for -- has six construction sites and **all six** set `vlan: ArrayVec::default()` and `net_ext: ArrayVec::default()`. It never produces a VLAN tag or an IPv6 extension header. Those are exactly the two things the view and matcher semantics are *about*: a tag the shape does not mention is a miss, extension headers are skipped silently until the shape enters the extension region and then `ExtGapCheck` turns strict. No existing generator could reach either. The tests were there; the inputs were not. Hence `ShapedHeaders`, which varies the structure: 0..=MAX_VLANS tags, and 0..=MAX_NET_EXTENSIONS extension headers of the variants that belong to the address family. Structural on purpose -- `step` walks in-memory layers with cursors, so whether `next_header` agrees with what follows it is a different property's business, and coupling the two would shrink the space this explores. Measured: **80% of generated packets carry a VLAN tag, 75% an extension header, and 20% match the shape under test.** Not vacuous. ## The oracle is the other implementation `Matcher` decides the same question safely and returns an `Option`, over the same `Within` graph and the same `ExtGapCheck`. Comparing the two is a differential test between implementations that both already exist, rather than against a third transcription of the rules. Layers are compared by **address**: two VLAN tags with equal contents pass an `assert_eq!` and are a bug if the two sides chose different ones. A shape starting at `Net` was tried and does not compile -- `Net` has no `Within<()>` -- so that half of the contract is enforced at compile time and needs no property. Left as a comment so the next person does not retry it. ## Verified, including what the verification cannot see The break test needed two attempts, which is the argument for always running it: making `matches` stricter in the **arity-1** arm changed nothing, because these shapes are arity 3 and 4. Patching the arity-3 arm fails in half a second with a shrunk packet. Both properties also run clean under miri, which is the only thing that can see the *unsound* direction -- a test that has already reached `unwrap_unchecked` on a `None` cannot report it. Recorded honestly at the property: bolero manages 5 cases a second under miri against ~35,000 native, and the miri recipe spawns its own `nix-shell` so the caller's `BOLERO_RANDOM_TEST_TIME_MS` never arrives, capping the run at 25 cases per property. A smoke test of the unsafe path, not a proof. Co-Authored-By: Claude Opus 5 (1M context) Signed-off-by: Daniel Noland --- net/src/headers/mod.rs | 67 +++++++++++++++++++++++- net/src/headers/view.rs | 113 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 179 insertions(+), 1 deletion(-) diff --git a/net/src/headers/mod.rs b/net/src/headers/mod.rs index 22fb5237ca..cf409cee4d 100644 --- a/net/src/headers/mod.rs +++ b/net/src/headers/mod.rs @@ -1194,7 +1194,7 @@ where mod contract { use crate::eth::ethtype::CommonEthType; use crate::eth::{Eth, GenWithEthType}; - use crate::headers::{Headers, Net, Transport}; + use crate::headers::{Headers, MAX_NET_EXTENSIONS, MAX_VLANS, Net, NetExt, Transport}; use crate::icmp4::Icmp4; use crate::icmp6::Icmp6; use crate::ipv4; @@ -1205,6 +1205,7 @@ mod contract { use crate::vxlan::Vxlan; use arrayvec::ArrayVec; use bolero::{Driver, TypeGenerator, ValueGenerator}; + use std::ops::Bound; impl TypeGenerator for Headers { /// Generate a completely arbitrary value of [`Headers`]. @@ -1250,6 +1251,70 @@ mod contract { } } + /// Draws [`Headers`] whose **layer structure** varies: VLAN tags, IPv6 extension headers, and + /// every combination of the two. + /// + /// [`CommonHeaders`] deliberately does not. All six of its construction sites set + /// `vlan: ArrayVec::default()` and `net_ext: ArrayVec::default()`, so it never produces a VLAN tag + /// or an extension header at all -- reasonable for the "sunny-day" packet processing it was written + /// for, and useless for the code whose whole subject is those two things: + /// + /// * [`HeadersView`](crate::headers::view::HeadersView) exists to decide whether a packet's + /// structure matches a type-level shape, and its contract is stated in terms of VLAN tags that + /// are not mentioned in the shape and extension regions the shape may or may not enter; + /// * [`Matcher`](crate::headers::pat::Matcher) threads the same `ExtGapCheck`. + /// + /// Neither semantics could be reached by any existing generator, which is a better explanation of + /// `view.rs` sitting at 42% line coverage than "nobody wrote tests" -- the tests are there. + /// + /// This is structural on purpose. `ViewStep::step` walks the in-memory layers with VLAN and + /// extension cursors, so what matters here is which slots are populated and how many, not whether + /// `next_header` agrees with the layer that follows it. A generator that also kept the wire fields + /// consistent would be the right tool for a parse round-trip property, and the wrong one for this: + /// it would couple a structural test to a byte-level invariant and shrink the space it explores. + #[allow(dead_code)] // constructed through `.with_generator()` + #[repr(transparent)] + pub struct ShapedHeaders; + + impl ValueGenerator for ShapedHeaders { + type Output = Headers; + + fn generate(&self, driver: &mut D) -> Option { + // Start from a common stack so the base layers are realistic, then vary the structure the + // view and matcher semantics actually turn on. + let mut headers = CommonHeaders.generate(driver)?; + + let vlans = driver.gen_usize(Bound::Included(&0), Bound::Included(&MAX_VLANS))?; + for _ in 0..vlans { + headers.vlan.push(driver.produce()?); + } + + // Extension headers hang off the net layer, so there is nothing to attach them to without + // one. The IPv4 authentication header is the only one that belongs on a v4 packet. + let ipv4 = matches!(headers.net, Some(Net::Ipv4(_))); + if headers.net.is_some() { + let exts = + driver.gen_usize(Bound::Included(&0), Bound::Included(&MAX_NET_EXTENSIONS))?; + for _ in 0..exts { + let ext = if ipv4 { + NetExt::Ipv4Auth(driver.produce()?) + } else { + match driver.gen_u8(Bound::Included(&0), Bound::Included(&4))? { + 0 => NetExt::HopByHop(driver.produce()?), + 1 => NetExt::DestOpts(driver.produce()?), + 2 => NetExt::Routing(driver.produce()?), + 3 => NetExt::Fragment(driver.produce()?), + _ => NetExt::Ipv6Auth(driver.produce()?), + } + }; + headers.net_ext.push(ext); + } + } + + Some(headers) + } + } + #[allow(dead_code)] // rustc not able to infer we construct this through .with_generator() #[repr(transparent)] pub struct CommonHeaders; diff --git a/net/src/headers/view.rs b/net/src/headers/view.rs index 516f60ff28..abb0479b89 100644 --- a/net/src/headers/view.rs +++ b/net/src/headers/view.rs @@ -2575,3 +2575,116 @@ mod tests { assert_eq!(v.vid(), vid_updated); } } + +/// The `unsafe` boundary, checked against the safe implementation of the same semantics. +/// +/// [`HeadersView`] earns its zero-cost extraction with `unwrap_unchecked`: [`Look::look`] repeats the +/// [`ViewStep::step`] chain that [`sealed::Sealed::matches`] already ran and tells the compiler the +/// `None` arms cannot happen. **The soundness of that rests entirely on the two chains agreeing**, and +/// they are written out separately for every arity by the macro above -- eight-odd hand-written pairs, +/// each threading the VLAN and extension cursors through by hand. A transposed cursor in one of them +/// is not a wrong answer, it is undefined behaviour. +/// +/// That is the same shape as every defect this campaign found in `routing`: an invariant enforced at a +/// distance by a different function from the one relying on it. The difference is the consequence. +/// +/// The oracle is [`Matcher`](super::pat::Matcher), which decides the same question safely and returns +/// an `Option`. Comparing the two is a differential test between two existing implementations rather +/// than against a third transcription of the rules -- if they disagree, either `matches` admits a +/// packet `look` cannot extract from (undefined behaviour) or `Matcher` mis-matches in the datapath. +/// Both are worth knowing. +/// +/// References are compared by **address**, not by value: two VLAN tags with identical contents are a +/// pass under `assert_eq!` and a bug if the two implementations picked different ones. +/// +/// # What this cannot catch +/// +/// Both `matches` and `look` call the same [`ViewStep::step`], so a bug *inside* `step` is invisible +/// here -- it moves both sides together. What is checked is the hand-written *chaining* around it, +/// which is where the duplication is. +/// +/// And it can only observe the safe direction of a divergence. If `matches` were ever too **strict**, +/// the matcher accepts where `as_view` refuses and this fails with a counterexample. If it were too +/// **permissive**, `look` reaches `unwrap_unchecked` on a `None`, and a test that has already invoked +/// undefined behaviour is in no position to report it. **That direction is what miri is for**, and +/// these run clean under `just miri test -p dataplane-net view_properties`. +/// +/// Note what that costs: bolero manages 5 cases a second under miri against roughly 35,000 native, and +/// the miri recipe spawns its own `nix-shell`, so `BOLERO_RANDOM_TEST_TIME_MS` from the caller's +/// environment does not reach it and the run stops at **25 cases per property**. That is a smoke test +/// of the unsafe path, not a proof. Raising it means setting the budget inside `miri.just`. +#[cfg(test)] +mod view_properties { + use crate::eth::Eth; + use crate::headers::view::Look; + use crate::headers::{Headers, Net, ShapedHeaders, Transport}; + use crate::vlan::Vlan; + + /// `as_view` and `Matcher` must agree on whether the packet has the shape, and on which layers. + #[test] + fn eth_net_transport_agrees_with_the_matcher() { + bolero::check!() + .with_generator(ShapedHeaders) + .for_each(|h: &Headers| { + let matched = h.pat().eth().net().transport().done(); + match h.as_view::<(&Eth, &Net, &Transport)>() { + None => assert!( + matched.is_none(), + "the matcher accepted a shape as_view refused: {h:?}" + ), + Some(view) => { + let Some((m_eth, m_net, m_transport)) = matched else { + panic!("as_view accepted a shape the matcher refused: {h:?}"); + }; + let (v_eth, v_net, v_transport) = view.look(); + assert!(std::ptr::eq(v_eth, m_eth), "eth differs: {h:?}"); + assert!(std::ptr::eq(v_net, m_net), "net differs: {h:?}"); + assert!( + std::ptr::eq(v_transport, m_transport), + "transport differs: {h:?}" + ); + } + } + }); + } + + /// The same, for a shape that names a VLAN tag. + /// + /// This is where the interesting half of the contract lives: a tag the shape does not mention is a + /// miss, so the two implementations have to agree about *how many* tags were consumed, not merely + /// that some were. + #[test] + fn eth_vlan_net_transport_agrees_with_the_matcher() { + bolero::check!() + .with_generator(ShapedHeaders) + .for_each(|h: &Headers| { + let matched = h.pat().eth().vlan().net().transport().done(); + match h.as_view::<(&Eth, &Vlan, &Net, &Transport)>() { + None => assert!( + matched.is_none(), + "the matcher accepted a vlan shape as_view refused: {h:?}" + ), + Some(view) => { + let Some((m_eth, m_vlan, m_net, m_transport)) = matched else { + panic!("as_view accepted a vlan shape the matcher refused: {h:?}"); + }; + let (v_eth, v_vlan, v_net, v_transport) = view.look(); + assert!(std::ptr::eq(v_eth, m_eth), "eth differs: {h:?}"); + assert!( + std::ptr::eq(v_vlan, m_vlan), + "the two implementations consumed different vlan tags: {h:?}" + ); + assert!(std::ptr::eq(v_net, m_net), "net differs: {h:?}"); + assert!( + std::ptr::eq(v_transport, m_transport), + "transport differs: {h:?}" + ); + } + } + }); + } + + // A shape starting at `Net` rather than `Eth` was tried here and does not compile: `Net` does not + // satisfy `Within<()>`, so the adjacency graph forbids a shape that begins mid-stack. That half of + // the contract is enforced at compile time and needs no property. +} From f670d95c7cae8bcf78f85fc541d33ace28650c4b Mon Sep 17 00:00:00 2001 From: Daniel Noland Date: Sat, 8 Aug 2026 14:55:36 -0600 Subject: [PATCH 10/33] docs(net): Correct where the HeadersView soundness check actually fires The previous commit claimed the unsound direction of a `matches`/`look` divergence could only be seen under miri, on the reasoning that a test which has already reached `unwrap_unchecked` on a `None` is in no position to report it. That is wrong, and Daniel caught it. `unwrap_unchecked` bottoms out in `hint::unreachable_unchecked`, whose `assert_unsafe_precondition!` is gated on `ub_checks`, which follows `-Cdebug-assertions` -- and `profile.fuzz` sets that **on**. Measured rather than reasoned this time: unsafe precondition(s) violated: hint::unreachable_unchecked must never be reached thread caused non-unwinding panic. aborting. ... (signal: 6, SIGABRT: process abort signal) So `profile=fuzz` already detects it, and that is where most of the assurance comes from. Miri is still worth having for what `ub_checks` does not model -- aliasing and provenance across the `as_ref_unchecked` boundary -- but it is not the only thing standing between this and undefined behaviour. Both guards are now documented in the order they fire, each demonstrated by breaking the arity-3 arm deliberately: 1. **the differential, first.** The `Matcher` comparison runs *before* `look`, so a divergence in either direction fails with a shrunk counterexample instead of invoking undefined behaviour. Over-strict and accept-everything both fail here in under a second. 2. **the standard library's check, as a backstop**, for a divergence that slipped past guard 1 -- if `Matcher` carried the same bug. Verified by calling `look` on an over-permissive `matches` with the differential removed: `SIGABRT`, fuzz profile, no miri. Worth recording that guard 2 is a *non-unwinding* panic, so bolero cannot catch it and the process dies. Under libfuzzer that is the right outcome -- a saved `crash-*` artifact rather than a silent pass -- but it does mean the failure surfaces as an abort rather than a counterexample, which is why guard 1 running first is a design choice and not an accident. No behaviour change; comments only. Co-Authored-By: Claude Opus 5 (1M context) Signed-off-by: Daniel Noland --- net/src/headers/view.rs | 42 +++++++++++++++++++++++++++++++---------- 1 file changed, 32 insertions(+), 10 deletions(-) diff --git a/net/src/headers/view.rs b/net/src/headers/view.rs index abb0479b89..1aa999690f 100644 --- a/net/src/headers/view.rs +++ b/net/src/headers/view.rs @@ -2597,22 +2597,44 @@ mod tests { /// References are compared by **address**, not by value: two VLAN tags with identical contents are a /// pass under `assert_eq!` and a bug if the two implementations picked different ones. /// -/// # What this cannot catch +/// # Two lines of defence, and where each one fires /// /// Both `matches` and `look` call the same [`ViewStep::step`], so a bug *inside* `step` is invisible /// here -- it moves both sides together. What is checked is the hand-written *chaining* around it, /// which is where the duplication is. /// -/// And it can only observe the safe direction of a divergence. If `matches` were ever too **strict**, -/// the matcher accepts where `as_view` refuses and this fails with a counterexample. If it were too -/// **permissive**, `look` reaches `unwrap_unchecked` on a `None`, and a test that has already invoked -/// undefined behaviour is in no position to report it. **That direction is what miri is for**, and -/// these run clean under `just miri test -p dataplane-net view_properties`. +/// Against a divergence in that chaining there are two independent guards, and both were demonstrated +/// by breaking the arity-3 arm on purpose: /// -/// Note what that costs: bolero manages 5 cases a second under miri against roughly 35,000 native, and -/// the miri recipe spawns its own `nix-shell`, so `BOLERO_RANDOM_TEST_TIME_MS` from the caller's -/// environment does not reach it and the run stops at **25 cases per property**. That is a smoke test -/// of the unsafe path, not a proof. Raising it means setting the budget inside `miri.just`. +/// 1. **This differential, which fires first.** The `Matcher` comparison happens *before* `look` is +/// called, so a divergence in either direction fails with a shrunk counterexample rather than by +/// invoking undefined behaviour. Making `matches` stricter, or making it accept everything, both +/// fail here in under a second. +/// 2. **The standard library's own check, as a backstop.** `unwrap_unchecked` bottoms out in +/// `hint::unreachable_unchecked`, whose `assert_unsafe_precondition!` is gated on `ub_checks` -- +/// which follows `-Cdebug-assertions`, and `profile.fuzz` sets that **on**. So if a divergence ever +/// slipped past guard 1 -- if `Matcher` carried the same bug, say -- reaching `look` aborts: +/// +/// ```text +/// unsafe precondition(s) violated: hint::unreachable_unchecked must never be reached +/// thread caused non-unwinding panic. aborting. +/// ``` +/// +/// Verified by calling `look` on a deliberately over-permissive `matches` with the differential +/// removed: `SIGABRT`, in the fuzz profile, no miri required. Note it is a *non-unwinding* panic, so +/// bolero cannot catch it and the process dies -- which under libfuzzer is exactly right, a saved +/// `crash-*` artifact rather than a silent pass. +/// +/// The practical consequence is that **`just fuzz` on the fuzz profile already detects the unsound +/// direction**, and that is where the assurance mostly comes from. Miri remains useful for what +/// `ub_checks` does not model -- aliasing and provenance across the `as_ref_unchecked` boundary -- but +/// it is not the only thing standing between this and undefined behaviour. +/// +/// These do run clean under `just miri test -p dataplane-net view_properties`. Worth knowing what that +/// is worth, though: bolero manages 5 cases a second under miri against roughly 35,000 native, and the +/// miri recipe spawns its own `nix-shell`, so `BOLERO_RANDOM_TEST_TIME_MS` from the caller's +/// environment never arrives and the run stops at **25 cases per property**. A smoke test, not a proof. +/// Raising it means setting the budget inside `miri.just`. #[cfg(test)] mod view_properties { use crate::eth::Eth; From 199c52b7b582e699ab1ec758061c6306d0470bd5 Mon Sep 17 00:00:00 2001 From: Daniel Noland Date: Sat, 8 Aug 2026 15:25:07 -0600 Subject: [PATCH 11/33] test(net): Check the mutable half of the HeadersView boundary, and make miri usable The read path was the easy half. `Look::look` and `sealed::Sealed::matches` at least walk the stack the same way -- both chain `ViewStep::step`. `look_mut` does not: it builds a `MatcherMut` from `pat_mut()` and chains `ViewStepMut::chain` over it, then calls `unreachable_unchecked` if that returns `None`. **So the invariant is established by one traversal and consumed by a different one.** Nothing makes `ViewStep::step` and `ViewStepMut::chain` agree except that they were written to. Where the read path risks a mis-threaded cursor between two copies of one walk, this risks two walks disagreeing outright. Worth saying what is *not* tested: `look_mut` against `MatcherMut`. `look_mut` **is** `MatcherMut` plus an `unreachable_unchecked`, so comparing them is the implementation against itself. The question worth asking is whether `matches` -- which licensed the unchecked call -- agrees with the walk that has to deliver on it, and that is checked without calling `look_mut` at all, so a divergence is a counterexample rather than undefined behaviour. `look_mut` hands back several `&mut` into one `Headers`, pre-split through `Fields`. If that split ever aliased, two references would point at the same layer -- and the `ub_checks` backstop cannot see it. It checks the `unreachable_unchecked` precondition and nothing about aliasing. Only miri sees that, and only with stacked borrows on. Two obstacles, both now fixed in `miri.just`: - **the budget.** The recipe launches its own `nix-shell`, which does not inherit the caller's environment, so `BOLERO_RANDOM_TEST_TIME_MS` never arrived and every property stopped at bolero's one-second default -- about 25 cases under miri. Enough to prove the harness runs and nothing else. - **`stacked_borrow_check` was unreachable.** `just` will not override a *module's* variables from the command line: `just miri stacked_borrow_check=... test` parses as a recipe name, and `--set` is refused as "not present in justfile". The knob existed and could only be changed by editing the file. Both now read `env()`, so `STACKED_BORROW_CHECK=enabled just miri test ...` works. Under miri bolero manages about five cases a second, and the cost is wall-clock. Raising the budget buys cases linearly on one core; sharding buys them across cores for free. The aliasing property -- the expensive one and the one that matters most -- is instantiated as sixteen shards, each seeded from the OS so they explore independent streams, and nextest runs them concurrently. Sixteen rather than sixty because miri's per-process memory footprint is large and the other properties want cores too. Result: **963 cases across 24 properties under miri with stacked borrows enabled, no undefined behaviour** -- against 25 per property with stacked borrows off before this. Co-Authored-By: Claude Opus 5 (1M context) Signed-off-by: Daniel Noland --- miri.just | 16 +++- net/src/headers/view.rs | 170 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 185 insertions(+), 1 deletion(-) diff --git a/miri.just b/miri.just index 091da10355..db51d93875 100644 --- a/miri.just +++ b/miri.just @@ -25,7 +25,20 @@ export target := cpu + "-unknown-linux-gnu" export provenance := "permissive" export schedule_seed := choose('5', "0123456789") export seeds := "1" -export stacked_borrow_check := "disabled" +# Overridable from the environment because `just` will not override a *module's* variables from the +# command line: `just miri stacked_borrow_check=enabled test` is parsed as a recipe name, and +# `just --set stacked_borrow_check enabled miri test` is refused outright ("not present in justfile"). +# Without `env()` these knobs can only be changed by editing this file, which is how +# `stacked_borrow_check` came to be effectively unreachable. +export stacked_borrow_check := env("STACKED_BORROW_CHECK", "disabled") + +# How long each bolero property gets, in milliseconds. +# +# Needed because this recipe launches its own `nix-shell`, which does not inherit the caller's +# environment: exporting `BOLERO_RANDOM_TEST_TIME_MS` before `just miri test` has no effect, and every +# property silently stops at bolero's one-second default. Under miri that is about 25 cases -- enough to +# prove the harness runs and nothing else. +export bolero_test_time_ms := env("BOLERO_TEST_TIME_MS", "30000") export preemption_rate := "0.10" export weak_failure_rate := "0.05" export randomize_struct_layout := "enabled" @@ -63,6 +76,7 @@ test *args="": # Umbrella cfg shared with the qemu-user path in nix/profiles.nix. RUSTFLAGS+="--cfg=emulated" declare -rx RUSTFLAGS + declare -rx BOLERO_RANDOM_TEST_TIME_MS="${bolero_test_time_ms}" declare -a cmd=("nice" "-n" "19" "cargo" "miri" "nextest" "run" "--profile=miri" "--target=${target}") if [ "${cores}" != "0" ]; then # nextest defaults --test-threads to the core count; the miri profile diff --git a/net/src/headers/view.rs b/net/src/headers/view.rs index 1aa999690f..2d3173367b 100644 --- a/net/src/headers/view.rs +++ b/net/src/headers/view.rs @@ -2710,3 +2710,173 @@ mod view_properties { // satisfy `Within<()>`, so the adjacency graph forbids a shape that begins mid-stack. That half of // the contract is enforced at compile time and needs no property. } + +/// The **mutable** half of the unsafe boundary, which is the more delicate one. +/// +/// [`Look::look`] and [`sealed::Sealed::matches`] at least walk the stack the same way: both chain +/// [`ViewStep::step`]. [`LookMut::look_mut`] does not. It builds a [`MatcherMut`](super::pat::MatcherMut) +/// from [`Headers::pat_mut`] and chains [`ViewStepMut::chain`] over it, then calls +/// `unreachable_unchecked` if that comes back `None`. +/// +/// So the invariant is established by one traversal and consumed by a **different** one. Nothing makes +/// `ViewStep::step` and `ViewStepMut::chain` agree except that they were written to; where the read path +/// risks a mis-threaded cursor between two copies of the same walk, this risks two walks disagreeing +/// outright. +/// +/// Note what is *not* worth testing here: `look_mut` against `MatcherMut` directly. `look_mut` **is** +/// `MatcherMut` plus an `unreachable_unchecked`, so that comparison is the implementation against +/// itself. The question worth asking is whether `matches` -- the `ViewStep` walk that licensed the +/// unchecked call -- agrees with the `ViewStepMut` walk that has to deliver on it. +/// +/// # Aliasing, and why `ub_checks` cannot help +/// +/// `look_mut` hands back several `&mut` into one [`Headers`], pre-split through +/// [`Fields`](super::pat::Fields). If that split ever aliased, two of those references would point at +/// the same layer -- undefined behaviour of a kind the `ub_checks` backstop does **not** model. It +/// checks the `unreachable_unchecked` precondition, nothing about aliasing. +/// +/// Only miri sees that, and only with stacked borrows *on*, which this repo turns off by default: +/// +/// ```text +/// STACKED_BORROW_CHECK=enabled just miri test -p dataplane-net view_mut_properties +/// ``` +/// +/// Through the environment, not `just`'s command line: a *module's* variables cannot be overridden +/// there. `just miri stacked_borrow_check=enabled test` parses as a recipe name and +/// `just --set stacked_borrow_check enabled miri test` is refused, which is why `miri.just` reads both +/// knobs via `env()`. +#[cfg(test)] +mod view_mut_properties { + use crate::eth::Eth; + use crate::headers::view::LookMut; + use crate::headers::{Headers, Net, ShapedHeaders, Transport}; + use crate::vlan::Vlan; + + /// Whatever licensed the unchecked call must be deliverable by the walk that has to deliver it. + /// + /// Checked without calling `look_mut`, so a divergence is a clean counterexample rather than + /// undefined behaviour: `as_view_mut` consults `matches`, and the chain below is the same one + /// `look_mut` would run, but safely. + #[test] + fn what_matches_licenses_the_mutable_walk_can_deliver() { + bolero::check!() + .with_generator(ShapedHeaders) + .for_each(|h: &Headers| { + let mut owned = h.clone(); + let licensed = owned.as_view_mut::<(&Eth, &Net, &Transport)>().is_some(); + let deliverable = owned.pat_mut().eth().net().transport().done().is_some(); + assert_eq!( + licensed, deliverable, + "`matches` and the mutable walk disagree, so `look_mut` would reach \ + `unreachable_unchecked`: {h:?}" + ); + }); + } + + /// The same for a shape naming a VLAN tag, where the walks have to agree on how many were consumed. + #[test] + fn what_matches_licenses_the_mutable_vlan_walk_can_deliver() { + bolero::check!() + .with_generator(ShapedHeaders) + .for_each(|h: &Headers| { + let mut owned = h.clone(); + let licensed = owned + .as_view_mut::<(&Eth, &Vlan, &Net, &Transport)>() + .is_some(); + let deliverable = owned + .pat_mut() + .eth() + .vlan() + .net() + .transport() + .done() + .is_some(); + assert_eq!( + licensed, deliverable, + "`matches` and the mutable vlan walk disagree: {h:?}" + ); + }); + } + + /// Exercise the multi-`&mut` split itself: write through every reference and read the writes back. + /// + /// The assertions are almost beside the point. What matters is that the references are *created and + /// written through*, so that miri with stacked borrows enabled can judge whether + /// [`Fields`](super::pat::Fields) handed out two paths to the same layer. Nothing else in the suite + /// does that. + fn exercise_the_mutable_split() { + bolero::check!() + .with_generator(ShapedHeaders) + .for_each(|h: &Headers| { + let mut owned = h.clone(); + let Some(view) = owned.as_view_mut::<(&Eth, &Net, &Transport)>() else { + return; + }; + let (eth, net, transport) = view.look_mut(); + + // A write through each reference, then a read back through the same one. If two of + // these aliased, the writes would interfere and miri would object to the borrow stack + // long before the values did. + let want_src = + crate::eth::mac::SourceMac::try_from(crate::eth::mac::Mac([2, 0, 0, 0, 0, 1])) + .unwrap_or_else(|_| { + unreachable!("a locally-administered unicast mac is a valid source") + }); + eth.set_source(want_src); + let seen_net = net.dst_addr(); + let seen_transport = transport.dst_port(); + + assert_eq!( + eth.source(), + want_src, + "the write through eth did not stick" + ); + assert_eq!(net.dst_addr(), seen_net, "net changed under a write to eth"); + assert_eq!( + transport.dst_port(), + seen_transport, + "transport changed under a write to eth" + ); + }); + } + + /// Shards of [`exercise_the_mutable_split`], so the machine can be used. + /// + /// Under miri this property is the expensive one and the one that matters most, and its cost is + /// wall-clock: bolero manages about five cases a second. Raising the time budget buys cases + /// linearly, but only on one core. + /// + /// Each shard seeds itself from the OS, so `N` shards explore `N` independent streams and nextest + /// runs them concurrently -- turning a machine with cores to spare into more cases for the same + /// wall time, which is the only lever that does not cost patience. Sixteen rather than sixty: miri + /// carries a large memory footprint per process, and the other properties want cores too. + macro_rules! split_shards { + ($($name:ident),* $(,)?) => { + $( + #[test] + fn $name() { + exercise_the_mutable_split(); + } + )* + }; + } + + split_shards!( + the_mutable_split_hands_out_distinct_layers, + split_shard_02, + split_shard_03, + split_shard_04, + split_shard_05, + split_shard_06, + split_shard_07, + split_shard_08, + split_shard_09, + split_shard_10, + split_shard_11, + split_shard_12, + split_shard_13, + split_shard_14, + split_shard_15, + split_shard_16, + ); +} From 697a0f810228c6238c2b983c6f9958a89bc28059 Mon Sep 17 00:00:00 2001 From: Daniel Noland Date: Sat, 8 Aug 2026 16:19:21 -0600 Subject: [PATCH 12/33] test(net): Extend the HeadersView differential to every arity the oracle reaches `matches`, `look` and `look_mut` are generated separately for each arity by a macro, with the VLAN and extension cursors threaded through by hand every time. Testing two arities tested two of eight copies, which is why the last commit moved `view.rs` coverage by two points and no more. Both walks are now checked at arities one through seven, read and mutable, from one macro so a new arity costs one line. ## Seven, not eight, and the reason is the oracle `Matcher`'s vocabulary is `eth`, `vlan`, `net`, `transport`, `vxlan`, `embedded`. The last two cannot be reached in a builder chain: `Vxlan: Within` and the embedded header sits under `Icmp4`/`Icmp6` -- both *concrete* layers -- and `Matcher` has no concrete-layer methods. No `.udp()`, no `.tcp()`. So its longest expressible chain is `Eth`, four VLAN tags (`MAX_VLANS`), `Net`, `Transport`. Two gaps follow, and both belong to the oracle rather than the code: - the **arity-8** arm is generated and stays unchecked, because nothing `Matcher` can say is eight elements long; - shapes entering the **IPv6 extension region** cannot be expressed at all, and that is the more interesting loss. `ExtGapCheck` is the subtlest part of the contract and the part the module documentation spends most of its words on, and it has no oracle. Closing it needs extension-header methods on `Matcher`, or a different oracle. Recorded at the call site so the next person does not have to rediscover why the list stops where it does. ## Every arity proves it is not vacuous A shape the generator never produces makes its property pass for the wrong reason, and the higher arities are exactly where that would happen quietly: arity 7 needs a packet carrying *exactly* four VLAN tags, since a tag the shape does not name is a miss. So each property reports its hit rate and fails if it never matched. Measured, and pleasingly uniform: arity 1 matches everything, and arities 2 through 7 each match about 20% -- which is `P(exactly N tags)` for a uniform 0..=4 draw. Every arity is exercised at roughly the same rate rather than the long shapes being starved. ## Verification 1,386 tests green. Under miri with stacked borrows enabled -- the configuration that can see an aliasing fault in the `Fields` split, which the `ub_checks` backstop cannot -- **4,560 cases across 31 properties, no undefined behaviour.** That is up from 963 before the arity work and from 25 per property before the miri budget was reachable at all. Coverage-guided runs on the two differentials, 60 workers: roughly 1.7 billion executions each, no crashes. Co-Authored-By: Claude Opus 5 (1M context) Signed-off-by: Daniel Noland --- net/src/headers/view.rs | 203 ++++++++++++++++++++++++++++++++-------- 1 file changed, 163 insertions(+), 40 deletions(-) diff --git a/net/src/headers/view.rs b/net/src/headers/view.rs index 2d3173367b..ee480085ca 100644 --- a/net/src/headers/view.rs +++ b/net/src/headers/view.rs @@ -2752,52 +2752,175 @@ mod view_mut_properties { use crate::headers::{Headers, Net, ShapedHeaders, Transport}; use crate::vlan::Vlan; - /// Whatever licensed the unchecked call must be deliverable by the walk that has to deliver it. + /// Both walks, at every arity the oracle can express. /// - /// Checked without calling `look_mut`, so a divergence is a clean counterexample rather than - /// undefined behaviour: `as_view_mut` consults `matches`, and the chain below is the same one - /// `look_mut` would run, but safely. - #[test] - fn what_matches_licenses_the_mutable_walk_can_deliver() { - bolero::check!() - .with_generator(ShapedHeaders) - .for_each(|h: &Headers| { - let mut owned = h.clone(); - let licensed = owned.as_view_mut::<(&Eth, &Net, &Transport)>().is_some(); - let deliverable = owned.pat_mut().eth().net().transport().done().is_some(); - assert_eq!( - licensed, deliverable, - "`matches` and the mutable walk disagree, so `look_mut` would reach \ - `unreachable_unchecked`: {h:?}" + /// The decision is what soundness turns on: if `matches` says yes where the walk that must deliver + /// says no, `look`/`look_mut` reach `unreachable_unchecked`. So this compares decisions across the + /// whole family, for the read path and the mutable path both. + /// + /// Arity matters because `matches`, `look` and `look_mut` are generated separately for each one, by + /// a macro, with the VLAN and extension cursors threaded through by hand every time. Testing two + /// arities tested two of eight copies. + /// + /// Pointer identity -- did the two implementations pick the *same* layer, not merely agree that one + /// exists -- is checked separately, at arities 3 and 4, where the tuple can be destructured + /// concretely. That is a correctness question rather than a soundness one, so it is checked deeply + /// at representative arities rather than shallowly at all of them. + macro_rules! arity_agrees { + ($read:ident, $mutable:ident, $shape:ty, $($layer:ident),+) => { + #[test] + fn $read() { + use concurrency::sync::atomic::{AtomicUsize, Ordering}; + static SEEN: AtomicUsize = AtomicUsize::new(0); + static HIT: AtomicUsize = AtomicUsize::new(0); + bolero::check!() + .with_generator(ShapedHeaders) + .for_each(|h: &Headers| { + SEEN.fetch_add(1, Ordering::Relaxed); + let licensed = h.as_view::<$shape>().is_some(); + if licensed { + HIT.fetch_add(1, Ordering::Relaxed); + } + let deliverable = h.pat()$(.$layer())+.done().is_some(); + assert_eq!( + licensed, deliverable, + concat!( + "`matches` and the read walk disagree for ", + stringify!($shape), + ", so `look` would reach `unreachable_unchecked`: {:?}" + ), + h + ); + }); + agreement_is_not_vacuous( + stringify!($shape), + SEEN.load(Ordering::Relaxed), + HIT.load(Ordering::Relaxed), ); - }); - } + } - /// The same for a shape naming a VLAN tag, where the walks have to agree on how many were consumed. - #[test] - fn what_matches_licenses_the_mutable_vlan_walk_can_deliver() { - bolero::check!() - .with_generator(ShapedHeaders) - .for_each(|h: &Headers| { - let mut owned = h.clone(); - let licensed = owned - .as_view_mut::<(&Eth, &Vlan, &Net, &Transport)>() - .is_some(); - let deliverable = owned - .pat_mut() - .eth() - .vlan() - .net() - .transport() - .done() - .is_some(); - assert_eq!( - licensed, deliverable, - "`matches` and the mutable vlan walk disagree: {h:?}" + #[test] + fn $mutable() { + use concurrency::sync::atomic::{AtomicUsize, Ordering}; + static SEEN: AtomicUsize = AtomicUsize::new(0); + static HIT: AtomicUsize = AtomicUsize::new(0); + bolero::check!() + .with_generator(ShapedHeaders) + .for_each(|h: &Headers| { + let mut owned = h.clone(); + SEEN.fetch_add(1, Ordering::Relaxed); + let licensed = owned.as_view_mut::<$shape>().is_some(); + if licensed { + HIT.fetch_add(1, Ordering::Relaxed); + } + let deliverable = owned.pat_mut()$(.$layer())+.done().is_some(); + assert_eq!( + licensed, deliverable, + concat!( + "`matches` and the mutable walk disagree for ", + stringify!($shape), + ", so `look_mut` would reach `unreachable_unchecked`: {:?}" + ), + h + ); + }); + agreement_is_not_vacuous( + stringify!($shape), + SEEN.load(Ordering::Relaxed), + HIT.load(Ordering::Relaxed), ); - }); + } + }; + } + + /// A shape the generator never produces makes its property pass for the wrong reason. + /// + /// Two implementations agreeing that nothing matches is not agreement worth having, and the higher + /// arities are where it would happen quietly: arity 7 needs a packet carrying exactly four VLAN + /// tags, since a tag the shape does not name is a miss. So every arity reports its own hit rate and + /// fails if it never matched at all. + fn agreement_is_not_vacuous(shape: &str, seen: usize, hit: usize) { + println!("{shape}: matched {hit} of {seen}"); + // The default run is a second long, which is plenty here -- these properties manage tens of + // thousands of cases a second -- but a short sample should say so rather than fail. + if seen > 500 { + assert!( + hit > 0, + "{shape} never matched in {seen} packets: the two walks agree only because the \ + generator cannot produce this shape" + ); + } } + // Every arity from one to seven, which is as far as the oracle reaches. + // + // `Matcher`'s vocabulary is `eth`, `vlan`, `net`, `transport`, `vxlan` and `embedded`. The last two + // are unreachable in a builder chain: `Vxlan: Within` and the embedded header sits under + // `Icmp4`/`Icmp6`, both *concrete* layers, and `Matcher` has no concrete-layer methods -- no + // `.udp()`, no `.tcp()`. So the longest chain it can express is `Eth`, four VLAN tags + // (`MAX_VLANS`), `Net`, `Transport`: seven. + // + // Two gaps follow, and both are about the oracle rather than the code: + // + // * the **arity-8** arm of the macro is generated and goes unchecked here, because nothing the + // oracle can say is eight elements long; + // * shapes entering the **IPv6 extension region** cannot be expressed at all, and that is the + // more interesting loss -- `ExtGapCheck` is the subtlest part of the contract, the part the + // module documentation spends most of its words on, and it has no oracle. Closing it means + // either extension-header methods on `Matcher` or a different oracle. + arity_agrees!(read_1, mutable_1, (&Eth,), eth); + arity_agrees!(read_2, mutable_2, (&Eth, &Net), eth, net); + arity_agrees!( + read_3, + mutable_3, + (&Eth, &Net, &Transport), + eth, + net, + transport + ); + arity_agrees!( + read_4, + mutable_4, + (&Eth, &Vlan, &Net, &Transport), + eth, + vlan, + net, + transport + ); + arity_agrees!( + read_5, + mutable_5, + (&Eth, &Vlan, &Vlan, &Net, &Transport), + eth, + vlan, + vlan, + net, + transport + ); + arity_agrees!( + read_6, + mutable_6, + (&Eth, &Vlan, &Vlan, &Vlan, &Net, &Transport), + eth, + vlan, + vlan, + vlan, + net, + transport + ); + arity_agrees!( + read_7, + mutable_7, + (&Eth, &Vlan, &Vlan, &Vlan, &Vlan, &Net, &Transport), + eth, + vlan, + vlan, + vlan, + vlan, + net, + transport + ); + /// Exercise the multi-`&mut` split itself: write through every reference and read the writes back. /// /// The assertions are almost beside the point. What matters is that the references are *created and From ddfc3e0179e8f8cd06e1ab26f6f0b4367effa9df Mon Sep 17 00:00:00 2001 From: Daniel Noland Date: Sat, 8 Aug 2026 17:40:07 -0600 Subject: [PATCH 13/33] test(net): Reach the quoted packet inside an ICMP error, and the extension region `embedded_view.rs` sat at 33.9% line coverage, and the reason was not that nobody had written tests for it. Its hand-written tests are careful and thorough. Every one of them builds its packet through `HeaderStack` or `header_chain`, which pin the shape at `(Eth, Ipv4, Icmp4)` outside and `(Ipv4, Tcp)` inside and offer no way to attach an extension header to either side. Six of the eight `as_embedded` arities and the whole embedded extension region were unreachable by construction. The generators were the binding constraint, again. All six construction sites of `CommonHeaders` set `embedded_ip: None`, so neither it nor `ShapedHeaders`, which builds on it, can produce an ICMP error carrying a quoted packet at all. `ShapedIcmpError` does: it varies the outer VLAN tags and extension headers so the outer arity spans the range of `as_embedded` impls, varies whether the quote is present, and varies the quoted packet's network layer, extensions and truncated transport. The quoted family usually follows the quoting family, because that is what a real ICMP error looks like, but not always -- a mismatch is where two independent structural walks are most likely to disagree. What the new differential properties compare is the pairing soundness rests on, which is not the obvious one. `as_embedded_mut` *decides* with `Sealed::matches`, walking `EmbeddedHeaders` through `EmbeddedStep`. `look_mut` then *delivers* through `EmbeddedMatcherMut`, a separate implementation with its own pre-split fields and its own gap check, and unwraps that chain with `unreachable_unchecked` on the strength of the first one's answer. If they disagree the result is undefined behaviour, not a wrong answer. Every arity of both is now checked against the `pat()` oracle, read path and mutable path. Two claims in `view.rs` were wrong and are corrected here. `Matcher` does have concrete-layer and extension-header methods -- `matcher_net!`, `matcher_ext!` and `matcher_transport!` give it `.ipv4()`, `.hop_by_hop()`, `.tcp()` and the rest -- so neither gap that comment recorded was real. The arity-8 arm is now checked, and so is the extension region, where `ExtGapCheck` switches from skipping extensions silently to requiring all of them consumed. That is the subtlest part of the contract and the part the module documentation spends most of its words on, and it had no oracle at all. Hit rates are measured and asserted rather than hoped for, which mattered twice. A shape naming three extensions in sequence matched 2 packets in 36,000 -- honest, and useless -- until `ext_run` learned to follow RFC 8200's recommended order a quarter of the time; it now matches about 200. And naming one extension inside a quoted packet compounds six conditions, including `P(no VLAN tags) = 1/5`, which put `(&Ipv6, &DestOpts, &TruncatedTcp)` at 6 hits in 23,910. `ShapedQuote` produces that shape every time and fuzzes the contents instead: 32,000 hits. The division is deliberate -- a property comparing hit against miss needs both, a property asking only which layer was selected gets nothing from a packet it skips. Verified by breaking the code three ways. An off-by-one on the embedded extension cursor fails exactly the four extension-region differentials, in one second, with a counterexample carrying exactly one extension -- the case where a stalled cursor makes the gap check see `len 1 != ec 0`. Dropping the shape check from arity 7's `as_embedded` fails exactly `read_outer_7`, in 87ms, while `mutable_outer_7` correctly stays green. Reading one extension slot too far fails the six `same_layers_ext_*` properties through the vacuity guard, which reports that the shape was never produced -- the same class of defect this campaign has now found five times. Coverage: `embedded_view.rs` 33.9% -> 86.0%, `pat.rs` 55.4% -> 63.2% without a test written for it, `view.rs` 50.1% -> 53.8%, `net/` 66.8% -> 69.8%. The 49 lines still uncovered are all unreachable: three `unreachable_unchecked` arms, which is the point of them, and 44 in the arity-1 and arity-2 `as_embedded` arms, which cannot be instantiated -- asking for either is a compile error, so they could be deleted. Co-Authored-By: Claude Opus 5 (1M context) Signed-off-by: Daniel Noland --- net/src/headers/embedded_view.rs | 646 +++++++++++++++++++++++++++++++ net/src/headers/mod.rs | 277 ++++++++++++- net/src/headers/view.rs | 108 +++++- 3 files changed, 997 insertions(+), 34 deletions(-) diff --git a/net/src/headers/embedded_view.rs b/net/src/headers/embedded_view.rs index fe6d5eba0e..ad52e28bf7 100644 --- a/net/src/headers/embedded_view.rs +++ b/net/src/headers/embedded_view.rs @@ -1790,3 +1790,649 @@ mod tests { } } } + +// =========================================================================== +// Differential properties +// =========================================================================== + +#[cfg(test)] +#[allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)] // fine to unwrap in tests +mod embedded_view_properties { + use super::*; + use crate::eth::Eth; + use crate::headers::{Headers, ShapedIcmpError, ShapedQuote}; + use crate::icmp4::Icmp4; + use crate::icmp6::Icmp6; + use crate::vlan::Vlan; + + /// Both structural walks over a quoted packet, across the range of outer and inner shapes. + /// + /// The pairing this checks is the one soundness rests on, and it is not the obvious one. + /// `as_embedded`/`as_embedded_mut` *decide* with [`embedded_sealed::Sealed::matches`], which + /// walks [`EmbeddedHeaders`] through [`EmbeddedStep`]. [`EmbeddedLookMut::look_mut`] then + /// *delivers* through [`EmbeddedMatcherMut`], a completely separate implementation with its own + /// pre-split [`EmbeddedFields`](super::super::pat::EmbeddedFields) and its own gap check, and it + /// unwraps that chain with `unreachable_unchecked` on the strength of the first one's answer. + /// Two implementations, one of them licensing the other to skip its own `None` branch. If they + /// ever disagree the result is undefined behaviour, not a wrong answer. + /// + /// So the oracle here is `pat()`/`pat_mut()`'s `.embedded()` chain, which is the same machinery + /// `look_mut` uses -- making this differential test exactly the invariant `look_mut` assumes. + /// + /// Arity matters twice over, because `matches`, `look` and `look_mut` are generated per *inner* + /// arity while `as_embedded`/`as_embedded_mut` are generated per *outer* arity, each threading + /// the extension cursor by hand. Before this the suite reached one outer arity and one inner + /// arity, of eight and three. + macro_rules! embedded_agrees { + ( + $read:ident, $mutable:ident, + ($outer:ty, $($ol:ident),+), + ($inner:ty, $($il:ident),+) + ) => { + #[test] + fn $read() { + use concurrency::sync::atomic::{AtomicUsize, Ordering}; + static SEEN: AtomicUsize = AtomicUsize::new(0); + static HIT: AtomicUsize = AtomicUsize::new(0); + bolero::check!() + .with_generator(ShapedIcmpError) + .for_each(|h: &Headers| { + SEEN.fetch_add(1, Ordering::Relaxed); + let licensed = h + .as_view::<$outer>() + .is_some_and(|w| w.as_embedded::<$inner>().is_some()); + if licensed { + HIT.fetch_add(1, Ordering::Relaxed); + } + let deliverable = + h.pat()$(.$ol())+.embedded()$(.$il())+.done().is_some(); + assert_eq!( + licensed, deliverable, + concat!( + "`matches` and the read walk disagree for ", + stringify!($inner), + " inside ", + stringify!($outer), + ", so `look` would reach `unreachable_unchecked`: {:?}" + ), + h + ); + }); + agreement_is_not_vacuous( + concat!(stringify!($inner), " in ", stringify!($outer)), + SEEN.load(Ordering::Relaxed), + HIT.load(Ordering::Relaxed), + ); + } + + #[test] + fn $mutable() { + use concurrency::sync::atomic::{AtomicUsize, Ordering}; + static SEEN: AtomicUsize = AtomicUsize::new(0); + static HIT: AtomicUsize = AtomicUsize::new(0); + bolero::check!() + .with_generator(ShapedIcmpError) + .for_each(|h: &Headers| { + let mut owned = h.clone(); + SEEN.fetch_add(1, Ordering::Relaxed); + // `as_embedded_mut`'s reference cannot outlive the `and_then` closure it + // would be produced in, so this asks the question without keeping it. + let licensed = match owned.as_view_mut::<$outer>() { + Some(w) => w.as_embedded_mut::<$inner>().is_some(), + None => false, + }; + if licensed { + HIT.fetch_add(1, Ordering::Relaxed); + } + let deliverable = + owned.pat_mut()$(.$ol())+.embedded()$(.$il())+.done().is_some(); + assert_eq!( + licensed, deliverable, + concat!( + "`matches` and the mutable walk disagree for ", + stringify!($inner), + " inside ", + stringify!($outer), + ", so `look_mut` would reach `unreachable_unchecked`: {:?}" + ), + h + ); + }); + agreement_is_not_vacuous( + concat!(stringify!($inner), " in ", stringify!($outer)), + SEEN.load(Ordering::Relaxed), + HIT.load(Ordering::Relaxed), + ); + } + }; + } + + /// A shape the generator never produces makes its property pass for the wrong reason. + /// + /// Two shapes compound here -- an outer arity and an inner one -- so a property can starve much + /// more quietly than in [`view`](super::super::view): demanding four VLAN tags *and* a + /// particular extension inside the quoted packet multiplies two small probabilities. Each pair + /// reports its own hit rate, and the pairs are chosen to vary one dimension at a time for + /// exactly that reason. + fn agreement_is_not_vacuous(shape: &str, seen: usize, hit: usize) { + println!("{shape}: matched {hit} of {seen}"); + // Twenty thousand rather than [`view`](super::super::view)'s five hundred, because + // compounding costs an order of magnitude: the widest pair here matches about one packet in + // twenty, and `(&Ipv6, &HopByHop, &TruncatedTcp)` inside an ICMPv6 error matches about one + // in a thousand. Five hundred cases at that rate would fail this check outright half the + // time it ran. A default one-second run manages twenty-five thousand, so the guard still + // bites in CI; a deliberately short run should say nothing rather than say something false. + if seen > 20_000 { + assert!( + hit > 0, + "{shape} never matched in {seen} packets: the two walks agree only because the \ + generator cannot produce this shape" + ); + } + } + + type O4V4 = (&'static Eth, &'static Ipv4, &'static Icmp4); + type O4V6 = (&'static Eth, &'static Ipv6, &'static Icmp6); + + // ---- Inner shapes, at the one outer arity the builders could already express ---------- + // + // Varying the inner shape alone: network layer, both enum forms, every truncated transport, + // and the extension region. + embedded_agrees!( + read_inner_v4_tcp, + mutable_inner_v4_tcp, + (O4V4, eth, ipv4, icmp4), + ((&Ipv4, &TruncatedTcp), ipv4, tcp) + ); + embedded_agrees!( + read_inner_v6_udp, + mutable_inner_v6_udp, + (O4V6, eth, ipv6, icmp6), + ((&Ipv6, &TruncatedUdp), ipv6, udp) + ); + embedded_agrees!( + read_inner_net_only, + mutable_inner_net_only, + (O4V4, eth, ipv4, icmp4), + ((&Net,), net) + ); + embedded_agrees!( + read_inner_v6_only, + mutable_inner_v6_only, + (O4V6, eth, ipv6, icmp6), + ((&Ipv6,), ipv6) + ); + // An ICMP error quoting an ICMP packet -- a `TruncatedIcmp4` inside an `Icmp4`. Legitimate: + // an unreachable in response to a ping quotes the echo request. + embedded_agrees!( + read_inner_icmp_in_icmp4, + mutable_inner_icmp_in_icmp4, + (O4V4, eth, ipv4, icmp4), + ((&Ipv4, &TruncatedIcmp4), ipv4, icmp4) + ); + embedded_agrees!( + read_inner_icmp_in_icmp6, + mutable_inner_icmp_in_icmp6, + (O4V6, eth, ipv6, icmp6), + ((&Ipv6, &TruncatedIcmp6), ipv6, icmp6) + ); + + // ---- The embedded extension region --------------------------------------------------- + // + // `ext_gap_ok_embedded` versus `ext_gap_ok_mut_embedded`: naming an extension switches the + // quoted packet from skip-extensions-silently to consume-them-all, so these match only a quote + // carrying that extension and no other. The read pair shares `ext_gap_ok_embedded` between the + // two walks and so tests the by-hand `ec` threading; the mutable pair runs two genuinely + // different implementations against each other, one over `EmbeddedHeaders` and one over a + // pre-split `EmbeddedFields`. + embedded_agrees!( + read_inner_ext_v6, + mutable_inner_ext_v6, + (O4V6, eth, ipv6, icmp6), + ((&Ipv6, &HopByHop, &TruncatedTcp), ipv6, hop_by_hop, tcp) + ); + embedded_agrees!( + read_inner_ext_v4_auth, + mutable_inner_ext_v4_auth, + (O4V4, eth, ipv4, icmp4), + ((&Ipv4, &Ipv4Auth, &TruncatedTcp), ipv4, ipv4_auth, tcp) + ); + // Entering the region and stopping there: the gap check runs at the transport step, so with no + // transport named the remaining extensions are simply left unvisited. Telling this apart from + // the two above is the whole reason the check sits where it does. + embedded_agrees!( + read_inner_ext_no_transport, + mutable_inner_ext_no_transport, + (O4V6, eth, ipv6, icmp6), + ((&Ipv6, &HopByHop), ipv6, hop_by_hop) + ); + + // ---- Outer arities ------------------------------------------------------------------- + // + // `as_embedded` and `as_embedded_mut` are written out once per outer arity, eight times. Arities + // 1 and 2 cannot be instantiated: both require `EmbeddedHeaders: Within`, which holds + // only for `Icmp4` and `Icmp6`, and neither can appear that early. `Eth` is the only layer with + // `Within<()>`, so an arity-1 shape must be `(&Eth,)`; an arity-2 shape must then end in + // something `Within`, and no ICMP layer is. Asking for either is a compile error -- + // `Icmp4: Within<()> is not satisfied` and `Icmp4: Within is not satisfied` -- so those + // two macro arms are 44 lines that can never run and could be deleted. + // + // 3 through 8 is therefore all of them. The inner shape is held cheap on purpose here, so that a + // thin outer shape is the only improbable thing each property asks for. + embedded_agrees!( + read_outer_4, + mutable_outer_4, + ((&Eth, &Vlan, &Ipv4, &Icmp4), eth, vlan, ipv4, icmp4), + ((&Ipv4, &TruncatedTcp), ipv4, tcp) + ); + embedded_agrees!( + read_outer_5, + mutable_outer_5, + ( + (&Eth, &Vlan, &Vlan, &Ipv6, &Icmp6), + eth, + vlan, + vlan, + ipv6, + icmp6 + ), + ((&Ipv6, &TruncatedUdp), ipv6, udp) + ); + embedded_agrees!( + read_outer_6, + mutable_outer_6, + ( + (&Eth, &Vlan, &Vlan, &Vlan, &Ipv4, &Icmp4), + eth, + vlan, + vlan, + vlan, + ipv4, + icmp4 + ), + // An inner shape that can miss on a quote that *is* present, so the shape check's own + // rejection is reached rather than only the absent-quote early return above it. + ((&Ipv4, &TruncatedTcp), ipv4, tcp) + ); + embedded_agrees!( + read_outer_7, + mutable_outer_7, + ( + (&Eth, &Vlan, &Vlan, &Vlan, &Vlan, &Ipv4, &Icmp4), + eth, + vlan, + vlan, + vlan, + vlan, + ipv4, + icmp4 + ), + ((&Ipv4, &TruncatedTcp), ipv4, tcp) + ); + // Arity 8, and the only outer shape that also enters the *outer* extension region: `Icmp6` + // directly after a `HopByHop` makes the outer gap check strict too. + embedded_agrees!( + read_outer_8, + mutable_outer_8, + ( + (&Eth, &Vlan, &Vlan, &Vlan, &Vlan, &Ipv6, &HopByHop, &Icmp6), + eth, + vlan, + vlan, + vlan, + vlan, + ipv6, + hop_by_hop, + icmp6 + ), + // `Ipv6` rather than `Net`: a quote of the other family misses, which is the only way to + // reach this arity's shape-check rejection. + ((&Ipv6,), ipv6) + ); + + // The `EmbeddedTransport` enum has no read-side property above, and that is an asymmetry in + // `pat.rs` rather than a choice here: `EmbeddedMatcherMut::transport` exists but + // `EmbeddedMatcher::transport` does not, so the immutable oracle cannot name the enum even + // though every concrete variant and every other layer is paired between the two. The mutable + // side is checked below; adding the missing method would let the read side follow in one line. + #[test] + fn mutable_inner_transport_enum() { + use concurrency::sync::atomic::{AtomicUsize, Ordering}; + static SEEN: AtomicUsize = AtomicUsize::new(0); + static HIT: AtomicUsize = AtomicUsize::new(0); + bolero::check!() + .with_generator(ShapedIcmpError) + .for_each(|h: &Headers| { + let mut owned = h.clone(); + SEEN.fetch_add(1, Ordering::Relaxed); + let licensed = match owned.as_view_mut::() { + Some(w) => w.as_embedded_mut::<(&Net, &EmbeddedTransport)>().is_some(), + None => false, + }; + if licensed { + HIT.fetch_add(1, Ordering::Relaxed); + } + let deliverable = owned + .pat_mut() + .eth() + .ipv4() + .icmp4() + .embedded() + .net() + .transport() + .done() + .is_some(); + assert_eq!( + licensed, deliverable, + "`matches` and the mutable walk disagree for (&Net, &EmbeddedTransport): {h:?}" + ); + }); + agreement_is_not_vacuous( + "(&Net, &EmbeddedTransport)", + SEEN.load(Ordering::Relaxed), + HIT.load(Ordering::Relaxed), + ); + } + + // ---- Delivering, not merely deciding ------------------------------------------------- + + /// Write through every reference `look_mut` hands out, in the extension-region shape. + /// + /// The properties above compare *decisions*. This one takes the decision up on its offer: it + /// calls `look_mut`, which runs the `unreachable_unchecked` the decision licenses, and then + /// writes through all three references. The assertions are close to beside the point -- what + /// matters is that three `&mut` into one `EmbeddedHeaders` are created and written through, so + /// that miri with stacked borrows enabled can judge whether + /// [`EmbeddedFields`](super::super::pat::EmbeddedFields) handed out two paths to the same + /// layer. The arity-3 shape is the one that borrows from all three of its fields at once, + /// including a slice element, which is the hardest of them to split soundly. + fn exercise_the_embedded_split() { + bolero::check!() + .with_generator(ShapedIcmpError) + .for_each(|h: &Headers| { + let mut owned = h.clone(); + let Some(outer) = owned.as_view_mut::() else { + return; + }; + let Some(ew) = outer.as_embedded_mut::<(&Ipv6, &HopByHop, &TruncatedTcp)>() else { + return; + }; + let (ip, ext, tcp) = ew.look_mut(); + + let want_hops = ip.hop_limit().wrapping_add(1); + ip.set_hop_limit(want_hops); + let seen_ext = ext.next_header(); + let seen_tcp = matches!(tcp, TruncatedTcp::FullHeader(_)); + + assert_eq!( + ip.hop_limit(), + want_hops, + "the write through ipv6 did not stick" + ); + assert_eq!( + ext.next_header(), + seen_ext, + "the extension header changed under a write to ipv6" + ); + assert_eq!( + matches!(tcp, TruncatedTcp::FullHeader(_)), + seen_tcp, + "the quoted transport changed under a write to ipv6" + ); + }); + } + + /// Shards of [`exercise_the_embedded_split`], so the machine can be used. + /// + /// Same reasoning as the shards in [`view`](super::super::view): under miri this is the + /// expensive property and the one that matters most, and its cost is wall-clock rather than + /// cores. Each shard seeds itself from the OS, so `N` shards explore `N` independent streams + /// and nextest runs them concurrently. Eight rather than sixteen -- `view`'s split is the wider + /// one and should keep the larger share of the cores. + macro_rules! split_shards { + ($($name:ident),* $(,)?) => { + $( + #[test] + fn $name() { + exercise_the_embedded_split(); + } + )* + }; + } + + split_shards!( + the_embedded_split_hands_out_distinct_layers, + embedded_split_shard_2, + embedded_split_shard_3, + embedded_split_shard_4, + embedded_split_shard_5, + embedded_split_shard_6, + embedded_split_shard_7, + embedded_split_shard_8, + ); + + /// `look` and `look_mut` pick the same slots, one inner shape per instantiation. + /// + /// Agreeing that a match exists is weaker than agreeing on *which* layers were selected, and + /// this is the only thing in the suite that asks the stronger question across the family. + /// + /// It is also the only thing that *calls* `look` and `look_mut` at more than a couple of shapes, + /// which matters more than it looks: the differential properties above go through + /// [`embedded_sealed::Sealed::matches`] and so exercise [`EmbeddedStep`], while `look_mut` is the + /// sole caller of [`EmbeddedStepMut`] -- a second, separate set of per-layer impls that delegate + /// into [`EmbeddedMatcherMut`]. Deciding a shape matches never runs a line of it. So there is one + /// instantiation per layer that has an `EmbeddedStepMut` impl, and the extension ones are the + /// point: one implementation indexes `net_ext` by a cursor, the other consumes a slice, and + /// nothing before this compared what they returned. + macro_rules! same_layers { + ($name:ident, $gen:expr, $outer:ty, $shape:ty, $($binding:ident),+) => { + #[test] + fn $name() { + use concurrency::sync::atomic::{AtomicUsize, Ordering}; + static SEEN: AtomicUsize = AtomicUsize::new(0); + static HIT: AtomicUsize = AtomicUsize::new(0); + bolero::check!() + .with_generator($gen) + .for_each(|h: &Headers| { + let mut owned = h.clone(); + SEEN.fetch_add(1, Ordering::Relaxed); + // `h` and `&mut h` cannot be held at once, so the packet is pinned across + // both calls and the raw addresses compared afterwards. + let immutable = { + let Some(outer) = owned.as_view::<$outer>() else { + return; + }; + let Some(ew) = outer.as_embedded::<$shape>() else { + return; + }; + let ($($binding,)+) = ew.look(); + ($(std::ptr::from_ref($binding),)+) + }; + HIT.fetch_add(1, Ordering::Relaxed); + let mutable = { + let outer = owned.as_view_mut::<$outer>().unwrap_or_else(|| { + unreachable!("the same packet matched a moment ago") + }); + let ew = outer.as_embedded_mut::<$shape>().unwrap_or_else(|| { + unreachable!("the same shape matched a moment ago") + }); + let ($($binding,)+) = ew.look_mut(); + ($(std::ptr::from_ref(&*$binding),)+) + }; + assert_eq!( + immutable, mutable, + concat!( + "`look` and `look_mut` selected different layers of the quoted \ + packet for ", + stringify!($shape) + ) + ); + }); + agreement_is_not_vacuous( + concat!("look/look_mut ", stringify!($shape)), + SEEN.load(Ordering::Relaxed), + HIT.load(Ordering::Relaxed), + ); + } + }; + } + + /// The strict gap check applies to the transport *enum* too, not just concrete variants. + /// + /// Stated as a direct assertion rather than a differential, because the immutable oracle cannot + /// express it -- see the note above `mutable_inner_transport_enum`. That leaves the enum's own + /// gap check with nothing else reaching it, and it is worth reaching: naming an extension makes + /// the check strict, so a second, unnamed extension must turn a hit into a miss even though the + /// transport pattern is the permissive one. + /// + /// The second extension is a clone of the first, which keeps this free of any assumption about + /// what other extension types exist while still being two headers where the shape names one. + #[test] + fn the_transport_enum_still_refuses_an_unconsumed_extension() { + bolero::check!() + .with_generator(ShapedQuote { ext: 0, v4: false }) + .for_each(|h: &Headers| { + let mut owned = h.clone(); + let quoted = owned + .embedded_ip_mut() + .unwrap_or_else(|| unreachable!("ShapedQuote always attaches a quote")); + let first = quoted + .net_ext + .first() + .unwrap_or_else(|| unreachable!("ShapedQuote always places one extension")) + .clone(); + quoted.net_ext.push(first); + + let outer = owned + .as_view::() + .unwrap_or_else(|| unreachable!("ShapedQuote draws an ICMPv6 error")); + assert!( + outer + .as_embedded::<(&Ipv6, &HopByHop, &EmbeddedTransport)>() + .is_none(), + "naming one extension of two matched anyway, so the gap check did not run for \ + the transport enum" + ); + // Not naming an extension skips the region silently, which is the other half of the + // same rule and shows the miss above is the gap check rather than a broken shape. + assert!( + outer.as_embedded::<(&Ipv6, &EmbeddedTransport)>().is_some(), + "a shape that never entered the extension region was refused" + ); + }); + } + + // Network layer, both concrete and as the enum, at inner arity 1. These shapes are common + // enough in the broad generator's output -- around one packet in twenty -- to use it. + same_layers!(same_layers_v4_only, ShapedIcmpError, O4V4, (&Ipv4,), ip); + same_layers!(same_layers_v6_only, ShapedIcmpError, O4V6, (&Ipv6,), ip); + same_layers!(same_layers_net_only, ShapedIcmpError, O4V4, (&Net,), net); + + // Every truncated transport, and the transport enum. + same_layers!( + same_layers_v4_tcp, + ShapedIcmpError, + O4V4, + (&Ipv4, &TruncatedTcp), + ip, + tcp + ); + same_layers!( + same_layers_v6_udp, + ShapedIcmpError, + O4V6, + (&Ipv6, &TruncatedUdp), + ip, + udp + ); + same_layers!( + same_layers_icmp_in_icmp4, + ShapedIcmpError, + O4V4, + (&Ipv4, &TruncatedIcmp4), + ip, + icmp + ); + same_layers!( + same_layers_icmp_in_icmp6, + ShapedIcmpError, + O4V6, + (&Ipv6, &TruncatedIcmp6), + ip, + icmp + ); + same_layers!( + same_layers_transport_enum, + ShapedIcmpError, + O4V4, + (&Net, &EmbeddedTransport), + net, + transport + ); + + // Every extension header, which is where the two walks diverge most: one indexes `net_ext` by a + // cursor, the other consumes a slice. These use the narrow generator, because on the broad one + // they matched six to twenty-three packets in twenty-four thousand -- see [`ShapedQuote`]. + // + // Each therefore sees a quoted packet with exactly one extension, and that is not a limitation + // of the generator but of what is expressible: embedded shapes stop at arity 3, so a shape can + // name at most one extension, and naming one makes the gap check strict -- a second, unconsumed + // extension is a miss. One is the only number of extensions a matching quote can carry. Cursor + // arithmetic past position zero is consequently unreachable from here, and is covered by the + // differential properties above, which do draw multi-extension quotes and compare the two walks + // on the misses. + same_layers!( + same_layers_ext_hop_by_hop, + ShapedQuote { ext: 0, v4: false }, + O4V6, + (&Ipv6, &HopByHop, &TruncatedTcp), + ip, + ext, + tcp + ); + same_layers!( + same_layers_ext_dest_opts, + ShapedQuote { ext: 1, v4: false }, + O4V6, + (&Ipv6, &DestOpts, &TruncatedTcp), + ip, + ext, + tcp + ); + same_layers!( + same_layers_ext_routing, + ShapedQuote { ext: 2, v4: false }, + O4V6, + (&Ipv6, &Routing, &TruncatedTcp), + ip, + ext, + tcp + ); + same_layers!( + same_layers_ext_fragment, + ShapedQuote { ext: 3, v4: false }, + O4V6, + (&Ipv6, &Fragment, &TruncatedTcp), + ip, + ext, + tcp + ); + same_layers!( + same_layers_ext_v6_auth, + ShapedQuote { ext: 4, v4: false }, + O4V6, + (&Ipv6, &Ipv6Auth, &TruncatedTcp), + ip, + ext, + tcp + ); + same_layers!( + same_layers_ext_v4_auth, + ShapedQuote { ext: 0, v4: true }, + O4V4, + (&Ipv4, &Ipv4Auth, &TruncatedTcp), + ip, + ext, + tcp + ); +} diff --git a/net/src/headers/mod.rs b/net/src/headers/mod.rs index cf409cee4d..68ed7e0176 100644 --- a/net/src/headers/mod.rs +++ b/net/src/headers/mod.rs @@ -1194,7 +1194,10 @@ where mod contract { use crate::eth::ethtype::CommonEthType; use crate::eth::{Eth, GenWithEthType}; - use crate::headers::{Headers, MAX_NET_EXTENSIONS, MAX_VLANS, Net, NetExt, Transport}; + use crate::headers::{ + EmbeddedHeaders, EmbeddedTransport, Headers, MAX_NET_EXTENSIONS, MAX_VLANS, Net, NetExt, + Transport, + }; use crate::icmp4::Icmp4; use crate::icmp6::Icmp6; use crate::ipv4; @@ -1290,31 +1293,267 @@ mod contract { } // Extension headers hang off the net layer, so there is nothing to attach them to without - // one. The IPv4 authentication header is the only one that belongs on a v4 packet. - let ipv4 = matches!(headers.net, Some(Net::Ipv4(_))); + // one. if headers.net.is_some() { - let exts = - driver.gen_usize(Bound::Included(&0), Bound::Included(&MAX_NET_EXTENSIONS))?; - for _ in 0..exts { - let ext = if ipv4 { - NetExt::Ipv4Auth(driver.produce()?) - } else { - match driver.gen_u8(Bound::Included(&0), Bound::Included(&4))? { - 0 => NetExt::HopByHop(driver.produce()?), - 1 => NetExt::DestOpts(driver.produce()?), - 2 => NetExt::Routing(driver.produce()?), - 3 => NetExt::Fragment(driver.produce()?), - _ => NetExt::Ipv6Auth(driver.produce()?), - } - }; - headers.net_ext.push(ext); - } + headers.net_ext = ext_run(driver, matches!(headers.net, Some(Net::Ipv4(_))))?; } Some(headers) } } + /// Draw a run of 0..=[`MAX_NET_EXTENSIONS`] extension headers for the given family. + /// + /// The IPv4 authentication header is the only extension that belongs on a v4 packet, so `v4` + /// selects between one choice and five rather than merely reweighting them. + /// + /// A quarter of the v6 runs follow RFC 8200's recommended order instead of drawing each slot + /// independently. Without that bias a shape naming three specific extensions in sequence is + /// drawn about once in fifteen thousand packets, which is enough to keep a property honest and + /// nowhere near enough for it to find anything: measured at two hits in 36,000 cases. The + /// ordered runs are also the ones a real peer sends, so this makes the generator both more + /// useful and more realistic. + fn ext_run( + driver: &mut D, + v4: bool, + ) -> Option> { + let mut out = ArrayVec::default(); + let count = driver.gen_usize(Bound::Included(&0), Bound::Included(&MAX_NET_EXTENSIONS))?; + let ordered = driver.gen_u8(Bound::Included(&0), Bound::Included(&3))? == 0; + for slot in 0..count { + let pick = if ordered { + u8::try_from(slot).unwrap_or(u8::MAX) + } else { + driver.gen_u8(Bound::Included(&0), Bound::Included(&4))? + }; + out.push(one_ext(driver, v4, pick)?); + } + Some(out) + } + + /// Draw one extension header, with fuzzed contents. + /// + /// `pick` indexes the IPv6 extension order RFC 8200 recommends -- 0 hop-by-hop, 1 destination + /// options, 2 routing, 3 fragment, anything else authentication -- and is ignored for `v4`, + /// where the IPv4 authentication header is the only extension there is. + fn one_ext(driver: &mut D, v4: bool, pick: u8) -> Option { + if v4 { + return Some(NetExt::Ipv4Auth(driver.produce()?)); + } + Some(match pick { + 0 => NetExt::HopByHop(driver.produce()?), + 1 => NetExt::DestOpts(driver.produce()?), + 2 => NetExt::Routing(driver.produce()?), + 3 => NetExt::Fragment(driver.produce()?), + _ => NetExt::Ipv6Auth(driver.produce()?), + }) + } + + /// Draws an ICMP error whose quoted packet carries exactly one chosen extension header, and + /// nothing else in the way. + /// + /// Companion to [`ShapedIcmpError`], and the division between them is deliberate. A property + /// comparing *hit against miss* needs both, so it wants the broad generator. A property that + /// only inspects matches -- does `look` pick the same layer `look_mut` does -- gets nothing from + /// a packet it skips, so for that one a narrow generator is not a weaker test but a stronger + /// one. + /// + /// The difference is not marginal. Naming a specific extension inside a quoted packet compounds + /// six independent conditions, one of which is `P(no VLAN tags) = 1/5`, since a tag the outer + /// shape does not name is a miss. Measured on [`ShapedIcmpError`], + /// `(&Ipv6, &DestOpts, &TruncatedTcp)` matched 6 packets in 23,910: a property doing real work + /// six times a second, and a hit rate low enough that asserting it is ever non-zero is itself a + /// coin flip. This generator produces that shape every time, and fuzzes the contents instead. + #[allow(dead_code)] // constructed through `.with_generator()` + pub struct ShapedQuote { + /// Which extension header to place, as the `pick` index [`one_ext`] uses. + pub ext: u8, + /// Family of both the quoting message and the packet it quotes. + pub v4: bool, + } + + impl ValueGenerator for ShapedQuote { + type Output = Headers; + + fn generate(&self, driver: &mut D) -> Option { + let eth_type = if self.v4 { + CommonEthType::Ipv4 + } else { + CommonEthType::Ipv6 + }; + let eth = GenWithEthType(eth_type.into()).generate(driver)?; + + let mut quoted_ext = ArrayVec::default(); + quoted_ext.push(one_ext(driver, self.v4, self.ext)?); + let quoted_transport = EmbeddedTransport::Tcp(driver.produce()?); + + let (net, transport, quoted_net) = if self.v4 { + ( + Net::Ipv4( + ipv4::GenWithNextHeader(ipv4::CommonNextHeader::Icmp4.into()) + .generate(driver)?, + ), + Transport::Icmp4(driver.produce()?), + Net::Ipv4( + ipv4::GenWithNextHeader(ipv4::CommonNextHeader::Tcp.into()) + .generate(driver)?, + ), + ) + } else { + ( + Net::Ipv6( + ipv6::GenWithNextHeader(ipv6::CommonNextHeader::Icmp6.into()) + .generate(driver)?, + ), + Transport::Icmp6(driver.produce()?), + Net::Ipv6( + ipv6::GenWithNextHeader(ipv6::CommonNextHeader::Tcp.into()) + .generate(driver)?, + ), + ) + }; + + Some(Headers { + eth: Some(eth), + // No VLAN tags and no outer extensions: a tag the outer shape does not name is a + // miss, and this generator exists to stop producing misses. + vlan: ArrayVec::default(), + net: Some(net), + net_ext: ArrayVec::default(), + transport: Some(transport), + udp_encap: None, + embedded_ip: Some(EmbeddedHeaders::new( + Some(quoted_net), + Some(quoted_transport), + quoted_ext, + None, + )), + }) + } + } + + /// Draws an ICMP error message quoting an inner packet, with the layer structure of *both* the + /// outer message and the quoted packet varying. + /// + /// [`CommonHeaders`] cannot produce one at all: all six of its construction sites set + /// `embedded_ip: None`, so [`ShapedHeaders`], which builds on it, cannot either. That is a + /// better explanation of [`embedded_view`](crate::headers::embedded_view) sitting at 34% line + /// coverage than any claim about missing tests -- its hand-written tests are thorough, but every + /// one of them builds its packet through `HeaderStack`/`header_chain`, which pin the shape at + /// `(Eth, Ipv4, Icmp4)` outside and `(Ipv4, Tcp)` inside and offer no way to attach an extension + /// header to either side. Six of the eight `as_embedded` arities and the whole embedded + /// extension region were unreachable by construction. + /// + /// What varies here is what `EmbeddedShape`, `EmbeddedStep` and + /// [`ExtGapCheck`](crate::headers::pat::ExtGapCheck) actually turn on: + /// + /// * outer VLAN tags and outer extension headers, so the outer arity spans the range of + /// `as_embedded` impls instead of only the one a builder can express; + /// * whether the embedded section is present at all -- an ICMP message that is not an error + /// quotes nothing, and every embedded shape must miss on it; + /// * the inner network layer, its extension headers, and its truncated transport, including a + /// quote that stops before the transport header and the ICMP-inside-ICMP case. + /// + /// The inner family follows the outer family most of the time, because that is what a real ICMP + /// error looks like: an `ICMPv4` message quotes an IPv4 packet. It deliberately does not always. + /// A family mismatch is where two independent structural walks are most likely to disagree, so + /// it wants drawing rather than assuming away. + /// + /// Structural on purpose, for the same reason as [`ShapedHeaders`]: `next_header` is set to + /// match the transport where one was drawn, but nothing keeps the extension chain's own + /// `next_header` fields consistent, because no code under test reads them. + #[allow(dead_code)] // constructed through `.with_generator()` + #[repr(transparent)] + pub struct ShapedIcmpError; + + impl ValueGenerator for ShapedIcmpError { + type Output = Headers; + + fn generate(&self, driver: &mut D) -> Option { + let outer_v4 = driver.produce::()?; + let eth_type = if outer_v4 { + CommonEthType::Ipv4 + } else { + CommonEthType::Ipv6 + }; + let eth = GenWithEthType(eth_type.into()).generate(driver)?; + + let mut vlan = ArrayVec::default(); + let vlans = driver.gen_usize(Bound::Included(&0), Bound::Included(&MAX_VLANS))?; + for _ in 0..vlans { + vlan.push(driver.produce()?); + } + + let (net, transport) = if outer_v4 { + let ip = ipv4::GenWithNextHeader(ipv4::CommonNextHeader::Icmp4.into()) + .generate(driver)?; + (Net::Ipv4(ip), Transport::Icmp4(driver.produce()?)) + } else { + let ip = ipv6::GenWithNextHeader(ipv6::CommonNextHeader::Icmp6.into()) + .generate(driver)?; + (Net::Ipv6(ip), Transport::Icmp6(driver.produce()?)) + }; + + // Echo requests and replies are not errors and quote nothing, so absence is a shape the + // API has to handle, not an edge case to skip. + let embedded_ip = if driver.produce::()? { + Some(quoted_packet(driver, outer_v4)?) + } else { + None + }; + + Some(Headers { + eth: Some(eth), + vlan, + net: Some(net), + net_ext: ext_run(driver, outer_v4)?, + transport: Some(transport), + udp_encap: None, + embedded_ip, + }) + } + } + + /// Draw the packet quoted inside an ICMP error. `outer_v4` is the family of the quoting + /// message; see [`ShapedIcmpError`] for why the quoted packet usually but not always shares it. + fn quoted_packet(driver: &mut D, outer_v4: bool) -> Option { + let mismatch = driver.gen_u8(Bound::Included(&0), Bound::Included(&7))? == 0; + let v4 = outer_v4 != mismatch; + + // Pick the transport first so the network layer's `next_header` can name it. A quote that + // stops before the transport header is the truncation the `Truncated*` types exist for. + let transport = match driver.gen_u8(Bound::Included(&0), Bound::Included(&3))? { + 0 => Some(EmbeddedTransport::Tcp(driver.produce()?)), + 1 => Some(EmbeddedTransport::Udp(driver.produce()?)), + 2 if v4 => Some(EmbeddedTransport::Icmp4(driver.produce()?)), + 2 => Some(EmbeddedTransport::Icmp6(driver.produce()?)), + _ => None, + }; + + let net = if v4 { + let next = match transport { + Some(EmbeddedTransport::Udp(_)) => ipv4::CommonNextHeader::Udp, + Some(EmbeddedTransport::Icmp4(_)) => ipv4::CommonNextHeader::Icmp4, + _ => ipv4::CommonNextHeader::Tcp, + }; + Net::Ipv4(ipv4::GenWithNextHeader(next.into()).generate(driver)?) + } else { + let next = match transport { + Some(EmbeddedTransport::Udp(_)) => ipv6::CommonNextHeader::Udp, + Some(EmbeddedTransport::Icmp6(_)) => ipv6::CommonNextHeader::Icmp6, + _ => ipv6::CommonNextHeader::Tcp, + }; + Net::Ipv6(ipv6::GenWithNextHeader(next.into()).generate(driver)?) + }; + + Some(EmbeddedHeaders::new( + Some(net), + transport, + ext_run(driver, v4)?, + None, + )) + } + #[allow(dead_code)] // rustc not able to infer we construct this through .with_generator() #[repr(transparent)] pub struct CommonHeaders; diff --git a/net/src/headers/view.rs b/net/src/headers/view.rs index ee480085ca..19fbe2bcf0 100644 --- a/net/src/headers/view.rs +++ b/net/src/headers/view.rs @@ -2750,6 +2750,9 @@ mod view_mut_properties { use crate::eth::Eth; use crate::headers::view::LookMut; use crate::headers::{Headers, Net, ShapedHeaders, Transport}; + use crate::ip_auth::Ipv4Auth; + use crate::ipv4::Ipv4; + use crate::ipv6::{DestOpts, HopByHop, Ipv6, Routing}; use crate::vlan::Vlan; /// Both walks, at every arity the oracle can express. @@ -2852,22 +2855,14 @@ mod view_mut_properties { } } - // Every arity from one to seven, which is as far as the oracle reaches. + // Every arity from one to eight, which is all of them. // - // `Matcher`'s vocabulary is `eth`, `vlan`, `net`, `transport`, `vxlan` and `embedded`. The last two - // are unreachable in a builder chain: `Vxlan: Within` and the embedded header sits under - // `Icmp4`/`Icmp6`, both *concrete* layers, and `Matcher` has no concrete-layer methods -- no - // `.udp()`, no `.tcp()`. So the longest chain it can express is `Eth`, four VLAN tags - // (`MAX_VLANS`), `Net`, `Transport`: seven. - // - // Two gaps follow, and both are about the oracle rather than the code: - // - // * the **arity-8** arm of the macro is generated and goes unchecked here, because nothing the - // oracle can say is eight elements long; - // * shapes entering the **IPv6 extension region** cannot be expressed at all, and that is the - // more interesting loss -- `ExtGapCheck` is the subtlest part of the contract, the part the - // module documentation spends most of its words on, and it has no oracle. Closing it means - // either extension-header methods on `Matcher` or a different oracle. + // `Matcher` names every layer the `Within` graph does -- `matcher_net!`, `matcher_ext!` and + // `matcher_transport!` give it `.ipv4()`, `.ipv6()`, `.hop_by_hop()`, `.dest_opts()`, + // `.routing()`, `.fragment()`, `.ipv4_auth()`, `.ipv6_auth()`, `.tcp()`, `.udp()`, `.icmp4()` and + // `.icmp6()` alongside the generic `.eth()` / `.vlan()` / `.net()` / `.transport()`. So the + // oracle reaches as far as the code does: `Eth` + four VLAN tags (`MAX_VLANS`) + `Ipv6` + + // `HopByHop` + `Transport` is eight, and the extension region is expressible. arity_agrees!(read_1, mutable_1, (&Eth,), eth); arity_agrees!(read_2, mutable_2, (&Eth, &Net), eth, net); arity_agrees!( @@ -2920,6 +2915,89 @@ mod view_mut_properties { net, transport ); + arity_agrees!( + read_8, + mutable_8, + ( + &Eth, &Vlan, &Vlan, &Vlan, &Vlan, &Ipv6, &HopByHop, &Transport + ), + eth, + vlan, + vlan, + vlan, + vlan, + ipv6, + hop_by_hop, + transport + ); + + // Shapes that enter the IPv6 extension region, where `ExtGapCheck` switches from + // skip-extensions-silently to consume-them-all. + // + // This is the subtlest part of the contract and the part the module documentation spends most of + // its words on. It is also where the two walks are least alike: on the read path both call + // `ExtGapCheck::ext_gap_ok`, so what is under test is the by-hand threading of the `ec` cursor + // through each separately generated arity -- which is the plausible bug. On the mutable path + // they call genuinely different implementations, `ext_gap_ok` against a `Headers` versus + // `ext_gap_ok_mut` against a pre-split `Fields`, so those two are checked against each other + // as well. + // + // Each shape below is deliberately exact: naming `HopByHop` and then `Transport` matches only a + // packet carrying that extension and no other, since an unconsumed extension is a miss once the + // chain has entered the region. + arity_agrees!( + read_ext_v6_one, + mutable_ext_v6_one, + (&Eth, &Ipv6, &HopByHop, &Transport), + eth, + ipv6, + hop_by_hop, + transport + ); + arity_agrees!( + read_ext_v6_two, + mutable_ext_v6_two, + (&Eth, &Ipv6, &HopByHop, &DestOpts, &Transport), + eth, + ipv6, + hop_by_hop, + dest_opts, + transport + ); + arity_agrees!( + read_ext_v6_three, + mutable_ext_v6_three, + (&Eth, &Ipv6, &HopByHop, &DestOpts, &Routing, &Transport), + eth, + ipv6, + hop_by_hop, + dest_opts, + routing, + transport + ); + // The IPv4 authentication header is the only extension that belongs on a v4 packet, and it is + // the only way to reach the strict branch without IPv6. + arity_agrees!( + read_ext_v4_auth, + mutable_ext_v4_auth, + (&Eth, &Ipv4, &Ipv4Auth, &Transport), + eth, + ipv4, + ipv4_auth, + transport + ); + // Entering the region and then *not* naming a transport: the gap check never runs, so every + // extension after the named one is simply left unvisited. Distinguishing this from the strict + // case above is the whole point of running the check at the transport step rather than the + // extension step. + arity_agrees!( + read_ext_v6_no_transport, + mutable_ext_v6_no_transport, + (&Eth, &Ipv6, &HopByHop), + eth, + ipv6, + hop_by_hop + ); /// Exercise the multi-`&mut` split itself: write through every reference and read the writes back. /// From 2fa9b45358a227f36e3a613028a6ffa04c56266b Mon Sep 17 00:00:00 2001 From: Daniel Noland Date: Sat, 8 Aug 2026 18:01:39 -0600 Subject: [PATCH 14/33] test(net): Cover the optional matchers, and find what their combinators watch `pat.rs` had 296 unreached lines and 250 of them were one thing: every `opt_*` method on all four matchers, plus `when`, `inspect` and `otherwise` on each. The optional half of the pattern-matching API had never been run. That half is not a thin wrapper over the strict half. Three families live under one naming convention. `opt_eth` cannot miss. `opt_vlan` and the optional extension methods cannot miss either, but advance their cursor only when they matched, so they skip rather than refuse -- and an optional extension still moves `Pos` into the extension region, which makes a later, unrelated transport step strict. `opt_net`, the optional transports and `opt_vxlan` are three-way: present and right is a hit, absent is a hit carrying `None`, present and wrong is a miss. Conflating the middle case with either neighbour is the mistake the design invites. Nothing could draw the middle case. `CommonHeaders` sets every layer on every path, so `net` and `transport` are always `Some` in anything it or `ShapedHeaders` produces, and the absent-layer arm of every optional method was unreachable by construction. `ThinHeaders` truncates the stack by suffix, which is the only shape a real short packet takes. `ShapedIcmpError` gained the quote too short to hold a network header, which RFC 792's header-plus- eight-bytes makes an ordinary thing rather than an exotic one. One invariant covers all three families and every layer: weakening a requirement cannot turn a match into a miss. It is worth stating because `map` and `and_then` differ by exactly that, and it is the direction a mis-wiring inverts. The guard on it counts both outcomes -- the strict form must sometimes match, and the optional form must sometimes accept what the strict form refused. An implication passes for free when its antecedent never holds, and just as quietly when the two sides never differ, which is the more likely failure and would leave the optional method's whole reason for existing untested. The property found something. `EmbeddedMatcher` and `EmbeddedMatcherMut` carry two accumulators and `done()` requires both, but `when`, `inspect` and `otherwise` all read the inner one alone. A packet whose outer chain fails -- an unconsumed VLAN tag will do it -- but whose quoted packet matches will run `inspect`, skip `otherwise`, and then return `None`. It happens to 4,974 packets in 25,000, so it is the common case rather than a corner. The doc comments say "inner accumulator" and "inner match", so this is documented rather than broken, but `otherwise` is the error-handling hook and there is a class of failure it stays silent for. `the_embedded_combinators_track_the_ inner_match_only` pins the behaviour as it stands and will fail if it is ever changed, so that becomes a decision rather than a discovery. A second finding, this one about what cannot be written down. The enum-level vocabulary is complete on the outer matchers and mostly missing on the embedded ones: `EmbeddedMatcher` has `net` but not `opt_net`, `transport` or `opt_transport`; `EmbeddedMatcherMut` has `net` and `transport` but neither optional form. All five missing methods are hand-written rather than macro-generated, which is likely how they came to be missing, since every per-variant method is present. The consequence is that a shape naming `Net` or `EmbeddedTransport` inside a quoted packet cannot be expressed as a matcher chain at all -- which is also why `embedded_view`'s differential properties have no read-side oracle for the enum forms. The table is in the source next to the tests that would use them. Verified by breaking the code twice. Making an optional extension advance its cursor unconditionally fails exactly the gap-check property, and nothing else. Making `opt_eth` refuse an absent Ethernet header fails exactly `read_opt_eth`, through the both-outcomes guard, reporting that the optional form never accepted anything the strict form refused. Coverage: `pat.rs` 63.2% -> 98.8%, 296 uncovered lines down to 10. `net/` 69.8% -> 73.1%. Of the 10 left, three are defensive `unreachable!()` and the rest are gap-fail arms on the mutable optional paths. Co-Authored-By: Claude Opus 5 (1M context) Signed-off-by: Daniel Noland --- net/src/headers/mod.rs | 57 ++++ net/src/headers/pat.rs | 735 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 792 insertions(+) diff --git a/net/src/headers/mod.rs b/net/src/headers/mod.rs index 68ed7e0176..58a05df01e 100644 --- a/net/src/headers/mod.rs +++ b/net/src/headers/mod.rs @@ -1331,6 +1331,56 @@ mod contract { Some(out) } + /// Draws a packet with layers deliberately missing, which is what the optional matchers are for. + /// + /// [`CommonHeaders`] sets every layer it knows about on every path, so `net` and `transport` are + /// always `Some` in anything it or [`ShapedHeaders`] produces. That makes the absent-layer arm of + /// every `opt_*` method in [`pat`](crate::headers::pat) -- `None => Some(a.append(None))`, the + /// arm that distinguishes "the layer is not there, which is fine" from "the layer is there and is + /// the wrong one, which is a miss" -- unreachable by any generator in the crate. The whole point + /// of an optional matcher is the packet that stops early, and nothing could draw one. + /// + /// Truncation here is by *suffix*, because that is the only shape a real short packet takes: a + /// layer sits inside the one below it, so a packet cannot carry a transport header without a + /// network header to carry it. Dropping a suffix also keeps `net_ext` consistent, since + /// extensions hang off the network layer and have to go when it does. + #[allow(dead_code)] // constructed through `.with_generator()` + #[repr(transparent)] + pub struct ThinHeaders; + + impl ValueGenerator for ThinHeaders { + type Output = Headers; + + fn generate(&self, driver: &mut D) -> Option { + let mut headers = ShapedHeaders.generate(driver)?; + // Keep the full stack a fifth of the time, so a property reading this generator still + // sees the ordinary case alongside the truncated ones. + match driver.gen_u8(Bound::Included(&0), Bound::Included(&4))? { + 0 => {} + 1 => headers.udp_encap = None, + 2 => { + headers.udp_encap = None; + headers.transport = None; + } + 3 => { + headers.udp_encap = None; + headers.transport = None; + headers.net_ext.clear(); + headers.net = None; + } + _ => { + headers.udp_encap = None; + headers.transport = None; + headers.net_ext.clear(); + headers.net = None; + headers.vlan.clear(); + headers.eth = None; + } + } + Some(headers) + } + } + /// Draw one extension header, with fuzzed contents. /// /// `pick` indexes the IPv6 extension order RFC 8200 recommends -- 0 hop-by-hop, 1 destination @@ -1517,6 +1567,13 @@ mod contract { /// Draw the packet quoted inside an ICMP error. `outer_v4` is the family of the quoting /// message; see [`ShapedIcmpError`] for why the quoted packet usually but not always shares it. fn quoted_packet(driver: &mut D, outer_v4: bool) -> Option { + // A quoting host copies as much of the offending packet as it can, and RFC 792 asked for + // only the header plus eight bytes. A quote can therefore be too short to hold even the + // network header -- which is the one case where the optional embedded matchers' absent-layer + // arm is reachable, since a quote that *has* a network layer always has a version. + if driver.gen_u8(Bound::Included(&0), Bound::Included(&7))? == 0 { + return Some(EmbeddedHeaders::new(None, None, ArrayVec::default(), None)); + } let mismatch = driver.gen_u8(Bound::Included(&0), Bound::Included(&7))? == 0; let v4 = outer_v4 != mismatch; diff --git a/net/src/headers/pat.rs b/net/src/headers/pat.rs index a5f640119d..623c5fc032 100644 --- a/net/src/headers/pat.rs +++ b/net/src/headers/pat.rs @@ -2955,3 +2955,738 @@ mod tests { }; } } + +// =========================================================================== +// Optional-layer and combinator properties +// =========================================================================== + +#[cfg(test)] +#[allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)] // fine to unwrap in tests +mod opt_properties { + use super::*; + use crate::headers::{Headers, ShapedIcmpError, ThinHeaders}; + use std::cell::Cell; + + /// `opt_X` is never stricter than `X`, for every layer and on both the read and mutable paths. + /// + /// One invariant, universally true, across three families whose internals differ sharply. + /// `opt_eth` cannot miss at all. `opt_vlan` and the optional extension methods cannot miss + /// either, but advance their cursor only when they matched, so they skip rather than refuse. + /// `opt_net`, the optional transport methods and `opt_vxlan` are three-way: present and right is + /// a hit, absent is a hit carrying `None`, present and wrong is a miss. Whatever the family, + /// weakening a requirement cannot turn a match into a miss -- and the direction is the thing a + /// mis-wiring would invert, since `and_then` and `map` differ by exactly that. + /// + /// Every one of these methods was uncovered before this: 250 of `pat.rs`'s 296 unreached lines + /// were the `opt_*` family and the combinators below. + macro_rules! opt_is_weaker { + ($read:ident, $mutable:ident, [$($pre:ident),*], $strict:ident, $opt:ident) => { + #[test] + fn $read() { + use concurrency::sync::atomic::{AtomicUsize, Ordering}; + static STRICT: AtomicUsize = AtomicUsize::new(0); + static OPT_ONLY: AtomicUsize = AtomicUsize::new(0); + bolero::check!() + .with_generator(ThinHeaders) + .for_each(|h: &Headers| { + let strict = h.pat()$(.$pre())*.$strict().done().is_some(); + let opt = h.pat()$(.$pre())*.$opt().done().is_some(); + assert!( + !strict || opt, + concat!( + "`", stringify!($strict), "` matched where `", stringify!($opt), + "` did not, so the optional form is the stricter one: {:?}" + ), + h + ); + if strict { + STRICT.fetch_add(1, Ordering::Relaxed); + } else if opt { + OPT_ONLY.fetch_add(1, Ordering::Relaxed); + } + }); + both_outcomes_seen( + concat!(stringify!($opt), " (read)"), + STRICT.load(Ordering::Relaxed), + OPT_ONLY.load(Ordering::Relaxed), + ); + } + + #[test] + fn $mutable() { + use concurrency::sync::atomic::{AtomicUsize, Ordering}; + static STRICT: AtomicUsize = AtomicUsize::new(0); + static OPT_ONLY: AtomicUsize = AtomicUsize::new(0); + bolero::check!() + .with_generator(ThinHeaders) + .for_each(|h: &Headers| { + let mut owned = h.clone(); + let strict = owned.pat_mut()$(.$pre())*.$strict().done().is_some(); + let opt = owned.pat_mut()$(.$pre())*.$opt().done().is_some(); + assert!( + !strict || opt, + concat!( + "`", stringify!($strict), "` matched where `", stringify!($opt), + "` did not on the mutable path: {:?}" + ), + h + ); + if strict { + STRICT.fetch_add(1, Ordering::Relaxed); + } else if opt { + OPT_ONLY.fetch_add(1, Ordering::Relaxed); + } + }); + both_outcomes_seen( + concat!(stringify!($opt), " (mut)"), + STRICT.load(Ordering::Relaxed), + OPT_ONLY.load(Ordering::Relaxed), + ); + } + }; + } + + /// An implication is satisfied for free when its antecedent never holds, and again when the two + /// sides never differ. + /// + /// `strict implies opt` would pass on a generator that produced nothing matching -- and it would + /// pass just as quietly on one where `opt` never accepted anything `strict` refused, which is the + /// more likely failure and the one that would make the optional method's whole reason for + /// existing untested. So both counts have to be non-zero: the requirement is sometimes met, and + /// relaxing it sometimes matters. + fn both_outcomes_seen(what: &str, strict: usize, opt_only: usize) { + println!("{what}: {strict} strict matches, {opt_only} matched only optionally"); + assert!( + strict > 0, + "{what}: the strict form never matched, so the implication held vacuously" + ); + assert!( + opt_only > 0, + "{what}: the optional form never accepted anything the strict form refused, so being \ + optional was never tested" + ); + } + + opt_is_weaker!(read_opt_eth, mut_opt_eth, [], eth, opt_eth); + opt_is_weaker!(read_opt_vlan, mut_opt_vlan, [eth], vlan, opt_vlan); + opt_is_weaker!(read_opt_net, mut_opt_net, [eth], net, opt_net); + opt_is_weaker!(read_opt_ipv4, mut_opt_ipv4, [eth], ipv4, opt_ipv4); + opt_is_weaker!(read_opt_ipv6, mut_opt_ipv6, [eth], ipv6, opt_ipv6); + opt_is_weaker!( + read_opt_hop_by_hop, + mut_opt_hop_by_hop, + [eth, ipv6], + hop_by_hop, + opt_hop_by_hop + ); + opt_is_weaker!( + read_opt_dest_opts, + mut_opt_dest_opts, + [eth, ipv6], + dest_opts, + opt_dest_opts + ); + opt_is_weaker!( + read_opt_routing, + mut_opt_routing, + [eth, ipv6], + routing, + opt_routing + ); + opt_is_weaker!( + read_opt_fragment, + mut_opt_fragment, + [eth, ipv6], + fragment, + opt_fragment + ); + opt_is_weaker!( + read_opt_ipv6_auth, + mut_opt_ipv6_auth, + [eth, ipv6], + ipv6_auth, + opt_ipv6_auth + ); + opt_is_weaker!( + read_opt_ipv4_auth, + mut_opt_ipv4_auth, + [eth, ipv4], + ipv4_auth, + opt_ipv4_auth + ); + opt_is_weaker!(read_opt_tcp, mut_opt_tcp, [eth, net], tcp, opt_tcp); + opt_is_weaker!(read_opt_udp, mut_opt_udp, [eth, net], udp, opt_udp); + opt_is_weaker!(read_opt_icmp4, mut_opt_icmp4, [eth, ipv4], icmp4, opt_icmp4); + opt_is_weaker!(read_opt_icmp6, mut_opt_icmp6, [eth, ipv6], icmp6, opt_icmp6); + opt_is_weaker!( + read_opt_transport, + mut_opt_transport, + [eth, net], + transport, + opt_transport + ); + opt_is_weaker!( + read_opt_vxlan, + mut_opt_vxlan, + [eth, net, udp], + vxlan, + opt_vxlan + ); + + /// The same invariant for the matchers over a quoted ICMP-error payload. + /// + /// Separate macro only because the chain has to pass through `.embedded()` partway, which is not + /// a layer name. The embedded families mirror the outer ones and are generated by their own set + /// of macros, so every arm needs reaching on its own -- covering `opt_hop_by_hop` says nothing + /// about the five sibling copies `embedded_ext!` emits. + macro_rules! embedded_opt_is_weaker { + ( + $read:ident, $mutable:ident, + [$($o:ident),*], [$($i:ident),*], + $strict:ident, $opt:ident + ) => { + #[test] + fn $read() { + use concurrency::sync::atomic::{AtomicUsize, Ordering}; + static STRICT: AtomicUsize = AtomicUsize::new(0); + static OPT_ONLY: AtomicUsize = AtomicUsize::new(0); + bolero::check!() + .with_generator(ShapedIcmpError) + .for_each(|h: &Headers| { + let strict = h.pat()$(.$o())*.embedded()$(.$i())*.$strict() + .done().is_some(); + let opt = h.pat()$(.$o())*.embedded()$(.$i())*.$opt() + .done().is_some(); + assert!( + !strict || opt, + concat!( + "quoted `", stringify!($strict), "` matched where `", + stringify!($opt), "` did not: {:?}" + ), + h + ); + if strict { + STRICT.fetch_add(1, Ordering::Relaxed); + } else if opt { + OPT_ONLY.fetch_add(1, Ordering::Relaxed); + } + }); + both_outcomes_seen( + concat!("quoted ", stringify!($opt), " (read)"), + STRICT.load(Ordering::Relaxed), + OPT_ONLY.load(Ordering::Relaxed), + ); + } + + #[test] + fn $mutable() { + use concurrency::sync::atomic::{AtomicUsize, Ordering}; + static STRICT: AtomicUsize = AtomicUsize::new(0); + static OPT_ONLY: AtomicUsize = AtomicUsize::new(0); + bolero::check!() + .with_generator(ShapedIcmpError) + .for_each(|h: &Headers| { + let mut owned = h.clone(); + let strict = owned.pat_mut()$(.$o())*.embedded()$(.$i())*.$strict() + .done().is_some(); + let opt = owned.pat_mut()$(.$o())*.embedded()$(.$i())*.$opt() + .done().is_some(); + assert!( + !strict || opt, + concat!( + "quoted `", stringify!($strict), "` matched where `", + stringify!($opt), "` did not on the mutable path: {:?}" + ), + h + ); + if strict { + STRICT.fetch_add(1, Ordering::Relaxed); + } else if opt { + OPT_ONLY.fetch_add(1, Ordering::Relaxed); + } + }); + both_outcomes_seen( + concat!("quoted ", stringify!($opt), " (mut)"), + STRICT.load(Ordering::Relaxed), + OPT_ONLY.load(Ordering::Relaxed), + ); + } + }; + } + + embedded_opt_is_weaker!( + read_quoted_opt_ipv4, + mut_quoted_opt_ipv4, + [eth, ipv4, icmp4], + [], + ipv4, + opt_ipv4 + ); + embedded_opt_is_weaker!( + read_quoted_opt_ipv6, + mut_quoted_opt_ipv6, + [eth, ipv6, icmp6], + [], + ipv6, + opt_ipv6 + ); + // No `opt_net` or `opt_transport` pair here, and no `transport` on the read side, because the + // embedded matchers do not have them. The enum-level vocabulary is complete on the outer + // matchers and mostly absent on the embedded ones: + // + // | | `net` | `opt_net` | `transport` | `opt_transport` | + // |----------------------|-------|-----------|-------------|-----------------| + // | `Matcher` | yes | yes | yes | yes | + // | `MatcherMut` | yes | yes | yes | yes | + // | `EmbeddedMatcher` | yes | no | no | no | + // | `EmbeddedMatcherMut` | yes | no | yes | no | + // + // Five methods missing, all hand-written rather than macro-generated, which is the likely reason + // -- the per-variant methods come from `embedded_net!` and `embedded_transport!` and are all + // present. The consequence is not cosmetic: a shape naming `Net` or `EmbeddedTransport` inside a + // quoted packet cannot be written as a matcher chain, so `embedded_view`'s differential + // properties have no read-side oracle for the enum forms either. + embedded_opt_is_weaker!( + read_quoted_opt_hop_by_hop, + mut_quoted_opt_hop_by_hop, + [eth, ipv6, icmp6], + [ipv6], + hop_by_hop, + opt_hop_by_hop + ); + embedded_opt_is_weaker!( + read_quoted_opt_dest_opts, + mut_quoted_opt_dest_opts, + [eth, ipv6, icmp6], + [ipv6], + dest_opts, + opt_dest_opts + ); + embedded_opt_is_weaker!( + read_quoted_opt_routing, + mut_quoted_opt_routing, + [eth, ipv6, icmp6], + [ipv6], + routing, + opt_routing + ); + embedded_opt_is_weaker!( + read_quoted_opt_fragment, + mut_quoted_opt_fragment, + [eth, ipv6, icmp6], + [ipv6], + fragment, + opt_fragment + ); + embedded_opt_is_weaker!( + read_quoted_opt_ipv6_auth, + mut_quoted_opt_ipv6_auth, + [eth, ipv6, icmp6], + [ipv6], + ipv6_auth, + opt_ipv6_auth + ); + embedded_opt_is_weaker!( + read_quoted_opt_ipv4_auth, + mut_quoted_opt_ipv4_auth, + [eth, ipv4, icmp4], + [ipv4], + ipv4_auth, + opt_ipv4_auth + ); + embedded_opt_is_weaker!( + read_quoted_opt_tcp, + mut_quoted_opt_tcp, + [eth, ipv4, icmp4], + [ipv4], + tcp, + opt_tcp + ); + embedded_opt_is_weaker!( + read_quoted_opt_udp, + mut_quoted_opt_udp, + [eth, ipv6, icmp6], + [ipv6], + udp, + opt_udp + ); + embedded_opt_is_weaker!( + read_quoted_opt_icmp4, + mut_quoted_opt_icmp4, + [eth, ipv4, icmp4], + [ipv4], + icmp4, + opt_icmp4 + ); + embedded_opt_is_weaker!( + read_quoted_opt_icmp6, + mut_quoted_opt_icmp6, + [eth, ipv6, icmp6], + [ipv6], + icmp6, + opt_icmp6 + ); + + // ---- Exact semantics, per family ------------------------------------------------------ + + /// The three-way families accept a matching layer or an absent one, and refuse a wrong one. + /// + /// `opt_is_weaker` above only pins the direction. This states the whole rule, and the arm it + /// exists for is the middle one: a packet that stops before the layer is a *hit* carrying `None`, + /// while a packet carrying the wrong layer is a miss. Conflating those two is the mistake this + /// family invites, and no generator could produce the first case until `ThinHeaders`. + #[test] + fn the_three_way_families_separate_an_absent_layer_from_a_wrong_one() { + bolero::check!() + .with_generator(ThinHeaders) + .for_each(|h: &Headers| { + // Reachability, spelled out once: a strict `.eth()` needs an Ethernet header, and + // the network step's gap check needs every VLAN tag consumed -- none were named. + let reached_net = h.eth().is_some() && h.vlan().is_empty(); + + assert_eq!( + h.pat().eth().opt_net().done().is_some(), + reached_net, + "the network enum is never the wrong variant, so absent or present must both \ + match: {h:?}" + ); + assert_eq!( + h.pat().eth().opt_ipv4().done().is_some(), + reached_net && !matches!(h.net(), Some(Net::Ipv6(_))), + "opt_ipv4 must accept IPv4 and absence, and refuse IPv6: {h:?}" + ); + assert_eq!( + h.pat().eth().opt_ipv6().done().is_some(), + reached_net && !matches!(h.net(), Some(Net::Ipv4(_))), + "opt_ipv6 must accept IPv6 and absence, and refuse IPv4: {h:?}" + ); + + let reached_transport = reached_net && h.net().is_some(); + assert_eq!( + h.pat().eth().net().opt_transport().done().is_some(), + reached_transport, + "the transport enum is never the wrong variant either: {h:?}" + ); + assert_eq!( + h.pat().eth().net().opt_tcp().done().is_some(), + reached_transport && matches!(h.transport(), None | Some(Transport::Tcp(_))), + "opt_tcp must accept TCP and absence, and refuse every other transport: {h:?}" + ); + + // `opt_vxlan` sits behind a concrete `.udp()`, so its own absent arm is the packet + // that carries UDP and no encapsulation -- much the commoner case in real traffic. + let reached_vxlan = + reached_transport && matches!(h.transport(), Some(Transport::Udp(_))); + assert_eq!( + h.pat().eth().net().udp().opt_vxlan().done().is_some(), + reached_vxlan, + "a UDP packet with no encapsulation must match opt_vxlan: {h:?}" + ); + }); + } + + /// The cursor families skip rather than refuse, and advance only on a hit. + /// + /// This is the sharpest observable difference between the two designs, and it is entirely about + /// the cursor. `opt_vlan` cannot miss, so a chain of them followed by a *strict* network step + /// succeeds exactly when the packet has no more tags than the chain has optional slots -- the + /// strict step's gap check is what makes the cursor's behaviour visible from outside. + #[test] + fn optional_vlans_absorb_one_tag_each_and_only_when_they_match() { + bolero::check!() + .with_generator(ThinHeaders) + .for_each(|h: &Headers| { + let reachable = h.eth().is_some() && h.net().is_some(); + let tags = h.vlan().len(); + assert_eq!( + h.pat().eth().opt_vlan().net().done().is_some(), + reachable && tags <= 1, + "one optional tag absorbed the wrong number of tags: {h:?}" + ); + assert_eq!( + h.pat().eth().opt_vlan().opt_vlan().net().done().is_some(), + reachable && tags <= 2, + "two optional tags absorbed the wrong number of tags: {h:?}" + ); + // `MAX_VLANS` is four, so four optional slots absorb any packet there can be. If + // `opt_vlan` advanced its cursor on a miss this would fail on the untagged packets. + assert_eq!( + h.pat() + .eth() + .opt_vlan() + .opt_vlan() + .opt_vlan() + .opt_vlan() + .net() + .done() + .is_some(), + reachable, + "four optional tags failed to absorb a packet with at most four: {h:?}" + ); + }); + } + + /// Naming an extension optionally still enters the region, which makes the gap check strict. + /// + /// The subtlety worth a test: `opt_hop_by_hop` cannot itself fail, so it looks harmless, but it + /// moves `Pos` to `HopByHop` and that is what `ExtGapCheck` dispatches on. The transport step + /// afterwards therefore demands every extension be consumed -- so an *optional* extension can + /// turn a later, unrelated step into a miss. + #[test] + fn an_optional_extension_still_makes_the_transport_gap_check_strict() { + bolero::check!() + .with_generator(ThinHeaders) + .for_each(|h: &Headers| { + let consumed = + usize::from(matches!(h.net_ext().first(), Some(NetExt::HopByHop(_)))); + let want = h.eth().is_some() + && h.vlan().is_empty() + && matches!(h.net(), Some(Net::Ipv6(_))) + && h.transport().is_some() + && h.net_ext().len() == consumed; + assert_eq!( + h.pat() + .eth() + .ipv6() + .opt_hop_by_hop() + .transport() + .done() + .is_some(), + want, + "an optional extension left the transport gap check lenient: {h:?}" + ); + }); + } + + // ---- Combinators ---------------------------------------------------------------------- + + /// `when`, `inspect` and `otherwise` on all four matchers. + /// + /// Three small methods repeated four times, and all twelve copies were unreached. Two of them + /// are side-effecting, which makes "did it run" the entire contract rather than a detail: + /// `inspect` must run exactly on a match and `otherwise` exactly on a miss, and neither may + /// change the outcome. `when` must be able to destroy a match and must never manufacture one -- + /// on a chain that already failed, a `true` predicate has nothing to revive. + macro_rules! combinators_fire_exactly_once_and_only_when_due { + ($name:ident, $gen:expr, $subject:expr, $fires:expr, $($chain:tt)*) => { + #[test] + fn $name() { + bolero::check!() + .with_generator($gen) + .for_each(|h: &Headers| { + // The chain is a token sequence rather than a closure because a closure + // returning a matcher borrowed from its own argument needs a higher-ranked + // lifetime, which closure inference will not produce. + let mut owned = h.clone(); + let base = owned $($chain)* .done().is_some(); + // What the combinators actually track, which is not always `base`: see + // `the_embedded_combinators_track_the_inner_match_only` below. + let fires: bool = $fires(h); + + let mut owned = h.clone(); + assert!( + owned $($chain)* .when(|_| false).done().is_none(), + concat!($subject, ": a false predicate left the match standing: {:?}"), + h + ); + let mut owned = h.clone(); + assert_eq!( + owned $($chain)* .when(|_| true).done().is_some(), + base, + concat!($subject, ": a true predicate was not a no-op: {:?}"), + h + ); + + let ran = Cell::new(false); + let mut owned = h.clone(); + let after = owned $($chain)* + .inspect(|_| ran.set(true)) + .done() + .is_some(); + assert_eq!( + ran.get(), fires, + concat!($subject, ": inspect ran on a miss or skipped a match: {:?}"), + h + ); + assert_eq!( + after, base, + concat!($subject, ": inspect changed the result: {:?}"), + h + ); + + let ran = Cell::new(false); + let mut owned = h.clone(); + let after = owned $($chain)* + .otherwise(|| ran.set(true)) + .done() + .is_some(); + assert_eq!( + ran.get(), !fires, + concat!($subject, ": otherwise ran on a match or skipped a miss: {:?}"), + h + ); + assert_eq!( + after, base, + concat!($subject, ": otherwise changed the result: {:?}"), + h + ); + }); + } + }; + } + + /// The outer matchers carry one accumulator, so their combinators fire exactly on the result. + fn whole_chain(h: &Headers) -> bool { + h.pat().eth().net().done().is_some() + } + + /// The embedded matchers carry two, and their combinators watch only the inner one. + /// + /// `.embedded().ipv4()` leaves the inner accumulator populated exactly when a quote is present + /// and its network layer is IPv4 -- a condition that says nothing about whether the outer chain + /// that led there succeeded. + fn quoted_ipv4_matched(h: &Headers) -> bool { + h.embedded_ip() + .is_some_and(|e| matches!(e.net(), Some(Net::Ipv4(_)))) + } + + combinators_fire_exactly_once_and_only_when_due!( + matcher_combinators, ThinHeaders, "Matcher", whole_chain, + .pat().eth().net() + ); + combinators_fire_exactly_once_and_only_when_due!( + matcher_mut_combinators, ThinHeaders, "MatcherMut", whole_chain, + .pat_mut().eth().net() + ); + combinators_fire_exactly_once_and_only_when_due!( + embedded_matcher_combinators, ShapedIcmpError, "EmbeddedMatcher", quoted_ipv4_matched, + .pat().eth().ipv4().icmp4().embedded().ipv4() + ); + combinators_fire_exactly_once_and_only_when_due!( + embedded_matcher_mut_combinators, ShapedIcmpError, "EmbeddedMatcherMut", + quoted_ipv4_matched, + .pat_mut().eth().ipv4().icmp4().embedded().ipv4() + ); + + /// `otherwise` does not run for every chain that returns `None`, and `inspect` runs for some. + /// + /// Found by the property above, which originally assumed the combinators tracked `.done()`. + /// They do not, and the difference is only observable on the embedded matchers, which carry two + /// accumulators: `done()` requires *both* to be populated, while `when`, `inspect` and + /// `otherwise` all read the inner one alone. + /// + /// So a packet whose outer chain fails -- an unconsumed VLAN tag will do it -- but whose quoted + /// packet matches will run `inspect`, skip `otherwise`, and then return `None`. The doc comments + /// are accurate ("apply a predicate to the inner accumulator", "run a closure if the inner match + /// has already failed"), so this is documented rather than broken. It still seems worth a + /// decision: `otherwise` is the error-handling hook, and there is a whole class of failure it + /// stays silent for. + /// + /// This test pins the behaviour as it stands. If the combinators are ever changed to track + /// `done()` it will fail, which is the point -- that should be a decision rather than a + /// discovery. + #[test] + fn the_embedded_combinators_track_the_inner_match_only() { + use concurrency::sync::atomic::{AtomicUsize, Ordering}; + static DIVERGED: AtomicUsize = AtomicUsize::new(0); + bolero::check!() + .with_generator(ShapedIcmpError) + .for_each(|h: &Headers| { + let whole = h + .pat() + .eth() + .ipv4() + .icmp4() + .embedded() + .ipv4() + .done() + .is_some(); + let inner = quoted_ipv4_matched(h); + if whole == inner { + return; + } + DIVERGED.fetch_add(1, Ordering::Relaxed); + // Divergence is one-directional: the inner match can succeed where the whole chain + // fails, never the reverse, since `done()` needs the inner accumulator too. + assert!( + inner && !whole, + "the whole chain matched while the inner one did not, which `done` forbids: \ + {h:?}" + ); + let ran = Cell::new(false); + h.pat() + .eth() + .ipv4() + .icmp4() + .embedded() + .ipv4() + .otherwise(|| ran.set(true)) + .done(); + assert!( + !ran.get(), + "`otherwise` ran on a chain whose inner match succeeded; the divergence \ + documented here has been fixed, so this test should be deleted: {h:?}" + ); + }); + let diverged = DIVERGED.load(Ordering::Relaxed); + println!("outer failed while the quote matched: {diverged} packets"); + assert!( + diverged > 0, + "the two never diverged, so this test proved nothing about which one the combinators \ + follow" + ); + } + + /// The quoted transport enum refuses an unconsumed extension on the mutable path too. + /// + /// `EmbeddedMatcherMut::transport` is the one enum-level embedded method that exists, and its + /// gap-check rejection had nothing reaching it. Stated directly rather than differentially, + /// because the read-side counterpart it would be compared against is one of the five missing + /// methods listed above. + #[test] + fn the_quoted_transport_enum_refuses_an_unconsumed_extension() { + bolero::check!() + .with_generator(crate::headers::ShapedQuote { ext: 0, v4: false }) + .for_each(|h: &Headers| { + let mut owned = h.clone(); + let quoted = owned + .embedded_ip_mut() + .unwrap_or_else(|| unreachable!("ShapedQuote always attaches a quote")); + let first = quoted + .net_ext + .first() + .unwrap_or_else(|| unreachable!("ShapedQuote always places one extension")) + .clone(); + quoted.net_ext.push(first); + + let mut two = owned.clone(); + assert!( + two.pat_mut() + .eth() + .ipv6() + .icmp6() + .embedded() + .ipv6() + .hop_by_hop() + .transport() + .done() + .is_none(), + "one extension named of two, yet the transport enum matched: {h:?}" + ); + let mut one = h.clone(); + assert!( + one.pat_mut() + .eth() + .ipv6() + .icmp6() + .embedded() + .ipv6() + .hop_by_hop() + .transport() + .done() + .is_some(), + "the sole extension was named and consumed, yet the chain missed: {h:?}" + ); + }); + } +} From 05e4c5b8a90d0bc24adea63d13b7eecb2b101249 Mon Sep 17 00:00:00 2001 From: Daniel Noland Date: Sat, 8 Aug 2026 19:32:46 -0600 Subject: [PATCH 15/33] test(net): Make the shape follow the chain, and reach every arity's look `view.rs` sat at 53.8% with 224 production lines unreached, and almost all of them were `look` and `look_mut`: the bodies at arities five through eight had never executed, nor had `look_mut` at one and two. The properties compared `matches` against the matcher chain -- the decision -- and stopped there. The delivery half was checked at arities three and four only, in the split test, because comparing two tuples means destructuring them into a fixed number of bindings and that has to be written out per arity. `Addrs` removes that constraint. Reducing a tuple of references to an array of addresses is arity-generic at the call site even though the impls are not, so the delivery check now runs wherever the decision check does. Addresses rather than values, because two VLAN tags can hold equal bytes without being the same tag, and picking the wrong one out of four is exactly the cursor bug this is looking for: `matches`, `look` and `look_mut` are generated separately at each arity, threading `vc` and `ec` through by hand every time, so a shape naming four tags has four chances to be off by one and the decision check cannot see any of them. Breaking arity six's `look` to read one tag early fails `read_6` and nothing else -- not even `read_ext_v6_three`, the other arity-six shape, whose third layer is an extension header and reads the other cursor. Two more gaps, both of the kind that hides behind a passing suite: `Vxlan` had no `ViewStep` coverage at all, not because the generators could not draw a VXLAN packet -- `CommonHeaders` has been drawing them all along -- but because no shape ever named the layer. It is the one step that runs no gap check and the one layer outside the linear stack, so nothing about it follows from the other arities. Two chains now name it; making the step refuse fails exactly those four tests and nothing else. The branch where the *first* step refuses is generated once per arity and was unreachable at every one of them, since `CommonHeaders` sets `eth` on all six of its paths. `ThinHeaders` can drop it but truncates four packets in five, which would cost the deep shapes most of their hit rate; `SometimesHeadless` drops it one packet in eight, enough for the branch and cheap enough that arity seven still matches six thousand times in thirty-five. The shapes are now derived rather than written. A shape and the chain matching it are one statement said twice, and every instantiation said it twice by hand: `(&Eth, &Ipv6, &HopByHop, &Transport)` next to `.eth().ipv6().hop_by_hop() .transport()`. A pair that disagrees compiles and passes and silently tests something else. `layer_ty!` holds the correspondence once, `shape_of!` builds the tuple from the chain, and the instantiations shrank to the chain alone. That is as far as generating tests from the macro tables usefully goes here. Enumerating the `Within` graph gives 4,755 legal chains up to arity eight -- 2,614 at arity eight alone -- so one property per chain is not a suite anyone would run, and the exhaustive version would have to be a shallow sweep over a fixed corpus rather than a fuzz run. It would also be worth less than it looks: an oracle enumerated from the same table the implementation is generated from cannot notice a wrong table entry. What survives the objection is the differential, since `matches` and the matcher chain are independent implementations and the table only chooses which chains to test, not the verdict. Coverage: `view.rs` 53.8% -> 98.4%, 224 uncovered lines to eight. The eight are the `unreachable_unchecked` arm of each arity, which must stay uncovered -- reaching one is the undefined behaviour the whole `HeadersView` invariant exists to prevent. `net` overall 74.2% -> 76.8%. Also fixes two lints in the previous commit that only appear under `--features bolero,test_buffer,builder` rather than `--all-features`. Co-Authored-By: Claude Opus 5 (1M context) Signed-off-by: Daniel Noland --- net/src/headers/mod.rs | 30 +++++ net/src/headers/pat.rs | 6 +- net/src/headers/view.rs | 263 +++++++++++++++++++++++++++++++++------- 3 files changed, 253 insertions(+), 46 deletions(-) diff --git a/net/src/headers/mod.rs b/net/src/headers/mod.rs index 58a05df01e..76a8712285 100644 --- a/net/src/headers/mod.rs +++ b/net/src/headers/mod.rs @@ -1381,6 +1381,36 @@ mod contract { } } + /// [`ShapedHeaders`] with the Ethernet header taken away one packet in eight. + /// + /// Every `Shape` starts at `Eth`, and the branch where that first step refuses is generated + /// separately at each of the eight arities -- eight copies of `return false`, none of which any + /// generator could reach, because `CommonHeaders` sets `eth` on all six of its paths. + /// + /// [`ThinHeaders`] reaches them, but it truncates four packets in five, and the deep shapes pay + /// for that: arity seven already needs exactly four VLAN tags, so another factor of five would + /// leave it matching too rarely to find anything. One in eight covers the branch at every arity + /// while leaving seven eighths of the draw doing the work it was doing before. + #[allow(dead_code)] // constructed through `.with_generator()` + #[repr(transparent)] + pub struct SometimesHeadless; + + impl ValueGenerator for SometimesHeadless { + type Output = Headers; + + fn generate(&self, driver: &mut D) -> Option { + let mut headers = ShapedHeaders.generate(driver)?; + if driver.gen_u8(Bound::Included(&0), Bound::Included(&7))? == 0 { + // The tags go with it. A VLAN tag is carried by the Ethernet header, so a packet + // holding one without the other is not a short packet, it is an impossible one, and + // the properties reading this generator are about short packets. + headers.vlan.clear(); + headers.eth = None; + } + Some(headers) + } + } + /// Draw one extension header, with fuzzed contents. /// /// `pick` indexes the IPv6 extension order RFC 8200 recommends -- 0 hop-by-hop, 1 destination diff --git a/net/src/headers/pat.rs b/net/src/headers/pat.rs index 623c5fc032..d38a1f0436 100644 --- a/net/src/headers/pat.rs +++ b/net/src/headers/pat.rs @@ -3469,6 +3469,9 @@ mod opt_properties { /// on a chain that already failed, a `true` predicate has nothing to revive. macro_rules! combinators_fire_exactly_once_and_only_when_due { ($name:ident, $gen:expr, $subject:expr, $fires:expr, $($chain:tt)*) => { + // The `mut` binding is what `pat_mut()` needs and what `pat()` does not, and the same + // macro serves both, so half the instantiations declare a `mut` they never use. + #[allow(unused_mut)] #[test] fn $name() { bolero::check!() @@ -3614,7 +3617,8 @@ mod opt_properties { {h:?}" ); let ran = Cell::new(false); - h.pat() + let _ = h + .pat() .eth() .ipv4() .icmp4() diff --git a/net/src/headers/view.rs b/net/src/headers/view.rs index 19fbe2bcf0..9f14ee8135 100644 --- a/net/src/headers/view.rs +++ b/net/src/headers/view.rs @@ -2748,12 +2748,123 @@ mod view_properties { #[cfg(test)] mod view_mut_properties { use crate::eth::Eth; - use crate::headers::view::LookMut; - use crate::headers::{Headers, Net, ShapedHeaders, Transport}; - use crate::ip_auth::Ipv4Auth; - use crate::ipv4::Ipv4; - use crate::ipv6::{DestOpts, HopByHop, Ipv6, Routing}; - use crate::vlan::Vlan; + use crate::headers::view::{Look, LookMut}; + use crate::headers::{Headers, Net, ShapedHeaders, SometimesHeadless, Transport}; + + /// The type a `Matcher` method name selects. + /// + /// A shape and the chain that matches it are the same statement written twice -- + /// `(&Eth, &Ipv6, &HopByHop, &Transport)` and `.eth().ipv6().hop_by_hop().transport()` -- and + /// writing both by hand at every instantiation is a standing invitation to write two different + /// statements. The pair that disagrees still compiles and still passes; it just quietly tests + /// something other than what it says. So the chain is the input and the shape is derived. + /// + /// The table is the only place the correspondence lives, and it is the whole of it: every entry + /// below names a method the `matcher_net!`, `matcher_ext!` and `matcher_transport!` invocations + /// in [`pat`](super::pat) generate, plus the four written out by hand. + macro_rules! layer_ty { + (eth) => { + crate::eth::Eth + }; + (vlan) => { + crate::vlan::Vlan + }; + (net) => { + crate::headers::Net + }; + (ipv4) => { + crate::ipv4::Ipv4 + }; + (ipv6) => { + crate::ipv6::Ipv6 + }; + (hop_by_hop) => { + crate::ipv6::HopByHop + }; + (dest_opts) => { + crate::ipv6::DestOpts + }; + (routing) => { + crate::ipv6::Routing + }; + (fragment) => { + crate::ipv6::Fragment + }; + (ipv4_auth) => { + crate::ip_auth::Ipv4Auth + }; + (ipv6_auth) => { + crate::ip_auth::Ipv6Auth + }; + (transport) => { + crate::headers::Transport + }; + (tcp) => { + crate::tcp::Tcp + }; + (udp) => { + crate::udp::Udp + }; + (icmp4) => { + crate::icmp4::Icmp4 + }; + (icmp6) => { + crate::icmp6::Icmp6 + }; + (vxlan) => { + crate::vxlan::Vxlan + }; + } + + /// The `Shape` a chain of matcher method names denotes. + macro_rules! shape_of { + ($($layer:ident),+ $(,)?) => { ($(&'static layer_ty!($layer),)+) }; + } + + /// The address of each layer in a tuple of references, without naming the arity. + /// + /// Agreeing that a shape matches is the soundness question; handing back the *same* layers is the + /// correctness one, and it was previously checked only at arities three and four, because that is + /// where a tuple can be destructured by hand into a fixed number of bindings. Reducing the tuple + /// to its addresses removes the need to name the arity at all, so the check applies wherever the + /// agreement check does. + /// + /// Addresses rather than values: two layers can hold equal bytes without being the same layer, and + /// picking the wrong VLAN tag out of four identical ones is exactly the cursor bug this is looking + /// for. + trait Addrs { + /// One address per element. + type Out: PartialEq + core::fmt::Debug; + /// Where each element of this tuple lives. + fn addrs(&self) -> Self::Out; + } + + macro_rules! impl_addrs { + ($n:literal; $($T:ident $idx:tt),+) => { + impl<'a, $($T),+> Addrs for ($(&'a $T,)+) { + type Out = [usize; $n]; + fn addrs(&self) -> [usize; $n] { + [$(core::ptr::from_ref::<$T>(self.$idx) as usize),+] + } + } + + impl<'a, $($T),+> Addrs for ($(&'a mut $T,)+) { + type Out = [usize; $n]; + fn addrs(&self) -> [usize; $n] { + [$(core::ptr::from_ref::<$T>(&*self.$idx) as usize),+] + } + } + }; + } + + impl_addrs!(1; A 0); + impl_addrs!(2; A 0, B 1); + impl_addrs!(3; A 0, B 1, C 2); + impl_addrs!(4; A 0, B 1, C 2, D 3); + impl_addrs!(5; A 0, B 1, C 2, D 3, E 4); + impl_addrs!(6; A 0, B 1, C 2, D 3, E 4, F 5); + impl_addrs!(7; A 0, B 1, C 2, D 3, E 4, F 5, G 6); + impl_addrs!(8; A 0, B 1, C 2, D 3, E 4, F 5, G 6, H 7); /// Both walks, at every arity the oracle can express. /// @@ -2765,38 +2876,60 @@ mod view_mut_properties { /// a macro, with the VLAN and extension cursors threaded through by hand every time. Testing two /// arities tested two of eight copies. /// - /// Pointer identity -- did the two implementations pick the *same* layer, not merely agree that one - /// exists -- is checked separately, at arities 3 and 4, where the tuple can be destructured - /// concretely. That is a correctness question rather than a soundness one, so it is checked deeply - /// at representative arities rather than shallowly at all of them. + /// Each property asks two things of a shape, and the second is the reason `look` and `look_mut` + /// appear here rather than only in the split test: + /// + /// * **the decision**, `matches` against the matcher chain. This is what soundness turns on: if + /// `matches` says yes where the walk that must deliver says no, `look`/`look_mut` reach + /// `unreachable_unchecked`. Disagreement here is undefined behaviour, not a wrong answer. + /// * **the delivery**, `look`/`look_mut` against the same chain's tuple, compared by address. + /// Agreeing that a shape is present is not the same as picking the same layers out of it, and + /// the cursor arithmetic that decides *which* VLAN tag or *which* extension header comes back + /// is written out by hand once per arity. A shape naming four tags has four chances to be off + /// by one and no way for the decision check to notice. + /// + /// Delivery was previously checked at arities three and four only, because a tuple has to be + /// destructured into a fixed number of bindings to be compared -- which is why [`Addrs`] exists. macro_rules! arity_agrees { - ($read:ident, $mutable:ident, $shape:ty, $($layer:ident),+) => { + ($read:ident, $mutable:ident, $gen:expr, $($layer:ident),+) => { #[test] fn $read() { + type Shape = shape_of!($($layer),+); use concurrency::sync::atomic::{AtomicUsize, Ordering}; static SEEN: AtomicUsize = AtomicUsize::new(0); static HIT: AtomicUsize = AtomicUsize::new(0); bolero::check!() - .with_generator(ShapedHeaders) + .with_generator($gen) .for_each(|h: &Headers| { SEEN.fetch_add(1, Ordering::Relaxed); - let licensed = h.as_view::<$shape>().is_some(); + let licensed = h.as_view::().is_some(); if licensed { HIT.fetch_add(1, Ordering::Relaxed); } - let deliverable = h.pat()$(.$layer())+.done().is_some(); + let chain = h.pat()$(.$layer())+.done(); assert_eq!( - licensed, deliverable, + licensed, chain.is_some(), concat!( "`matches` and the read walk disagree for ", - stringify!($shape), + stringify!(($($layer),+)), ", so `look` would reach `unreachable_unchecked`: {:?}" ), h ); + if let (Some(view), Some(chain)) = (h.as_view::(), chain) { + assert_eq!( + view.look().addrs(), chain.addrs(), + concat!( + "`look` and the read walk agreed that ", + stringify!(($($layer),+)), + " is present and then handed back different layers: {:?}" + ), + h + ); + } }); agreement_is_not_vacuous( - stringify!($shape), + stringify!(($($layer),+)), SEEN.load(Ordering::Relaxed), HIT.load(Ordering::Relaxed), ); @@ -2804,31 +2937,46 @@ mod view_mut_properties { #[test] fn $mutable() { + type Shape = shape_of!($($layer),+); use concurrency::sync::atomic::{AtomicUsize, Ordering}; static SEEN: AtomicUsize = AtomicUsize::new(0); static HIT: AtomicUsize = AtomicUsize::new(0); bolero::check!() - .with_generator(ShapedHeaders) + .with_generator($gen) .for_each(|h: &Headers| { let mut owned = h.clone(); SEEN.fetch_add(1, Ordering::Relaxed); - let licensed = owned.as_view_mut::<$shape>().is_some(); + let licensed = owned.as_view_mut::().is_some(); if licensed { HIT.fetch_add(1, Ordering::Relaxed); } - let deliverable = owned.pat_mut()$(.$layer())+.done().is_some(); + // The two walks each want the whole of `owned` mutably, so they take turns + // and hand back addresses rather than references. Nothing is written + // between the two, so the addresses stay comparable. + let chain = owned.pat_mut()$(.$layer())+.done().map(|t| t.addrs()); assert_eq!( - licensed, deliverable, + licensed, chain.is_some(), concat!( "`matches` and the mutable walk disagree for ", - stringify!($shape), + stringify!(($($layer),+)), ", so `look_mut` would reach `unreachable_unchecked`: {:?}" ), h ); + if let Some(view) = owned.as_view_mut::() { + assert_eq!( + Some(view.look_mut().addrs()), chain, + concat!( + "`look_mut` and the mutable walk agreed that ", + stringify!(($($layer),+)), + " is present and then handed back different layers: {:?}" + ), + h + ); + } }); agreement_is_not_vacuous( - stringify!($shape), + stringify!(($($layer),+)), SEEN.load(Ordering::Relaxed), HIT.load(Ordering::Relaxed), ); @@ -2863,20 +3011,17 @@ mod view_mut_properties { // `.icmp6()` alongside the generic `.eth()` / `.vlan()` / `.net()` / `.transport()`. So the // oracle reaches as far as the code does: `Eth` + four VLAN tags (`MAX_VLANS`) + `Ipv6` + // `HopByHop` + `Transport` is eight, and the extension region is expressible. - arity_agrees!(read_1, mutable_1, (&Eth,), eth); - arity_agrees!(read_2, mutable_2, (&Eth, &Net), eth, net); - arity_agrees!( - read_3, - mutable_3, - (&Eth, &Net, &Transport), - eth, - net, - transport - ); + // + // `SometimesHeadless` rather than `ShapedHeaders`: the first step of every one of these is `Eth`, + // and the branch where that step refuses is generated once per arity. Nothing that always sets + // `eth` can reach any of them. + arity_agrees!(read_1, mutable_1, SometimesHeadless, eth); + arity_agrees!(read_2, mutable_2, SometimesHeadless, eth, net); + arity_agrees!(read_3, mutable_3, SometimesHeadless, eth, net, transport); arity_agrees!( read_4, mutable_4, - (&Eth, &Vlan, &Net, &Transport), + SometimesHeadless, eth, vlan, net, @@ -2885,7 +3030,7 @@ mod view_mut_properties { arity_agrees!( read_5, mutable_5, - (&Eth, &Vlan, &Vlan, &Net, &Transport), + SometimesHeadless, eth, vlan, vlan, @@ -2895,7 +3040,7 @@ mod view_mut_properties { arity_agrees!( read_6, mutable_6, - (&Eth, &Vlan, &Vlan, &Vlan, &Net, &Transport), + SometimesHeadless, eth, vlan, vlan, @@ -2906,7 +3051,7 @@ mod view_mut_properties { arity_agrees!( read_7, mutable_7, - (&Eth, &Vlan, &Vlan, &Vlan, &Vlan, &Net, &Transport), + SometimesHeadless, eth, vlan, vlan, @@ -2918,9 +3063,7 @@ mod view_mut_properties { arity_agrees!( read_8, mutable_8, - ( - &Eth, &Vlan, &Vlan, &Vlan, &Vlan, &Ipv6, &HopByHop, &Transport - ), + SometimesHeadless, eth, vlan, vlan, @@ -2948,7 +3091,7 @@ mod view_mut_properties { arity_agrees!( read_ext_v6_one, mutable_ext_v6_one, - (&Eth, &Ipv6, &HopByHop, &Transport), + SometimesHeadless, eth, ipv6, hop_by_hop, @@ -2957,7 +3100,7 @@ mod view_mut_properties { arity_agrees!( read_ext_v6_two, mutable_ext_v6_two, - (&Eth, &Ipv6, &HopByHop, &DestOpts, &Transport), + SometimesHeadless, eth, ipv6, hop_by_hop, @@ -2967,7 +3110,7 @@ mod view_mut_properties { arity_agrees!( read_ext_v6_three, mutable_ext_v6_three, - (&Eth, &Ipv6, &HopByHop, &DestOpts, &Routing, &Transport), + SometimesHeadless, eth, ipv6, hop_by_hop, @@ -2980,7 +3123,7 @@ mod view_mut_properties { arity_agrees!( read_ext_v4_auth, mutable_ext_v4_auth, - (&Eth, &Ipv4, &Ipv4Auth, &Transport), + SometimesHeadless, eth, ipv4, ipv4_auth, @@ -2993,12 +3136,42 @@ mod view_mut_properties { arity_agrees!( read_ext_v6_no_transport, mutable_ext_v6_no_transport, - (&Eth, &Ipv6, &HopByHop), + SometimesHeadless, eth, ipv6, hop_by_hop ); + // The UDP encapsulation layer, which no shape named until now. + // + // `Vxlan` is the one `ViewStep` that runs no gap check at all -- it reads `udp_encap` and ignores + // both cursors -- and the one layer that is not part of the linear header stack, so nothing about + // it follows from the arities above. `CommonHeaders` has been drawing VXLAN packets the whole + // time; the gap was that no property ever asked for one. + // + // `ShapedHeaders` rather than `SometimesHeadless` here, because this shape is already narrow: + // `Udp` is one of three next headers, the encapsulation is a coin flip on top of that, and the + // net step demands no VLAN tags. Removing the Ethernet header as well would spend the hit rate + // on a branch the eight arities above already cover. + arity_agrees!( + read_vxlan_v4, + mutable_vxlan_v4, + ShapedHeaders, + eth, + ipv4, + udp, + vxlan + ); + arity_agrees!( + read_vxlan_net, + mutable_vxlan_net, + ShapedHeaders, + eth, + net, + udp, + vxlan + ); + /// Exercise the multi-`&mut` split itself: write through every reference and read the writes back. /// /// The assertions are almost beside the point. What matters is that the references are *created and From 22a947e3633ac0fcf785143548465a67278b51c0 Mon Sep 17 00:00:00 2001 From: Daniel Noland Date: Sat, 8 Aug 2026 20:37:05 -0600 Subject: [PATCH 16/33] test(net): Stack the extension headers the builder always could `within.rs` sat at 32.7%, and the shape of the gap was unusually clean: thirty-two of the thirty-three `conform` bodies that do any work had never run, and the one that had was `DestOpts` inside `HopByHop`. Every unreached one was an IPv6 extension header transition, plus `Vlan` inside `Vlan`. The cause is a seam rather than an oversight. `conform` runs only from `HeaderStack::stack`, and the builder has had `.hop_by_hop()`, `.dest_opts()`, `.routing()`, `.fragment()`, `.ipv4_auth()` and `.ipv6_auth()` all along -- no test ever called one. The generators reach the extension region constantly, but they assemble `Headers` field by field and never go near the builder, so they never conform anything. Two ways to build a packet, and the fuzzing all went down the one that skips this trait. Seventeen chains cover the thirty-two transitions between them. Three extension headers is the ceiling, `MAX_NET_EXTENSIONS`, so the deeper corners of the graph need several short chains rather than one long one. The oracle is deparse-then-parse. Reading back the field `conform` just wrote would check the implementation against itself; the parser decides what follows an IPv6 header by reading that same field, so a `conform` naming the wrong protocol produces bytes that parse as a different packet, or as no packet at all. Naming TCP where the fragment header goes fails exactly the two chains carrying `routing -> fragment`. Each layer's protocol field is scrambled to a fuzzed byte before the next layer is stacked, so `conform` always overwrites a wrong value instead of filling in a blank one. That turns out to be load-bearing rather than cautious, and `Ipv4` inside `Eth` is the proof: `Blank for Eth` already produces `EthType::IPV4`, so on a blank header the conform setting `EthType::IPV4` has nothing to do. Deleting its body passes all seventeen chains unscrambled and fails two of them scrambled. `Vlan` has the same blank and the same exposure. Coverage: `within.rs` 32.7% -> 88.9%. `net` overall 76.8% -> 79.9%. The nineteen lines left are the no-op bodies -- the enum-level impls, the `EmbeddedStart` impls, and everything `impl_truncated_within!` generates. Those are unreachable through the builder, and the compiler says so twice over: `stack::` fails on both `Net: Blank` and `Headers: Install`, either of which would be enough on its own. They exist to give the pattern matcher its `Within` edges, which need the trait but not the method. Documented rather than deleted; whether to keep nineteen uncallable bodies belongs to whoever owns the trait. Co-Authored-By: Claude Opus 5 (1M context) Signed-off-by: Daniel Noland --- net/src/headers/mod.rs | 2 +- net/src/headers/within.rs | 218 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 219 insertions(+), 1 deletion(-) diff --git a/net/src/headers/mod.rs b/net/src/headers/mod.rs index 76a8712285..6514cfec17 100644 --- a/net/src/headers/mod.rs +++ b/net/src/headers/mod.rs @@ -1781,7 +1781,7 @@ mod test { use crate::tcp::{TcpChecksum, TcpChecksumPayload, TcpPort}; use crate::udp::{UdpChecksum, UdpChecksumPayload, UdpPort}; - fn parse_back_test(headers: &Headers) { + pub(crate) fn parse_back_test(headers: &Headers) { let mut buffer = [0_u8; 1024]; let bytes_written = match headers.deparse(&mut buffer[..headers.size().into_non_zero_usize().get()]) { diff --git a/net/src/headers/within.rs b/net/src/headers/within.rs index 2fc8646360..7e72cf2939 100644 --- a/net/src/headers/within.rs +++ b/net/src/headers/within.rs @@ -554,3 +554,221 @@ impl Within for Tcp { impl Within for Udp { fn conform(_parent: &mut Net) {} } + +/// What [`Within::conform`] is for, checked against the parser. +/// +/// `conform` writes the parent's protocol field so it names the child: `EthType::IPV6` on an +/// Ethernet header carrying IPv6, `NextHeader::ROUTING` on an IPv6 header carrying a routing +/// extension. It is the only reason this trait has a method at all -- the ordering half of the +/// contract is enforced by the compiler, by the presence or absence of an impl, and needs no test. +/// +/// Thirty-two of these bodies had never run. Every one of them was an IPv6 extension header +/// transition or `Vlan` inside `Vlan`, which is to say: the whole extension region, plus the second +/// tag of a double-tagged frame. The reason is that `conform` is reached only through +/// [`HeaderStack::stack`](crate::headers::builder::HeaderStack::stack), and while the builder has had +/// `.hop_by_hop()`, `.dest_opts()`, `.routing()`, `.fragment()`, `.ipv4_auth()` and `.ipv6_auth()` +/// all along, no test ever called one. The generators reach the extension region constantly, but +/// they assemble [`Headers`](crate::headers::Headers) field by field and never go through the +/// builder, so they never conform anything. +/// +/// # The oracle +/// +/// A test that builds a packet and then reads back the field `conform` just wrote would be checking +/// the implementation against itself. So the check is deparse-then-parse: the parser decides what +/// follows an IPv6 header by reading its next-header field, which is the field `conform` sets, so a +/// `conform` that names the wrong protocol produces bytes the parser reads as a different packet -- +/// or as no valid packet at all. +/// +/// # Scrambling +/// +/// Each layer's next-header field is set to a fuzzed byte *before* the next layer is stacked, so +/// `conform` is always overwriting a wrong value rather than filling in a blank one. +/// +/// That is not a precaution, it is load-bearing, and the case that proves it is `Ipv4` inside `Eth`. +/// [`Blank`] for `Eth` produces `EthType::IPV4`, so on a blank Ethernet header the conform that sets +/// `EthType::IPV4` has nothing to do: delete its body and every packet still round-trips. Scrambled, +/// the same deletion fails two chains. `Vlan` has the same blank and the same exposure. +/// +/// # What is left, and why it stays uncovered +/// +/// Nineteen no-op `conform` bodies remain unrun: the enum-level impls, the `EmbeddedStart` impls and +/// everything `impl_truncated_within!` generates. They are not merely untested, they are unreachable +/// through the builder, and the compiler says so twice -- writing +/// `HeaderStack::new().eth(..).stack::(..)` fails with both `Net: Blank is not satisfied` and +/// `Headers: Install is not satisfied`. Either bound alone would be enough. +/// +/// They exist to give the pattern matcher its `Within` edges, which need the trait but not the +/// method. `conform` is a public trait method, so they are callable in principle by anyone holding +/// the parent; nothing in the tree does. Whether nineteen uncallable bodies are worth keeping is a +/// question for whoever owns the trait, not something a test can settle. +/// +/// [`Blank`]: crate::headers::builder::Blank +#[cfg(test)] +mod conform_properties { + use crate::eth::Eth; + use crate::eth::ethtype::EthType; + use crate::headers::builder::HeaderStack; + use crate::headers::test::parse_back_test; + use crate::icmp4::Icmp4; + use crate::icmp6::Icmp6; + use crate::ip::NextHeader; + use crate::ip_auth::{Ipv4Auth, Ipv6Auth}; + use crate::ipv4::Ipv4; + use crate::ipv6::{DestOpts, Fragment, HopByHop, Ipv6, Routing}; + use crate::tcp::Tcp; + use crate::udp::Udp; + use crate::vlan::Vlan; + + /// Put a wrong protocol number in the field `conform` is responsible for. + /// + /// Implemented for every layer the builder can stack. The transport types are the leaves of + /// every chain -- nothing is ever stacked on top of one, so nothing ever conforms one -- and + /// their impls are deliberately empty rather than absent, so that the chain macro does not have + /// to know which layers are interior. + trait Scramble { + /// Overwrite the protocol field with something derived from `seed`. + fn scramble(&mut self, seed: u8); + } + + macro_rules! scramble_next_header { + ($($T:ty),+ $(,)?) => {$( + impl Scramble for $T { + fn scramble(&mut self, seed: u8) { + self.set_next_header(NextHeader::new(seed)); + } + } + )+}; + } + + macro_rules! scramble_leaf { + ($($T:ty),+ $(,)?) => {$( + impl Scramble for $T { + fn scramble(&mut self, _seed: u8) {} + } + )+}; + } + + scramble_next_header!( + Ipv4, Ipv6, HopByHop, DestOpts, Routing, Fragment, Ipv4Auth, Ipv6Auth + ); + scramble_leaf!(Tcp, Udp, Icmp4, Icmp6); + + impl Scramble for Eth { + fn scramble(&mut self, seed: u8) { + self.set_ether_type(EthType::new(u16::from(seed))); + } + } + + impl Scramble for Vlan { + fn scramble(&mut self, seed: u8) { + self.set_inner_ethtype(EthType::new(u16::from(seed))); + } + } + + /// Build the named chain through the builder, scrambling as it goes, and round-trip it. + /// + /// The chain is a list of `HeaderStack` method names, and the closures are written here rather + /// than at the call site: a closure written at the call site could not name the `seed` this + /// macro binds, macro hygiene being what it is, and every call site would then have to repeat + /// the same closure once per layer. + macro_rules! conform_chain { + ($name:ident, $($layer:ident),+ $(,)?) => { + #[test] + fn $name() { + bolero::check!().with_type().for_each(|seed: &u8| { + let seed = *seed; + let built = HeaderStack::new() + $(.$layer(|l| l.scramble(seed)))+ + .build_headers(); + let headers = built.unwrap_or_else(|e| { + unreachable!("a blank {} chain does not overflow: {e:?}", stringify!($name)) + }); + parse_back_test(&headers); + }); + } + }; + } + + // Seventeen chains, chosen to cover all thirty-two unreached transitions between them. Three + // extension headers is the ceiling -- `MAX_NET_EXTENSIONS` -- so the deeper regions of the graph + // have to be reached by several chains rather than one long one. + conform_chain!( + double_tag_then_three_extensions, + eth, + vlan, + vlan, + ipv6, + dest_opts, + routing, + fragment, + tcp + ); + conform_chain!( + hop_by_hop_routing_dest_opts, + eth, + ipv6, + hop_by_hop, + routing, + dest_opts, + udp + ); + conform_chain!( + routing_fragment_auth, + eth, + ipv6, + routing, + fragment, + ipv6_auth, + tcp + ); + conform_chain!( + fragment_then_dest_opts, + eth, + ipv6, + fragment, + dest_opts, + icmp6 + ); + conform_chain!( + hop_by_hop_then_fragment, + eth, + ipv6, + hop_by_hop, + fragment, + udp + ); + conform_chain!( + dest_opts_then_fragment, + eth, + ipv6, + dest_opts, + fragment, + icmp6 + ); + conform_chain!(auth_then_dest_opts, eth, ipv6, ipv6_auth, dest_opts, udp); + conform_chain!( + hop_by_hop_then_auth, + eth, + ipv6, + hop_by_hop, + ipv6_auth, + icmp6 + ); + conform_chain!(dest_opts_then_auth, eth, ipv6, dest_opts, ipv6_auth, tcp); + conform_chain!(routing_then_auth, eth, ipv6, routing, ipv6_auth, udp); + conform_chain!(hop_by_hop_then_udp, eth, ipv6, hop_by_hop, udp); + conform_chain!(routing_then_udp, eth, ipv6, routing, udp); + conform_chain!( + hop_by_hop_then_routing_then_tcp, + eth, + ipv6, + hop_by_hop, + routing, + tcp + ); + conform_chain!(hop_by_hop_then_icmp6, eth, ipv6, hop_by_hop, icmp6); + conform_chain!(routing_then_icmp6, eth, ipv6, routing, icmp6); + // The IPv4 authentication header is the only extension that belongs on a v4 packet. + conform_chain!(v4_auth_then_udp, eth, ipv4, ipv4_auth, udp); + conform_chain!(v4_auth_then_icmp4, eth, ipv4, ipv4_auth, icmp4); +} From 97e8d393872ca54cb377219c09267f5abbb9d7b5 Mon Sep 17 00:00:00 2001 From: Daniel Noland Date: Sat, 8 Aug 2026 21:09:45 -0600 Subject: [PATCH 17/33] test(net): Specialize the ICMP subtypes, and find that conform is dead The last of `net/headers`. `builder.rs` had two untouched regions, both the same shape as everything else this campaign has turned up: methods the builder has always offered that no test ever called. The twelve ICMP message subtypes are one. `.dest_unreachable()`, `.redirect()`, `.time_exceeded()`, `.param_problem()`, `.echo_request()`, `.echo_reply()` and their v6 twins were unused, and ten of the twelve `Blank` impls behind them had never been called. The two that had are why the shared macro bodies looked covered -- a `macro_rules!` line counts as run once any one of its expansions runs, so a table of twelve generated impls reports green when one of the twelve is exercised, and only the hand-written part of each shows the difference. Scrambling had to change for these. `conform` writes a message type here rather than a protocol number, and the first attempt scrambled it to `Unknown` with a fuzzed type byte -- which fails six chains for a reason that has nothing to do with `conform`: `Unknown { type_u8: 3 }` deparses to the bytes of a destination-unreachable message and parses back as one. The scramble now uses 253 and 200, reserved for experimentation, which belong to no variant and survive the round trip as themselves. The round trip alone cannot check these. The scrambled type is a well-formed ICMP message, so a packet that never got specialized still deparses and parses back perfectly. The subtype chains assert separately that the scramble did not survive the build. That check is what exposes the finding: all twelve `Within for ` conform bodies are dead. Empty them and every test still passes. `Install` runs unconditionally from `build_headers`, after `conform`, and overwrites whatever `conform` wrote; nothing can be stacked on a subtype, so there is no arrangement in which `conform` gets the last word. The two are indistinguishable until the caller customizes the subtype, because until then both write the same value -- `a_customized_subtype_survives_the_build` is the one test that separates them, and it fails on `Install` and not on `conform`. Pinned, not acted on; emptying them is a call for whoever owns the builder. The other region is an ICMP error quoting an ICMP packet. `EmbeddedAssembler::icmp4` and `::icmp6` were the two inner-transport methods nothing called, and with them the arm of `fixup_embedded` that writes `NextHeader::ICMP` onto the quoted IP header. A ping drawing a destination-unreachable is the ordinary way to produce one. The protocol number is asserted directly rather than left to the shape match, which would pass without it; naming TCP there fails exactly the one test. Coverage: `builder.rs` 79.1% -> 97.7%, `within.rs` unchanged at 88.9%, `net` 79.9% -> 81.1%. Eight lines left in `builder.rs`: two defensive `unreachable!` arms, two absent- layer arms, `Blank for ()` which `stack` never instantiates, and `Default for HeaderStack`. A test written to touch the last of those would be a test that exists to move a number, which is the failure mode this campaign has been finding, not one to add. Co-Authored-By: Claude Opus 5 (1M context) Signed-off-by: Daniel Noland --- net/src/headers/embedded_view.rs | 46 ++++++++++ net/src/headers/within.rs | 142 ++++++++++++++++++++++++++++++- 2 files changed, 185 insertions(+), 3 deletions(-) diff --git a/net/src/headers/embedded_view.rs b/net/src/headers/embedded_view.rs index ad52e28bf7..b07cd5e1f6 100644 --- a/net/src/headers/embedded_view.rs +++ b/net/src/headers/embedded_view.rs @@ -1285,6 +1285,52 @@ mod tests { assert!(matches::<(&Ipv4, &TruncatedTcp)>(e)); } + /// An ICMP error quoting an ICMP packet, which the builder had never been asked to assemble. + /// + /// `EmbeddedAssembler::icmp4` and `::icmp6` were the two inner-transport methods no test called, + /// and with them the `EmbeddedTransport::Icmp4 => NextHeader::ICMP` arm of the fixup that sets + /// the quoted packet's protocol field. A ping that draws a destination-unreachable is the + /// ordinary way to produce one, so the gap was in the tests rather than in the scenario. + /// + /// The inner protocol number is checked directly: `fixup_embedded` is what writes it, the shape + /// match below would pass without it, and a quoted packet whose IP header disagrees with the + /// transport it carries is exactly what a peer cannot parse. + #[test] + fn icmp4_quoted_inside_an_icmp4_error() { + use crate::headers::builder::Blank; + use crate::icmp4::Icmp4; + use crate::ip::NextHeader; + let h = icmp4_with_embedded(|a| a.ipv4(|_| {}).icmp4(Icmp4::blank())); + let e = h.embedded_ip().expect("embedded must be present"); + assert!(matches::<(&Ipv4, &TruncatedIcmp4)>(e)); + let Some(Net::Ipv4(ip)) = e.net() else { + unreachable!("the quoted packet carries an IPv4 header") + }; + assert_eq!( + ip.next_header(), + NextHeader::ICMP, + "the quoted IP header does not name the transport it carries" + ); + } + + #[test] + fn icmp6_quoted_inside_an_icmp6_error() { + use crate::headers::builder::Blank; + use crate::icmp6::Icmp6; + use crate::ip::NextHeader; + let h = icmp6_with_embedded(|a| a.ipv6(|_| {}).icmp6(Icmp6::blank())); + let e = h.embedded_ip().expect("embedded must be present"); + assert!(matches::<(&Ipv6, &TruncatedIcmp6)>(e)); + let Some(Net::Ipv6(ip)) = e.net() else { + unreachable!("the quoted packet carries an IPv6 header") + }; + assert_eq!( + ip.next_header(), + NextHeader::ICMP6, + "the quoted IP header does not name the transport it carries" + ); + } + #[test] fn ipv6_truncated_udp_matches_full_inner_packet() { use crate::udp::UdpPort; diff --git a/net/src/headers/within.rs b/net/src/headers/within.rs index 7e72cf2939..c2e2240537 100644 --- a/net/src/headers/within.rs +++ b/net/src/headers/within.rs @@ -609,8 +609,15 @@ mod conform_properties { use crate::eth::ethtype::EthType; use crate::headers::builder::HeaderStack; use crate::headers::test::parse_back_test; - use crate::icmp4::Icmp4; - use crate::icmp6::Icmp6; + use crate::headers::{Headers, Transport}; + use crate::icmp4::{ + Icmp4, Icmp4DestUnreachable, Icmp4EchoReply, Icmp4EchoRequest, Icmp4ParamProblem, + Icmp4Redirect, Icmp4TimeExceeded, Icmp4Type, + }; + use crate::icmp6::{ + Icmp6, Icmp6DestUnreachable, Icmp6EchoReply, Icmp6EchoRequest, Icmp6PacketTooBig, + Icmp6ParamProblem, Icmp6TimeExceeded, Icmp6Type, + }; use crate::ip::NextHeader; use crate::ip_auth::{Ipv4Auth, Ipv6Auth}; use crate::ipv4::Ipv4; @@ -651,7 +658,52 @@ mod conform_properties { scramble_next_header!( Ipv4, Ipv6, HopByHop, DestOpts, Routing, Fragment, Ipv4Auth, Ipv6Auth ); - scramble_leaf!(Tcp, Udp, Icmp4, Icmp6); + scramble_leaf!(Tcp, Udp); + // The ICMP subtypes are the only layers that can sit on top of an ICMP header, and nothing sits + // on top of them. + scramble_leaf!( + Icmp4DestUnreachable, + Icmp4Redirect, + Icmp4TimeExceeded, + Icmp4ParamProblem, + Icmp4EchoRequest, + Icmp4EchoReply, + Icmp6DestUnreachable, + Icmp6PacketTooBig, + Icmp6TimeExceeded, + Icmp6ParamProblem, + Icmp6EchoRequest, + Icmp6EchoReply, + ); + + // ICMP is the one place where `conform` writes a message type rather than a protocol number, so + // `Unknown` is the scramble: it is the one variant matching no subtype, which makes it wrong for + // every chain below rather than accidentally right for one of them. + // + // The type byte is fixed rather than fuzzed, and has to be. `Unknown` stores the raw byte, so + // `Unknown { type_u8: 3 }` deparses to the same three bytes a destination-unreachable message + // does and parses back as one -- a header the round-trip is right to reject, and nothing to do + // with `conform`. 253 for v4 and 200 for v6 are reserved for experimentation and belong to no + // variant, so they survive the round trip as themselves. The rest of the message stays fuzzed. + impl Scramble for Icmp4 { + fn scramble(&mut self, seed: u8) { + self.set_type(crate::icmp4::Icmp4Type::Unknown { + type_u8: 253, + code_u8: seed, + bytes5to8: [seed; 4], + }); + } + } + + impl Scramble for Icmp6 { + fn scramble(&mut self, seed: u8) { + self.set_type(crate::icmp6::Icmp6Type::Unknown { + type_u8: 200, + code_u8: seed, + bytes5to8: [seed; 4], + }); + } + } impl Scramble for Eth { fn scramble(&mut self, seed: u8) { @@ -673,6 +725,12 @@ mod conform_properties { /// the same closure once per layer. macro_rules! conform_chain { ($name:ident, $($layer:ident),+ $(,)?) => { + conform_chain!(@build $name, |_| {}, $($layer),+); + }; + (specialized $name:ident, $($layer:ident),+ $(,)?) => { + conform_chain!(@build $name, icmp_type_was_specialized, $($layer),+); + }; + (@build $name:ident, $check:expr, $($layer:ident),+) => { #[test] fn $name() { bolero::check!().with_type().for_each(|seed: &u8| { @@ -684,11 +742,35 @@ mod conform_properties { unreachable!("a blank {} chain does not overflow: {e:?}", stringify!($name)) }); parse_back_test(&headers); + $check(&headers); }); } }; } + /// The scrambled ICMP type did not survive into the built packet. + /// + /// Only for chains that end in a subtype layer. A chain ending at a bare `.icmp4()` has nothing + /// above it to conform it, so the scramble is *supposed* to survive there, and round-tripping is + /// the only thing to check. + /// + /// Worth stating why this is separate from the round trip rather than folded into it: the + /// scrambled type is a well-formed ICMP message, so a packet still carrying it deparses and + /// parses back perfectly. The round trip cannot tell that the specialization never happened. + fn icmp_type_was_specialized(headers: &Headers) { + match headers.transport() { + Some(Transport::Icmp4(icmp)) => assert!( + !matches!(icmp.icmp_type(), Icmp4Type::Unknown { type_u8: 253, .. }), + "the scrambled ICMPv4 type survived the build, so nothing specialized it" + ), + Some(Transport::Icmp6(icmp)) => assert!( + !matches!(icmp.icmp_type(), Icmp6Type::Unknown { type_u8: 200, .. }), + "the scrambled ICMPv6 type survived the build, so nothing specialized it" + ), + other => unreachable!("a subtype chain builds an ICMP transport, got {other:?}"), + } + } + // Seventeen chains, chosen to cover all thirty-two unreached transitions between them. Three // extension headers is the ceiling -- `MAX_NET_EXTENSIONS` -- so the deeper regions of the graph // have to be reached by several chains rather than one long one. @@ -771,4 +853,58 @@ mod conform_properties { // The IPv4 authentication header is the only extension that belongs on a v4 packet. conform_chain!(v4_auth_then_udp, eth, ipv4, ipv4_auth, udp); conform_chain!(v4_auth_then_icmp4, eth, ipv4, ipv4_auth, icmp4); + + // The ICMP message subtypes, which the builder can specialize into and which no test had ever + // asked for. Ten of the twelve `Blank` impls behind them had never been called either -- the two + // that had were what made the shared macro bodies look covered, since a `macro_rules!` line + // counts as run once any one of its expansions runs. Worth knowing generally: a table of twelve + // generated impls reports as covered when one of the twelve is exercised, and only the + // hand-written part of each -- here `blank()` -- shows the difference. + conform_chain!(specialized icmp4_dest_unreachable, eth, ipv4, icmp4, dest_unreachable); + conform_chain!(specialized icmp4_redirect, eth, ipv4, icmp4, redirect); + conform_chain!(specialized icmp4_time_exceeded, eth, ipv4, icmp4, time_exceeded); + conform_chain!(specialized icmp4_param_problem, eth, ipv4, icmp4, param_problem); + conform_chain!(specialized icmp4_echo_request, eth, ipv4, icmp4, echo_request); + conform_chain!(specialized icmp4_echo_reply, eth, ipv4, icmp4, echo_reply); + conform_chain!(specialized icmp6_dest_unreachable, eth, ipv6, icmp6, dest_unreachable6); + conform_chain!(specialized icmp6_packet_too_big, eth, ipv6, icmp6, packet_too_big6); + conform_chain!(specialized icmp6_time_exceeded, eth, ipv6, icmp6, time_exceeded6); + conform_chain!(specialized icmp6_param_problem, eth, ipv6, icmp6, param_problem6); + conform_chain!(specialized icmp6_echo_request, eth, ipv6, icmp6, echo_request6); + conform_chain!(specialized icmp6_echo_reply, eth, ipv6, icmp6, echo_reply6); + + /// A subtype the caller customized, which is what separates the two writers of the ICMP type. + /// + /// The chains above cannot tell `Within::conform` from `Install` for these layers, and no test + /// could, because both write the same value: `conform` sets `DestUnreachable(blank())` and + /// `Install` sets `DestUnreachable(value)`, and `value` is `blank()` whenever the caller does not + /// change it. Empty either one alone and the other still produces the right packet. + /// + /// Choosing a code other than the blank one separates them, and the answer is that `conform` is + /// the redundant half. `Install` runs unconditionally from `build_headers`, after `conform`, and + /// overwrites whatever `conform` wrote -- so all twelve of these `conform` bodies could be empty + /// with no observable change. Nothing can be stacked on a subtype, so there is no arrangement in + /// which `conform` gets the last word. + /// + /// Pinned here rather than acted on: emptying them is a call for whoever owns the builder. + #[test] + fn a_customized_subtype_survives_the_build() { + let headers = HeaderStack::new() + .eth(|l| l.scramble(0)) + .ipv4(|l| l.scramble(0)) + .icmp4(|l| l.scramble(0)) + .dest_unreachable(|d| *d = Icmp4DestUnreachable::Port) + .build_headers() + .unwrap_or_else(|e| unreachable!("a blank chain does not overflow: {e:?}")); + + let Some(Transport::Icmp4(icmp)) = headers.transport() else { + unreachable!("the chain builds an ICMPv4 transport") + }; + assert_eq!( + icmp.icmp_type(), + Icmp4Type::DestUnreachable(Icmp4DestUnreachable::Port), + "the code the caller chose did not reach the built packet" + ); + parse_back_test(&headers); + } } From 332eaf2b17911de18dee30fb5dbc532a267f54cb Mon Sep 17 00:00:00 2001 From: Daniel Noland Date: Mon, 17 Aug 2026 14:31:57 -0600 Subject: [PATCH 18/33] test(net): Keep the embedded vacuity guard armed under coverage The guard only asserts once a run is long enough for a miss to mean something, and the threshold was set at the twenty-five thousand cases a default one-second run managed when it was written. That leaves no room: coverage instrumentation costs about a fifth of the throughput here -- 17 hits in 20,206 cases against 21 in 25,857 without it -- so the threshold now sits a couple of hundred cases below what CI actually draws. A busier runner drops under it and the check disappears without saying so, which is the failure mode the guard exists to prevent. Ten thousand instead. The thinnest shape in this module draws about eight hits per ten thousand, so at that many draws a shape that really is reachable comes up empty about three times in ten thousand runs. The instrumentation cost is small because these properties are bound by the generator rather than by a counter loop, which is the opposite of the fib test that `--cfg=instrumented` was added for. No iteration counts need cutting here. Co-Authored-By: Claude Opus 5 (1M context) Signed-off-by: Daniel Noland --- net/src/headers/embedded_view.rs | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/net/src/headers/embedded_view.rs b/net/src/headers/embedded_view.rs index b07cd5e1f6..f5aeee0ad6 100644 --- a/net/src/headers/embedded_view.rs +++ b/net/src/headers/embedded_view.rs @@ -1966,9 +1966,15 @@ mod embedded_view_properties { // compounding costs an order of magnitude: the widest pair here matches about one packet in // twenty, and `(&Ipv6, &HopByHop, &TruncatedTcp)` inside an ICMPv6 error matches about one // in a thousand. Five hundred cases at that rate would fail this check outright half the - // time it ran. A default one-second run manages twenty-five thousand, so the guard still - // bites in CI; a deliberately short run should say nothing rather than say something false. - if seen > 20_000 { + // time it ran. A deliberately short run should say nothing rather than say something false. + // + // Ten thousand, not the twenty-five thousand a default run manages here. The thinnest shape + // in this module draws about eight hits per ten thousand, and coverage instrumentation costs + // about a fifth of the throughput -- measured at 17 hits in 20,206 cases against 21 in + // 25,857 without. A threshold set at the observed count would therefore switch itself off on + // any busier runner, silently. At ten thousand draws a shape that really is drawable misses + // entirely about three times in ten thousand runs, which is the trade this wants. + if seen > 10_000 { assert!( hit > 0, "{shape} never matched in {seen} packets: the two walks agree only because the \ From d752ab179c4ffb9933e7528a540941256988668e Mon Sep 17 00:00:00 2001 From: Daniel Noland Date: Fri, 7 Aug 2026 12:31:14 -0600 Subject: [PATCH 19/33] test(routing): Property-test the rib-to-fib conversion First property tests in this crate, which had none -- 59 tests over 15,311 lines, and bolero already a dev-dependency waiting to be used. Two targets, both on the path from the rib to the fib the forwarder reads. `FibEntry::squash` folds every egress instruction of an entry into one, keeping the first interface and the last address of the resolution chain. That asymmetry is load-bearing: rib2fib notes that without it the address of a recursive next-hop never reaches the fib and the egress stage resolves the packet's destination instead, which is right only for a directly connected host. Four properties -- the other instructions survive in order, at most one egress and it goes last, the merge follows first-interface / last-address / first-name, and squashing twice is squashing once. Values come from a small alphabet so that egress objects disagreeing about the same field is the common case; independently drawn ones would almost never collide. `Nhop::build_nhop_fibgroup` walks the resolver graph and emits one entry per root-to-leaf path, squashed and filtered by `FibEntry::is_valid`, with a drop injected if nothing survives. The oracle enumerates the paths from the edge list instead of walking the same recursion. Removing the unresolved-leaf filter fails it; removing the drop fallback fails it and the companion property that every entry a group offers is one the forwarder can execute. `resolves_with` gets its own property, that it answers reachability in the resolver graph, checked against a closure over the edge list. It is worth stating plainly because everything above depends on it: neither `build_nhop_fibgroup_rec` nor `resolves_with` itself has a base case for a cycle. Acyclicity is an inductive invariant maintained entirely by `lazy_resolve` refusing an edge whose target already reaches the source. The generator produces graphs in topological order for that reason -- a generated cycle would not find a bug, it would exhaust the stack. Co-Authored-By: Claude Opus 5 (1M context) Signed-off-by: Daniel Noland --- routing/src/fib/fibobjects.rs | 194 ++++++++++++++++++++++++++++ routing/src/rib/nexthop.rs | 233 ++++++++++++++++++++++++++++++++++ 2 files changed, 427 insertions(+) diff --git a/routing/src/fib/fibobjects.rs b/routing/src/fib/fibobjects.rs index 70c7b25151..4830ebd96f 100644 --- a/routing/src/fib/fibobjects.rs +++ b/routing/src/fib/fibobjects.rs @@ -262,3 +262,197 @@ pub enum PktInstruction { Encap(Encapsulation), /* encapsulate the packet */ Egress(EgressObject), /* send the packet over interface to some ip */ } + +#[cfg(test)] +mod squash_properties { + use super::*; + use crate::rib::encapsulation::VxlanEncapsulation; + use bolero::{Driver, ValueGenerator}; + use std::net::Ipv4Addr; + use std::num::NonZero; + use std::ops::Bound::Included; + + // Values come from a small alphabet, so that several egress objects disagreeing about the same + // field is the common case rather than a rarity. Deciding between them is the whole of what + // `squash` does; independently drawn values would almost never collide. + const ADDRESSES: [IpAddr; 3] = [ + IpAddr::V4(Ipv4Addr::new(10, 0, 0, 1)), + IpAddr::V4(Ipv4Addr::new(10, 0, 0, 2)), + IpAddr::V4(Ipv4Addr::new(10, 0, 0, 3)), + ]; + const IFNAMES: [&str; 2] = ["eth0", "eth1"]; + + fn index(raw: u8) -> InterfaceIndex { + InterfaceIndex::new(NonZero::new(u32::from(raw)).unwrap_or_else(|| unreachable!())) + } + + // Zero selects `None`, which is one of the choices for every field: an unresolved ifindex or a + // missing address is exactly what the merge rules are about. The draw is separate from the + // choice so that a driver running out of input stays distinct from a generated `None`. + fn choose(pick: u8, choices: &[T]) -> Option { + (pick > 0).then(|| choices[usize::from(pick - 1)].clone()) + } + + fn egress(driver: &mut D) -> Option { + let ifindex = driver.gen_u8(Included(&0), Included(&3))?; + let address = driver.gen_u8(Included(&0), Included(&3))?; + let ifname = driver.gen_u8(Included(&0), Included(&2))?; + Some(EgressObject::new( + choose(ifindex, &[index(1), index(2), index(3)]), + choose(address, &ADDRESSES), + choose(ifname, &IFNAMES).map(str::to_string), + )) + } + + fn instruction(driver: &mut D) -> Option { + Some(match driver.gen_u8(Included(&0), Included(&3))? { + // `Local` carries a distinguishable index so that order preservation can be checked + // rather than merely counted. + 0 => PktInstruction::Local(index(driver.gen_u8(Included(&1), Included(&3))?)), + 1 => PktInstruction::Drop, + 2 => PktInstruction::Encap(Encapsulation::Vxlan(VxlanEncapsulation::new( + Vni::new_checked(u32::from(driver.gen_u8(Included(&1), Included(&3))?)) + .unwrap_or_else(|_| unreachable!()), + ADDRESSES[0], + ))), + _ => PktInstruction::Egress(egress(driver)?), + }) + } + + #[derive(Debug, Clone, Copy, Default)] + struct Entry; + + impl ValueGenerator for Entry { + type Output = FibEntry; + + fn generate(&self, driver: &mut D) -> Option { + let count = driver.gen_u8(Included(&0), Included(&5))?; + let mut entry = FibEntry::new(); + for _ in 0..count { + entry.add(instruction(driver)?); + } + Some(entry) + } + } + + fn egresses(entry: &FibEntry) -> Vec<&EgressObject> { + entry + .iter() + .filter_map(|inst| match inst { + PktInstruction::Egress(e) => Some(e), + _ => None, + }) + .collect() + } + + fn others(entry: &FibEntry) -> Vec<&PktInstruction> { + entry + .iter() + .filter(|inst| !matches!(inst, PktInstruction::Egress(_))) + .collect() + } + + /// Squashing an entry leaves everything that is not an egress exactly as it was. + /// + /// The instructions before the egress are what gets executed on the way out -- encapsulation + /// above all -- so reordering or dropping one of them changes what goes on the wire. + #[test] + fn squash_preserves_the_other_instructions_in_order() { + bolero::check!() + .with_generator(Entry) + .cloned() + .for_each(|entry: FibEntry| { + let before: Vec = others(&entry).into_iter().cloned().collect(); + let mut squashed = entry.clone(); + squashed.squash(); + // A single instruction is returned untouched, egress or not. + if entry.len() == 1 { + assert_eq!(squashed, entry); + return; + } + let after: Vec = others(&squashed).into_iter().cloned().collect(); + assert_eq!(after, before, "for {entry:?}"); + }); + } + + /// What comes out has at most one egress, and it is last. + /// + /// Last because `FibEntry::is_valid` requires it: a multi-instruction entry is only usable if + /// its final instruction is an egress with a known interface. + #[test] + fn squash_leaves_at_most_one_egress_and_puts_it_last() { + bolero::check!() + .with_generator(Entry) + .cloned() + .for_each(|entry: FibEntry| { + if entry.len() == 1 { + return; + } + let mut squashed = entry.clone(); + squashed.squash(); + + assert!(egresses(&squashed).len() <= 1, "for {entry:?}"); + if let Some(position) = squashed + .iter() + .position(|inst| matches!(inst, PktInstruction::Egress(_))) + { + assert_eq!(position, squashed.len() - 1, "for {entry:?}"); + } + }); + } + + /// The merged egress is the first interface, the last address and the first name. + /// + /// The asymmetry is deliberate and load-bearing: `rib2fib` relies on it so that the address of + /// a recursive next-hop reaches the fib while the interface of the nearest resolver wins. An + /// oracle worked out from the input directly, rather than by folding with `merge` again. + #[test] + fn squash_merges_first_interface_last_address_first_name() { + bolero::check!() + .with_generator(Entry) + .cloned() + .for_each(|entry: FibEntry| { + if entry.len() == 1 { + return; + } + let inputs = egresses(&entry); + let ifindex = inputs.iter().find_map(|e| *e.ifindex()); + let address = inputs.iter().rev().find_map(|e| *e.address()); + let ifname = inputs.iter().find_map(|e| e.ifname().clone()); + + let mut squashed = entry.clone(); + squashed.squash(); + + match egresses(&squashed).first() { + Some(merged) => { + assert_eq!(*merged.ifindex(), ifindex, "interface, for {entry:?}"); + assert_eq!(*merged.address(), address, "address, for {entry:?}"); + assert_eq!(*merged.ifname(), ifname, "name, for {entry:?}"); + } + // An egress survives exactly when one of the inputs knew an interface. With + // none, there is nowhere to send the packet and the egress is dropped -- which + // is what leaves the entry to be refused by `is_valid`. + None => assert!(ifindex.is_none(), "for {entry:?}"), + } + }); + } + + /// Squashing twice is squashing once. + /// + /// Worth pinning because the entries are built by accumulating down a resolution chain and + /// squashed at the end of it; a squash that drifted on a second application would make the + /// result depend on how many times the chain was walked. + #[test] + fn squash_is_idempotent() { + bolero::check!() + .with_generator(Entry) + .cloned() + .for_each(|entry: FibEntry| { + let mut once = entry.clone(); + once.squash(); + let mut twice = once.clone(); + twice.squash(); + assert_eq!(twice, once, "for {entry:?}"); + }); + } +} diff --git a/routing/src/rib/nexthop.rs b/routing/src/rib/nexthop.rs index f4a52e0582..34480eda70 100644 --- a/routing/src/rib/nexthop.rs +++ b/routing/src/rib/nexthop.rs @@ -1045,3 +1045,236 @@ mod tests { assert!(a.resolves_with(checked.as_ref())); } } + +#[cfg(test)] +mod fibgroup_properties { + use super::*; + use crate::fib::fibobjects::FibEntry; + use bolero::{Driver, ValueGenerator}; + use std::ops::Bound::Included; + + const MAX_NODES: u8 = 6; + + /// A next-hop graph, given as a topological order. + /// + /// Node `i` may only resolve via nodes after it, so the graph is acyclic by construction. That + /// is a precondition rather than a simplification: `build_nhop_fibgroup_rec` has no loop guard + /// of its own, and neither does `resolves_with`. Acyclicity is maintained inductively by + /// `lazy_resolve`, which refuses an edge whose target already resolves via the source -- so a + /// generated cycle here would not find a bug, it would recurse until the stack ran out. See + /// `a_cycle_is_refused_before_it_is_added` for the other half. + #[derive(Debug, Clone)] + struct Dag { + /// `shape[i]` are the offsets, relative to `i`, of the nodes `i` resolves via. + shape: Vec>, + /// Whether node `i` knows an interface, and so needs no resolving. + grounded: Vec, + } + + impl Dag { + /// The edges, as concrete (from, to) index pairs. + /// + /// Shared by the graph and the oracles below: the edge list is the *input*, so sharing it + /// keeps them describing the same graph. What the oracles must not share is the traversal + /// under test. + fn edges(&self) -> Vec<(usize, usize)> { + let mut edges = Vec::new(); + for (from, offsets) in self.shape.iter().enumerate() { + for offset in offsets { + let to = from + usize::from(*offset); + if to < self.shape.len() { + edges.push((from, to)); + } + } + } + edges + } + + /// Which nodes are reachable from `start`, itself included. A plain closure, computed + /// without asking any next-hop anything. + fn reachable_from(&self, start: usize) -> Vec { + let edges = self.edges(); + let mut seen = vec![false; self.shape.len()]; + let mut stack = vec![start]; + while let Some(node) = stack.pop() { + if std::mem::replace(&mut seen[node], true) { + continue; + } + for (from, to) in &edges { + if *from == node { + stack.push(*to); + } + } + } + seen + } + } + + /// Draws [`Dag`]s. + #[derive(Debug, Clone, Copy, Default)] + struct Graphs; + + impl ValueGenerator for Graphs { + type Output = Dag; + + fn generate(&self, driver: &mut D) -> Option { + let nodes = usize::from(driver.gen_u8(Included(&1), Included(&MAX_NODES))?); + let mut shape = Vec::with_capacity(nodes); + let mut grounded = Vec::with_capacity(nodes); + for index in 0..nodes { + let behind = u8::try_from(nodes - index - 1).ok()?; + let count = driver.gen_u8(Included(&0), Included(&behind.min(2)))?; + let mut edges = Vec::new(); + for _ in 0..count { + edges.push(driver.gen_u8(Included(&1), Included(&behind.max(1)))?); + } + shape.push(edges); + grounded.push(driver.produce::()?); + } + Some(Dag { shape, grounded }) + } + } + + // Build the graph in an `NhopStore`, returning the nodes in topological order. + fn realize(dag: &Dag) -> (NhopStore, Vec>) { + let mut store = NhopStore::new(); + let nodes: Vec> = (0..dag.shape.len()) + .map(|index| { + let raw = u8::try_from(index).unwrap_or_else(|_| unreachable!()); + let mut key = NhopKey::from_address(&format!("10.0.0.{}", raw + 1)); + if dag.grounded[index] { + key.ifindex = Some( + InterfaceIndex::try_new(u32::from(raw) + 1) + .unwrap_or_else(|_| unreachable!()), + ); + } + store.add_nhop(&key) + }) + .collect(); + + for (from, to) in dag.edges() { + nodes[from].add_resolver(&nodes[to]); + } + (store, nodes) + } + + // The oracle: every root-to-leaf path, concatenated, squashed, and kept if usable. + // + // Worked out from the graph directly rather than by walking the same recursion the code does. + fn expected(node: &Rc, prefix: &FibEntry, out: &mut Vec) { + let mut entry = prefix.clone(); + entry.extend_from_slice(&node.instructions.borrow().clone()); + + let resolvers: Vec> = node + .resolvers + .borrow() + .iter() + .filter_map(Weak::upgrade) + .collect(); + + if resolvers.is_empty() { + // A next-hop with neither an interface nor a way to reach one contributes nothing. + if node.must_be_resolved() { + return; + } + entry.squash(); + if entry.is_valid() { + out.push(entry); + } + } else { + for resolver in resolvers { + expected(&resolver, &entry, out); + } + } + } + + /// A next-hop's fib group is one entry per usable resolution path, and never empty. + #[test] + fn a_fibgroup_is_the_usable_paths_through_the_graph() { + let rstore = RmacStore::new(); + bolero::check!() + .with_generator(Graphs) + .cloned() + .for_each(|dag: Dag| { + let (_store, nodes) = realize(&dag); + for node in &nodes { + node.build_nhop_instructions(&rstore); + } + + let root = &nodes[0]; + let mut want = Vec::new(); + expected(root, &FibEntry::new(), &mut want); + if want.is_empty() { + // Nothing usable: the group carries a drop so packets are not misrouted. + want.push(FibEntry::drop_fibentry()); + } + + let got = root.build_nhop_fibgroup(); + assert_eq!(got.entries(), &want, "for {dag:?}"); + }); + } + + /// Every entry a fib group offers is one the forwarder can execute. + /// + /// `FibEntry::is_valid` is the written-down form of that, and `rib2fib` filters on it -- but + /// the drop injected when nothing is usable bypasses the filter, so it is worth asserting over + /// the group rather than trusting the one call site. + #[test] + fn every_entry_in_a_fibgroup_is_usable() { + let rstore = RmacStore::new(); + bolero::check!() + .with_generator(Graphs) + .cloned() + .for_each(|dag: Dag| { + let (_store, nodes) = realize(&dag); + for node in &nodes { + node.build_nhop_instructions(&rstore); + } + let group = nodes[0].build_nhop_fibgroup(); + assert!(!group.is_empty(), "for {dag:?}"); + for entry in group.iter() { + assert!(entry.is_valid(), "unusable entry {entry:?} for {dag:?}"); + } + }); + } + + /// `resolves_with` answers reachability in the resolver graph. + /// + /// That is the whole of what the loop guard rests on: `lazy_resolve` refuses an edge from `a` + /// to `r` exactly when `r.resolves_with(a)`, which is to say when `a` is already reachable + /// from `r` and the edge would close a cycle. Checked against a closure computed over the edge + /// list, which asks no next-hop anything. + #[test] + fn resolves_with_answers_reachability() { + bolero::check!() + .with_generator(Graphs) + .cloned() + .for_each(|dag: Dag| { + let (_store, nodes) = realize(&dag); + for (from, node) in nodes.iter().enumerate() { + let reachable = dag.reachable_from(from); + for (to, other) in nodes.iter().enumerate() { + assert_eq!( + node.resolves_with(other), + reachable[to], + "{from} -> {to}, for {dag:?}" + ); + } + } + }); + } + + /// A next-hop always resolves via itself, which is what makes the guard refuse a self-loop. + #[test] + fn a_next_hop_resolves_via_itself() { + bolero::check!() + .with_generator(Graphs) + .cloned() + .for_each(|dag: Dag| { + let (_store, nodes) = realize(&dag); + for node in &nodes { + assert!(node.resolves_with(node)); + } + }); + } +} From 19b34b643dc5bb4f1016956fe053e2f15b069a2c Mon Sep 17 00:00:00 2001 From: Daniel Noland Date: Fri, 7 Aug 2026 13:00:51 -0600 Subject: [PATCH 20/33] fix(routing): Survive a resolution loop in the next-hop walks Four separate recursions walk the next-hop resolver graph, and not one of them had a base case for a cycle: - `Nhop::resolves_with` - `Nhop::build_nhop_fibgroup_rec` - `fmt_nhop_resolvers` and `fmt_nhop_rec`, behind `Display for Nhop` and `Display for NhopStore` - `Nhop::quick_resolve_rec` (tests only) Acyclicity was an inductive invariant maintained entirely by `lazy_resolve`, which consults `resolves_with` before wiring each edge and refuses one that would close a cycle. That guard is correct, but nothing in the types connects it to the four recursions that depend on it, and `add_resolver` applies no guard at all. Worse, the recursion protecting the others could not protect itself: `resolves_with` is the first thing a cycle would break. Each walk now carries the set of next-hops it has already visited and stops rather than going round again. A next-hop is identified by address rather than by key, because a next-hop may hold resolvers belonging to another store, where an equal key would name a different object -- a resolution loop is a loop in the object graph. `build_nhop_fibgroup_rec` contributes nothing from a looping path, so a next-hop with no other way out ends up with the drop entry that `build_nhop_fibgroup` already injects for an empty group. That is the right answer: a packet caught in a routing loop should be dropped rather than forwarded round it. Along the way this found a live bug in the display path. `fmt_nhop_resolvers` tracked depth in a `u8` and incremented it per level, so a cycle recursed until that counter overflowed -- a panic raised from inside a `Display` impl, reachable from the CLI and from the very warning the new fib-group guard logs about a resolution loop. The counter is now saturating as well as guarded, which also removes the overflow for a legitimately deep chain. The property tests added in 317cc5bbb generated graphs in topological order and said so in a comment, because a generated cycle would have found the stack rather than a bug. They now generate an arbitrary adjacency list, self-loops included, which is where the interesting inputs were all along. Five million cases pass. Each guard was confirmed load-bearing by removing it: without the `resolves_with` visited set or the `build_nhop_fibgroup_rec` path set, the covering property overflows the stack and aborts the test process; without the display guard, `test_display_of_a_resolution_loop_terminates` panics at the increment. Co-Authored-By: Claude Opus 5 (1M context) Signed-off-by: Daniel Noland --- routing/src/cli/display.rs | 65 +++++--- routing/src/rib/nexthop.rs | 294 ++++++++++++++++++++++++------------- routing/src/rib/rib2fib.rs | 44 +++++- 3 files changed, 276 insertions(+), 127 deletions(-) diff --git a/routing/src/cli/display.rs b/routing/src/cli/display.rs index 9c40338de0..e400c9793e 100644 --- a/routing/src/cli/display.rs +++ b/routing/src/cli/display.rs @@ -21,7 +21,7 @@ use crate::router::cpi::{CpiStats, CpiStatus, StatsRow}; use crate::rib::VrfTable; use crate::rib::encapsulation::{Encapsulation, VxlanEncapsulation}; -use crate::rib::nexthop::{FwAction, Nhop, NhopKey, NhopStore}; +use crate::rib::nexthop::{FwAction, Nhop, NhopKey, NhopStore, Visited}; use crate::rib::vrf::{Route, RouteFlags, RouteOrigin, ShimNhop, Vrf, VrfStatus}; use crate::interfaces::iftable::IfTable; @@ -40,7 +40,7 @@ use net::vxlan::Vni; use std::fmt::Display; use std::fmt::Write; use std::os::unix::net::SocketAddr; -use std::rc::Rc; +use std::rc::{Rc, Weak}; use std::time::Duration; use std::time::Instant; @@ -126,27 +126,40 @@ impl Display for Nhop { if self.is_unresolved() { write!(f, " (unresolved)")?; } - fmt_nhop_resolvers(f, self, 2) + fmt_nhop_resolvers(f, self, 2, &mut vec![self.id()]) } } -fn fmt_nhop_resolvers(f: &mut std::fmt::Formatter<'_>, rc: &Nhop, depth: u8) -> std::fmt::Result { +/// Print a next-hop's resolvers, and theirs, and so on down. +/// +/// `path` holds the next-hops between the one being displayed and this one. A resolver already on +/// that path closes a resolution loop: we name it and stop, since the recursion has nothing new to +/// show and would otherwise run until `depth` overflowed -- which it did, panicking from inside a +/// `Display` impl that both the CLI and the warning about resolution loops go through. +fn fmt_nhop_resolvers( + f: &mut std::fmt::Formatter<'_>, + rc: &Nhop, + depth: u8, + path: &mut Visited, +) -> std::fmt::Result { let Ok(resolvers) = rc.resolvers.try_borrow() else { warn!("Try-borrow on nhop resolvers failed!"); return Ok(()); }; let tab = 5 * depth as usize; let indent = " ".repeat(tab); - if !resolvers.is_empty() { - for r in resolvers.iter() { - if let Some(r) = r.upgrade().as_ref() { - write!(f, "\n{indent} {}", r.key)?; - if r.is_unresolved() { - write!(f, " (UNRESOLVED)")?; - } - fmt_nhop_resolvers(f, r, depth + 1)?; - } + for r in resolvers.iter().filter_map(Weak::upgrade) { + write!(f, "\n{indent} {}", r.key)?; + if r.is_unresolved() { + write!(f, " (UNRESOLVED)")?; + } + if path.contains(&r.id()) { + write!(f, " (LOOP)")?; + continue; } + path.push(r.id()); + fmt_nhop_resolvers(f, &r, depth.saturating_add(1), path)?; + path.pop(); } Ok(()) } @@ -168,7 +181,14 @@ fn fmt_nhop_instruction(f: &mut std::fmt::Formatter<'_>, rc: &Nhop) -> std::fmt: // formats nhop using the display of the key, recoursing over resolvers // Does not use Nhop::fmt(). -fn fmt_nhop_rec(f: &mut std::fmt::Formatter<'_>, rc: &Rc, depth: u8) -> std::fmt::Result { +// +// `path` guards against a resolution loop, as in `fmt_nhop_resolvers` above. +fn fmt_nhop_rec( + f: &mut std::fmt::Formatter<'_>, + rc: &Rc, + depth: u8, + path: &mut Visited, +) -> std::fmt::Result { let tab = 8 * depth as usize; let indent = " ".repeat(tab); @@ -184,6 +204,9 @@ fn fmt_nhop_rec(f: &mut std::fmt::Formatter<'_>, rc: &Rc, depth: u8) -> st if rc.is_unresolved() { write!(f, " (UNRESOLVED)")?; } + if path.contains(&rc.id()) { + return writeln!(f, " (LOOP)"); + } writeln!(f)?; // fmt_nhop_instruction(f, rc)?; @@ -191,11 +214,11 @@ fn fmt_nhop_rec(f: &mut std::fmt::Formatter<'_>, rc: &Rc, depth: u8) -> st error!("Try-borrow on next-hop resolvers failed!"); return Ok(()); }; - for r in resolvers.iter() { - if let Some(r) = r.upgrade().as_ref() { - fmt_nhop_rec(f, r, depth + 1)?; - } + path.push(rc.id()); + for r in resolvers.iter().filter_map(Weak::upgrade) { + fmt_nhop_rec(f, &r, depth.saturating_add(1), path)?; } + path.pop(); // if let Ok(fg) = rc.as_ref().fibgroup.read() { // writeln!(f, "FibG {}", fg)?; // } @@ -206,7 +229,7 @@ impl Display for NhopStore { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { Heading(format!("Next-hop Store ({})", self.len())).fmt(f)?; for nhop in self.iter() { - fmt_nhop_rec(f, nhop, 0)?; + fmt_nhop_rec(f, nhop, 0, &mut Visited::new())?; fmt_nhop_instruction(f, nhop)?; } line(f) @@ -407,7 +430,7 @@ impl Display for VrfV4Nexthops<'_> { .filter(|nh| nh.key.address.is_none_or(|a| a.is_ipv4())); for nhop in iter { - fmt_nhop_rec(f, nhop, 0)?; + fmt_nhop_rec(f, nhop, 0, &mut Visited::new())?; } line(f) } @@ -425,7 +448,7 @@ impl Display for VrfV6Nexthops<'_> { .filter(|nh| nh.key.address.is_none_or(|a| a.is_ipv6())); for nhop in iter { - fmt_nhop_rec(f, nhop, 0)?; + fmt_nhop_rec(f, nhop, 0, &mut Visited::new())?; } line(f) } diff --git a/routing/src/rib/nexthop.rs b/routing/src/rib/nexthop.rs index 34480eda70..0d48ebe2fd 100644 --- a/routing/src/rib/nexthop.rs +++ b/routing/src/rib/nexthop.rs @@ -161,6 +161,20 @@ impl Hash for Nhop { } } +/// A next-hop's address in memory, used to tell next-hops apart in the visited sets that the +/// walks over the resolver graph keep. +/// +/// Address rather than key: keys are unique within one [`NhopStore`], but a next-hop may hold +/// resolvers that belong to another one, where an equal key would name a different object. A +/// resolution loop is a loop in the object graph, and next-hops live in `Rc`s, which do not move. +pub(crate) type NhopId = *const Nhop; + +/// The next-hops a walk over the resolver graph has already visited. +/// +/// A `Vec` rather than a set: resolver chains are a handful of next-hops long and fan out by two +/// or three, so a linear scan costs less than hashing. +pub(crate) type Visited = Vec; + impl Nhop { /// Create a new Nhop object from a key object fn from_key(key: &NhopKey) -> Self { @@ -184,23 +198,47 @@ impl Nhop { self } - /// Recursive method to check if a next-hop resolves via another, `checked`. - /// We use this method to avoid resolution loops that would happen in case of routing loops. - /// Resolution loops would cause us to stack overflow. This method is recursive, but - /// short-circuits in case of loop. The method takes the advantage that there cannot be two - /// next-hops with the same key. + /// This next-hop's identity for the visited sets of the walks over the resolver graph. + pub(crate) fn id(&self) -> NhopId { + std::ptr::from_ref(self) + } + + /// Tell if a next-hop resolves via another, `checked`: that is, whether `checked` is reachable + /// from `self` along resolver edges, `self` included. + /// + /// This is the guard against routing loops. `lazy_resolve` refuses an edge from `a` to `r` + /// exactly when `r.resolves_with(a)`, since such an edge would close a cycle, and a cycle in + /// the resolver graph would send the walks over it round for ever. + /// + /// The walk is total whether or not the graph already holds a cycle, because `visited` stops + /// it from entering any next-hop twice. That matters for two reasons: the guard should not + /// depend on the very invariant it exists to maintain, and a diamond-shaped graph is otherwise + /// re-walked once per path through it. fn resolves_with(&self, checked: &Nhop) -> bool { + self.resolves_with_rec(checked, &mut Visited::new()) + } + + fn resolves_with_rec(&self, checked: &Nhop, visited: &mut Visited) -> bool { // resolve to oneself is forbidden if self.key == checked.key { error!("Loop detected for next-hop {}!", self.key); return true; } + // a next-hop already visited leads nowhere new: either we are inside a cycle, or we + // reached it by another path and have already looked at everything beyond it. + // a next-hop already visited leads nowhere new: either we are inside a cycle, or we + // reached it by another path and have already looked at everything beyond it. + if visited.contains(&self.id()) { + return false; + } + visited.push(self.id()); + // resolvers should not refer back to the checked next-hop let resolvers = self.resolvers.borrow(); resolvers .iter() .filter_map(Weak::upgrade) - .any(|res| res.resolves_with(checked)) + .any(|res| res.resolves_with_rec(checked, visited)) } /// Tell if a next-hop requires resolution @@ -264,8 +302,15 @@ impl Nhop { } /// Auxiliary recursive method used by `Nhop::quick_resolve()`. + /// + /// `visited` guards against a resolution loop, as in `resolves_with` above. #[cfg(test)] - fn quick_resolve_rec(&self, result: &mut BTreeSet) { + fn quick_resolve_rec(&self, result: &mut BTreeSet, visited: &mut Visited) { + if visited.contains(&self.id()) { + return; + } + visited.push(self.id()); + let Ok(resolvers) = self.resolvers.try_borrow_mut() else { error!("Try-borrow-mut() failed on next-hop resolvers!"); return; @@ -297,7 +342,7 @@ impl Nhop { self.key.ifname.clone(), )); } else { - r.quick_resolve_rec(result); + r.quick_resolve_rec(result, visited); } } } @@ -311,7 +356,7 @@ impl Nhop { #[cfg(test)] pub fn quick_resolve(&self) -> BTreeSet { let mut out: BTreeSet = BTreeSet::new(); - self.quick_resolve_rec(&mut out); + self.quick_resolve_rec(&mut out, &mut Visited::new()); out } } @@ -1044,6 +1089,30 @@ mod tests { a.add_resolver(&checked); assert!(a.resolves_with(checked.as_ref())); } + + #[cfg_attr(not(emulated), traced_test)] + #[test] + /// Displaying a next-hop caught in a resolution loop terminates, and says which edge closes it. + /// + /// It used to walk the resolvers with an untracked `u8` depth, so a loop recursed until that + /// depth overflowed -- a panic raised from inside a `Display` impl that both the CLI and the + /// warning this module logs about resolution loops go through. + fn test_display_of_a_resolution_loop_terminates() { + let mut store = NhopStore::new(); + let a = store.add_nhop(&NhopKey::from_address("7.0.0.1")); + let b = store.add_nhop(&NhopKey::from_address("8.0.0.2")); + a.add_resolver(&b); + b.add_resolver(&a); + + let nhop = format!("{a}"); + assert!(nhop.contains("(LOOP)"), "loop not reported in {nhop}"); + + let whole_store = format!("{store}"); + assert!( + whole_store.contains("(LOOP)"), + "loop not reported in {whole_store}" + ); + } } #[cfg(test)] @@ -1054,95 +1123,71 @@ mod fibgroup_properties { use std::ops::Bound::Included; const MAX_NODES: u8 = 6; + const MAX_RESOLVERS: u8 = 2; - /// A next-hop graph, given as a topological order. + /// A next-hop graph, given as an adjacency list over node indices. /// - /// Node `i` may only resolve via nodes after it, so the graph is acyclic by construction. That - /// is a precondition rather than a simplification: `build_nhop_fibgroup_rec` has no loop guard - /// of its own, and neither does `resolves_with`. Acyclicity is maintained inductively by - /// `lazy_resolve`, which refuses an edge whose target already resolves via the source -- so a - /// generated cycle here would not find a bug, it would recurse until the stack ran out. See - /// `a_cycle_is_refused_before_it_is_added` for the other half. + /// **Cycles included**, self-loops among them. A routing loop is exactly a cycle here, and + /// both walks over the resolver graph have to survive one -- `resolves_with` because it is the + /// guard that keeps cycles out and so cannot presume its own success, and + /// `build_nhop_fibgroup_rec` because nothing in the types ties it to that guard. #[derive(Debug, Clone)] - struct Dag { - /// `shape[i]` are the offsets, relative to `i`, of the nodes `i` resolves via. - shape: Vec>, + struct Graph { + /// `edges[i]` are the nodes that `i` resolves via, in the order they were wired. + edges: Vec>, /// Whether node `i` knows an interface, and so needs no resolving. grounded: Vec, } - impl Dag { - /// The edges, as concrete (from, to) index pairs. - /// - /// Shared by the graph and the oracles below: the edge list is the *input*, so sharing it - /// keeps them describing the same graph. What the oracles must not share is the traversal - /// under test. - fn edges(&self) -> Vec<(usize, usize)> { - let mut edges = Vec::new(); - for (from, offsets) in self.shape.iter().enumerate() { - for offset in offsets { - let to = from + usize::from(*offset); - if to < self.shape.len() { - edges.push((from, to)); - } - } - } - edges - } - - /// Which nodes are reachable from `start`, itself included. A plain closure, computed - /// without asking any next-hop anything. + impl Graph { + /// Which nodes are reachable from `start`, itself included. A plain closure over the + /// adjacency list, computed without asking any next-hop anything. fn reachable_from(&self, start: usize) -> Vec { - let edges = self.edges(); - let mut seen = vec![false; self.shape.len()]; + let mut seen = vec![false; self.edges.len()]; let mut stack = vec![start]; while let Some(node) = stack.pop() { if std::mem::replace(&mut seen[node], true) { continue; } - for (from, to) in &edges { - if *from == node { - stack.push(*to); - } - } + stack.extend_from_slice(&self.edges[node]); } seen } } - /// Draws [`Dag`]s. + /// Draws [`Graph`]s. #[derive(Debug, Clone, Copy, Default)] struct Graphs; impl ValueGenerator for Graphs { - type Output = Dag; + type Output = Graph; - fn generate(&self, driver: &mut D) -> Option { + fn generate(&self, driver: &mut D) -> Option { let nodes = usize::from(driver.gen_u8(Included(&1), Included(&MAX_NODES))?); - let mut shape = Vec::with_capacity(nodes); + let last = u8::try_from(nodes - 1).ok()?; + let mut edges = Vec::with_capacity(nodes); let mut grounded = Vec::with_capacity(nodes); - for index in 0..nodes { - let behind = u8::try_from(nodes - index - 1).ok()?; - let count = driver.gen_u8(Included(&0), Included(&behind.min(2)))?; - let mut edges = Vec::new(); + for _ in 0..nodes { + let count = driver.gen_u8(Included(&0), Included(&MAX_RESOLVERS))?; + let mut resolvers = Vec::with_capacity(usize::from(count)); for _ in 0..count { - edges.push(driver.gen_u8(Included(&1), Included(&behind.max(1)))?); + resolvers.push(usize::from(driver.gen_u8(Included(&0), Included(&last))?)); } - shape.push(edges); + edges.push(resolvers); grounded.push(driver.produce::()?); } - Some(Dag { shape, grounded }) + Some(Graph { edges, grounded }) } } - // Build the graph in an `NhopStore`, returning the nodes in topological order. - fn realize(dag: &Dag) -> (NhopStore, Vec>) { + // Build the graph in an `NhopStore`, returning the next-hops by index. + fn realize(graph: &Graph) -> (NhopStore, Vec>) { let mut store = NhopStore::new(); - let nodes: Vec> = (0..dag.shape.len()) + let nodes: Vec> = (0..graph.edges.len()) .map(|index| { let raw = u8::try_from(index).unwrap_or_else(|_| unreachable!()); let mut key = NhopKey::from_address(&format!("10.0.0.{}", raw + 1)); - if dag.grounded[index] { + if graph.grounded[index] { key.ifindex = Some( InterfaceIndex::try_new(u32::from(raw) + 1) .unwrap_or_else(|_| unreachable!()), @@ -1152,40 +1197,51 @@ mod fibgroup_properties { }) .collect(); - for (from, to) in dag.edges() { - nodes[from].add_resolver(&nodes[to]); + for (from, resolvers) in graph.edges.iter().enumerate() { + for to in resolvers { + nodes[from].add_resolver(&nodes[*to]); + } } (store, nodes) } - // The oracle: every root-to-leaf path, concatenated, squashed, and kept if usable. + // The oracle: every *simple* root-to-leaf path, its next-hops' instructions concatenated, + // squashed, and kept if the forwarder could execute it. // - // Worked out from the graph directly rather than by walking the same recursion the code does. - fn expected(node: &Rc, prefix: &FibEntry, out: &mut Vec) { - let mut entry = prefix.clone(); - entry.extend_from_slice(&node.instructions.borrow().clone()); + // Enumerated over the generated adjacency list rather than by walking the recursion under + // test. "Simple" is the loop guard restated: a path that would revisit a node stops there and + // contributes nothing, because going round a loop is not forwarding. + fn expected( + graph: &Graph, + nodes: &[Rc], + from: usize, + path: &mut Vec, + prefix: &FibEntry, + out: &mut Vec, + ) { + if path.contains(&from) { + return; + } + path.push(from); - let resolvers: Vec> = node - .resolvers - .borrow() - .iter() - .filter_map(Weak::upgrade) - .collect(); + let mut entry = prefix.clone(); + entry.extend_from_slice(&nodes[from].instructions.borrow()); - if resolvers.is_empty() { + if graph.edges[from].is_empty() { // A next-hop with neither an interface nor a way to reach one contributes nothing. - if node.must_be_resolved() { - return; - } - entry.squash(); - if entry.is_valid() { - out.push(entry); + if !nodes[from].must_be_resolved() { + entry.squash(); + if entry.is_valid() { + out.push(entry); + } } } else { - for resolver in resolvers { - expected(&resolver, &entry, out); + for to in &graph.edges[from] { + expected(graph, nodes, *to, path, &entry, out); } } + + path.pop(); } /// A next-hop's fib group is one entry per usable resolution path, and never empty. @@ -1195,22 +1251,28 @@ mod fibgroup_properties { bolero::check!() .with_generator(Graphs) .cloned() - .for_each(|dag: Dag| { - let (_store, nodes) = realize(&dag); + .for_each(|graph: Graph| { + let (_store, nodes) = realize(&graph); for node in &nodes { node.build_nhop_instructions(&rstore); } - let root = &nodes[0]; let mut want = Vec::new(); - expected(root, &FibEntry::new(), &mut want); + expected( + &graph, + &nodes, + 0, + &mut Vec::new(), + &FibEntry::new(), + &mut want, + ); if want.is_empty() { // Nothing usable: the group carries a drop so packets are not misrouted. want.push(FibEntry::drop_fibentry()); } - let got = root.build_nhop_fibgroup(); - assert_eq!(got.entries(), &want, "for {dag:?}"); + let got = nodes[0].build_nhop_fibgroup(); + assert_eq!(got.entries(), &want, "for {graph:?}"); }); } @@ -1225,39 +1287,67 @@ mod fibgroup_properties { bolero::check!() .with_generator(Graphs) .cloned() - .for_each(|dag: Dag| { - let (_store, nodes) = realize(&dag); + .for_each(|graph: Graph| { + let (_store, nodes) = realize(&graph); for node in &nodes { node.build_nhop_instructions(&rstore); } let group = nodes[0].build_nhop_fibgroup(); - assert!(!group.is_empty(), "for {dag:?}"); + assert!(!group.is_empty(), "for {graph:?}"); for entry in group.iter() { - assert!(entry.is_valid(), "unusable entry {entry:?} for {dag:?}"); + assert!(entry.is_valid(), "unusable entry {entry:?} for {graph:?}"); } }); } + /// A next-hop every one of whose paths loops back gets a drop, not an infinite walk. + /// + /// The general case falls out of the two properties above -- they only terminate because the + /// walk does -- but a routing loop is the failure this guard exists for, so it is worth one + /// case that says so in as many words. + #[test] + fn a_next_hop_in_a_resolution_loop_drops() { + let rstore = RmacStore::new(); + let mut store = NhopStore::new(); + + // 7.0.0.1 -> 8.0.0.2 -> 9.0.0.3 -> 7.0.0.1, and no way out to an interface. + let a = store.add_nhop(&NhopKey::from_address("7.0.0.1")); + let b = store.add_nhop(&NhopKey::from_address("8.0.0.2")); + let c = store.add_nhop(&NhopKey::from_address("9.0.0.3")); + a.add_resolver(&b); + b.add_resolver(&c); + c.add_resolver(&a); + store.rebuild_nhop_instructions(&rstore); + + let group = a.build_nhop_fibgroup(); + assert_eq!( + group.entries(), + &vec![FibEntry::drop_fibentry()], + "a packet caught in a routing loop must be dropped" + ); + } + /// `resolves_with` answers reachability in the resolver graph. /// /// That is the whole of what the loop guard rests on: `lazy_resolve` refuses an edge from `a` /// to `r` exactly when `r.resolves_with(a)`, which is to say when `a` is already reachable - /// from `r` and the edge would close a cycle. Checked against a closure computed over the edge - /// list, which asks no next-hop anything. + /// from `r` and the edge would close a cycle. Checked against a closure computed over the + /// adjacency list, which asks no next-hop anything -- and, now that the graphs may contain + /// cycles, over graphs where `resolves_with` has to terminate on its own account. #[test] fn resolves_with_answers_reachability() { bolero::check!() .with_generator(Graphs) .cloned() - .for_each(|dag: Dag| { - let (_store, nodes) = realize(&dag); + .for_each(|graph: Graph| { + let (_store, nodes) = realize(&graph); for (from, node) in nodes.iter().enumerate() { - let reachable = dag.reachable_from(from); + let reachable = graph.reachable_from(from); for (to, other) in nodes.iter().enumerate() { assert_eq!( node.resolves_with(other), reachable[to], - "{from} -> {to}, for {dag:?}" + "{from} -> {to}, for {graph:?}" ); } } @@ -1270,8 +1360,8 @@ mod fibgroup_properties { bolero::check!() .with_generator(Graphs) .cloned() - .for_each(|dag: Dag| { - let (_store, nodes) = realize(&dag); + .for_each(|graph: Graph| { + let (_store, nodes) = realize(&graph); for node in &nodes { assert!(node.resolves_with(node)); } diff --git a/routing/src/rib/rib2fib.rs b/routing/src/rib/rib2fib.rs index 99ae73cd0f..738c921062 100644 --- a/routing/src/rib/rib2fib.rs +++ b/routing/src/rib/rib2fib.rs @@ -9,7 +9,7 @@ use tracing::{debug, trace, warn}; use crate::evpn::RmacStore; use crate::fib::fibobjects::{EgressObject, FibEntry, FibGroup, PktInstruction}; use crate::rib::encapsulation::{Encapsulation, VxlanEncapsulation}; -use crate::rib::nexthop::{FwAction, Nhop}; +use crate::rib::nexthop::{FwAction, Nhop, Visited}; use crate::rib::vrf::RouteOrigin; use std::rc::Weak; @@ -103,8 +103,44 @@ impl Nhop { ////////////////////////////////////////////////////////////////////// /// Recursive helper to build [`FibGroup`] for a next-hop. We accumulate /// a next-hop's packet instructions with those of its resolvers. + /// + /// `path` holds the next-hops between the root of the walk and this one. A next-hop that turns + /// up on its own resolution path closes a routing loop: following it would recurse until the + /// stack ran out, so we stop there and contribute nothing. If that leaves no usable path at + /// all, `build_nhop_fibgroup` injects a drop, which is what a packet caught in a routing loop + /// should meet anyway. + /// + /// This makes the walk safe on any graph rather than only on the acyclic ones that + /// `Nhop::resolves_with` lets `lazy_resolve` build. The two guards are deliberately + /// independent: nothing in the types ties this recursion to the one that used to be its only + /// protection, and a caller wiring resolvers by another route would lose it silently. ////////////////////////////////////////////////////////////////////// - fn build_nhop_fibgroup_rec(&self, fibgroup: &mut FibGroup, mut entry: FibEntry) { + fn build_nhop_fibgroup_rec( + &self, + fibgroup: &mut FibGroup, + entry: FibEntry, + path: &mut Visited, + ) { + if path.contains(&self.id()) { + warn!("Resolution loop at next-hop {self}: will not use this path"); + return; + } + path.push(self.id()); + self.build_nhop_fibgroup_visit(fibgroup, entry, path); + path.pop(); + } + + ////////////////////////////////////////////////////////////////////// + /// The body of [`Nhop::build_nhop_fibgroup_rec`], for a next-hop known not to be on its own + /// resolution path already. Split out so that the push and the pop of `path` sit next to each + /// other and no early return here can leave the path unbalanced. + ////////////////////////////////////////////////////////////////////// + fn build_nhop_fibgroup_visit( + &self, + fibgroup: &mut FibGroup, + mut entry: FibEntry, + path: &mut Visited, + ) { // add the instructions for a next-hop to the entry let instructions = self.instructions.borrow().clone(); entry.extend_from_slice(&instructions); @@ -136,7 +172,7 @@ impl Nhop { } } else { for resolver in resolvers.iter().filter_map(Weak::upgrade) { - resolver.build_nhop_fibgroup_rec(fibgroup, entry.clone()); + resolver.build_nhop_fibgroup_rec(fibgroup, entry.clone(), path); } } } @@ -149,7 +185,7 @@ impl Nhop { ////////////////////////////////////////////////////////////////////// pub(crate) fn build_nhop_fibgroup(&self) -> FibGroup { let mut fibgroup = FibGroup::new(); - self.build_nhop_fibgroup_rec(&mut fibgroup, FibEntry::new()); + self.build_nhop_fibgroup_rec(&mut fibgroup, FibEntry::new(), &mut Visited::new()); if fibgroup.is_empty() { warn!("Next-hop {self} has empty fibgroup: will add DROP FibEntry"); fibgroup.add(FibEntry::drop_fibentry()); From 2e2d6b4e928cc5f07de4008a28085fa65b3f6dc8 Mon Sep 17 00:00:00 2001 From: Daniel Noland Date: Fri, 7 Aug 2026 13:17:55 -0600 Subject: [PATCH 21/33] test(routing): Model-check a fib against its change log Builds a generator for a populated `Fib` and four properties over it. The generator is the one a pipeline harness will want: it reaches a fib the way production does, by pushing a sequence of `FibChange`s through a `FibWriter`, rather than by reaching into the tries. Alongside it runs a model -- two `BTreeMap`s, which is the fib with its tries, its group store's reference counting and its `UnsafeCell` sharing all taken away. What the model has to get right is not the data structure but which changes the fib *refuses*, and that turns out to be the valuable part. Four separate decisions, in three files, none of them stated where the next one can see it: - `FibGroupStore::add_mod_group` refuses a group with no entries - `FibGroupStore::del` keeps a group any route still names, by refcount - `FibWriter::add_fibroute` refuses a route with no next-hop keys, and `FibRoute::from_nhopkeys` refuses one naming an unregistered group - `Fib::del_fibroute` resets a root route to drop instead of deleting it, and purges unreferenced groups afterwards Together those are what keep `Fib::lpm` from reaching its `unreachable!()` and `Fib::lpm_entry_prefix` from reaching its outright `panic!` -- both on the forwarding path, for every packet that arrives. So the second property says that in as many words: every route a lookup lands on has at least one entry to execute, and the index arithmetic that picks among them is total over the range it is given. The prefix pool is nested so a longest match has something to be longer than, and holds both roots so that deleting one is reachable. The next-hop key pool is deliberately small, because the behaviour worth exercising is the collisions: a route pinning a group against deletion, a registration mutating a group two routes share. Verified by breaking each of the four decisions in turn. Letting the store accept an empty group fails in one change -- registering an empty group over the drop key empties the route both roots point at. Making the default route deletable fails three of the four properties. Ignoring the refcount in `del`, and dropping the purge after a route deletion, each fail the model. Five hundred thousand cases pass. Co-Authored-By: Claude Opus 5 (1M context) Signed-off-by: Daniel Noland --- routing/src/fib/fibtype.rs | 409 +++++++++++++++++++++++++++++++++++++ 1 file changed, 409 insertions(+) diff --git a/routing/src/fib/fibtype.rs b/routing/src/fib/fibtype.rs index ea368bb36b..f2d597c40f 100644 --- a/routing/src/fib/fibtype.rs +++ b/routing/src/fib/fibtype.rs @@ -504,3 +504,412 @@ impl FibReaderFactory { FibReader(self.0.handle()) } } + +/// Model-based properties over a [`Fib`] driven through its writer. +/// +/// The generator here is the one the pipeline harness will want: it produces a *populated* fib, +/// reached the way production reaches one -- a sequence of `FibChange`s through a `FibWriter` -- +/// rather than by reaching into the tries. Everything the fib is asked afterwards is checked +/// against a model kept alongside it. +#[cfg(test)] +mod fib_properties { + use super::*; + use crate::fib::fibgroupstore::tests::{build_fib_entry_egress, build_fibgroup}; + use bolero::{Driver, ValueGenerator}; + use std::collections::{BTreeMap, BTreeSet}; + use std::ops::Bound::Included; + use std::str::FromStr; + + const NUM_NHOPS: u8 = 4; + const NUM_PREFIXES: u8 = 8; + const NUM_ENTRIES: u8 = 4; + const MAX_CHANGES: u8 = 12; + const MAX_KEYS_PER_ROUTE: u8 = 3; + const MAX_ENTRIES_PER_GROUP: u8 = 3; + + /// The drop next-hop, first in [`nhop_keys`]. The store creates its group at construction and + /// refuses to delete it. + const DROP_KEY: usize = 0; + /// `0.0.0.0/0` and `::/0`, at these indices in [`prefixes`]. A fib always carries a route for + /// both: `Fib::lpm` has no answer for an address nothing covers, and says so with an + /// `unreachable!()` on the forwarding path. + const ROOT_V4: usize = 0; + const ROOT_V6: usize = 5; + + /// The next-hop keys a generated fib may mention. + /// + /// A small pool on purpose. What is worth exercising is the collisions -- a route pinning a + /// group against deletion, a registration mutating a group two routes share -- and collisions + /// need a small pool to happen often. The drop key is in it because the rib does register + /// groups under it, and because the store treats it as permanent. + fn nhop_keys() -> Vec { + vec![ + NhopKey::with_drop(), + NhopKey::with_addr_ifindex("10.0.0.1", 1), + NhopKey::with_addr_ifindex("10.0.0.2", 2), + NhopKey::with_ifindex(3), + ] + } + + /// The prefixes a generated fib may carry routes for. Nested on purpose, so a longest match + /// has something to be longer than, and both roots so that deleting one is reachable. + fn prefixes() -> Vec { + [ + "0.0.0.0/0", + "10.0.0.0/8", + "10.1.0.0/16", + "10.1.2.0/24", + "10.1.2.3/32", + "::/0", + "2001:db8::/32", + "2001:db8:1::/48", + ] + .iter() + .map(|p| Prefix::from_str(p).unwrap_or_else(|_| unreachable!())) + .collect() + } + + /// Addresses to look up: one inside each level of the nesting, and one outside all of them in + /// each family, so a lookup that falls through to the root is exercised too. + fn probes() -> Vec { + [ + "9.9.9.9", + "10.9.9.9", + "10.1.9.9", + "10.1.2.9", + "10.1.2.3", + "2000::1", + "2001:db8::1", + "2001:db8:1::1", + ] + .iter() + .map(|a| IpAddr::from_str(a).unwrap_or_else(|_| unreachable!())) + .collect() + } + + /// The entries a generated group may be built from. + fn entry_pool() -> Vec { + (1..=u32::from(NUM_ENTRIES)) + .map(|i| build_fib_entry_egress(i, &format!("10.0.9.{i}"), &format!("eth{i}"))) + .collect() + } + + /// One change, as the writer API exposes them, over indices into the pools above. + #[derive(Debug, Clone)] + enum Change { + RegisterGroup { key: usize, entries: Vec }, + UnregisterGroup { key: usize }, + AddRoute { prefix: usize, keys: Vec }, + DelRoute { prefix: usize }, + } + + /// Draws sequences of [`Change`]s. + #[derive(Debug, Clone, Copy, Default)] + struct ChangeSequences; + + fn index(driver: &mut D, count: u8) -> Option { + driver + .gen_u8(Included(&0), Included(&(count - 1))) + .map(usize::from) + } + + fn indices(driver: &mut D, count: u8, most: u8) -> Option> { + // Deliberately able to draw none: an empty group and a route with no next-hops are both + // things the fib is supposed to refuse, and refusing is behaviour worth checking. + let len = driver.gen_u8(Included(&0), Included(&most))?; + let mut out = Vec::with_capacity(usize::from(len)); + for _ in 0..len { + out.push(index(driver, count)?); + } + Some(out) + } + + impl ValueGenerator for ChangeSequences { + type Output = Vec; + + fn generate(&self, driver: &mut D) -> Option> { + let len = driver.gen_u8(Included(&0), Included(&MAX_CHANGES))?; + let mut out = Vec::with_capacity(usize::from(len)); + for _ in 0..len { + let change = match driver.gen_u8(Included(&0), Included(&3))? { + 0 => Change::RegisterGroup { + key: index(driver, NUM_NHOPS)?, + entries: indices(driver, NUM_ENTRIES, MAX_ENTRIES_PER_GROUP)?, + }, + 1 => Change::UnregisterGroup { + key: index(driver, NUM_NHOPS)?, + }, + 2 => Change::AddRoute { + prefix: index(driver, NUM_PREFIXES)?, + keys: indices(driver, NUM_NHOPS, MAX_KEYS_PER_ROUTE)?, + }, + _ => Change::DelRoute { + prefix: index(driver, NUM_PREFIXES)?, + }, + }; + out.push(change); + } + Some(out) + } + } + + /// What the fib should hold, tracked beside it. + /// + /// Not a reimplementation: two maps, which is the fib with the tries, the group store's + /// reference counting and its `UnsafeCell` sharing all taken away. What the model does have to + /// get right is which changes the fib *refuses*, and that is the part worth writing down -- + /// three guards in three files decide it, and none of them says so where the next one can see. + #[derive(Debug, Clone)] + struct Model { + /// next-hop key index -> the entries of its group. + groups: BTreeMap>, + /// prefix index -> the next-hop keys of its route, in order. + routes: BTreeMap>, + } + + impl Model { + /// A fresh fib: a drop group, and a route to it for each root. + fn new() -> Self { + Self { + groups: BTreeMap::from([(DROP_KEY, vec![FibEntry::drop_fibentry()])]), + routes: BTreeMap::from([(ROOT_V4, vec![DROP_KEY]), (ROOT_V6, vec![DROP_KEY])]), + } + } + + fn referenced(&self, key: usize) -> bool { + self.routes.values().any(|keys| keys.contains(&key)) + } + + /// Drop every group no route points at. The store does this by reference count; here the + /// routes are the reference count. + fn purge(&mut self) { + let referenced: BTreeSet = self + .routes + .values() + .flatten() + .copied() + .collect::>(); + self.groups + .retain(|key, _| *key == DROP_KEY || referenced.contains(key)); + } + + fn apply(&mut self, change: &Change, pool: &[FibEntry]) { + match change { + Change::RegisterGroup { key, entries } => { + // a group with no entries is refused: a route reaching one would leave the + // forwarder with nothing to execute + if entries.is_empty() { + return; + } + let entries = entries.iter().map(|i| pool[*i].clone()).collect(); + self.groups.insert(*key, entries); + } + Change::UnregisterGroup { key } => { + // the drop group is permanent, and a group a route still names is pinned + if *key == DROP_KEY || self.referenced(*key) { + return; + } + self.groups.remove(key); + } + Change::AddRoute { prefix, keys } => { + // a route with no next-hops is refused, and so is one naming a group that was + // never registered -- whole, not in part + if keys.is_empty() || keys.iter().any(|k| !self.groups.contains_key(k)) { + return; + } + // note: no purge here. Replacing a route releases the old route's hold on its + // groups, but the fib leaves them in the store until something purges. + self.routes.insert(*prefix, keys.clone()); + } + Change::DelRoute { prefix } => { + // a root route is not deleted but reset to drop, so that a lookup always has + // an answer + let removed = if *prefix == ROOT_V4 || *prefix == ROOT_V6 { + self.routes.insert(*prefix, vec![DROP_KEY]) + } else { + self.routes.remove(prefix) + }; + if removed.is_some() { + self.purge(); + } + } + } + } + + /// The longest prefix carrying a route that covers `addr`. + fn lpm(&self, addr: &IpAddr, prefixes: &[Prefix]) -> Option { + self.routes + .keys() + .copied() + .filter(|i| prefixes[*i].covers_addr(addr)) + .max_by_key(|i| prefixes[*i].length()) + } + + /// The entries a route offers: its groups' entries, concatenated in next-hop key order. + fn entries_for(&self, prefix: usize) -> Vec { + self.routes[&prefix] + .iter() + .flat_map(|key| self.groups[key].iter().cloned()) + .collect() + } + } + + fn apply_to_fib(writer: &mut FibWriter, change: &Change, pool: &[FibEntry], keys: &[NhopKey]) { + let prefixes = prefixes(); + match change { + Change::RegisterGroup { key, entries } => { + let entries: Vec = entries.iter().map(|i| pool[*i].clone()).collect(); + writer.register_fibgroup(&keys[*key], &build_fibgroup(&entries), true); + } + Change::UnregisterGroup { key } => writer.unregister_fibgroup(&keys[*key], true), + Change::AddRoute { + prefix, + keys: route, + } => { + let route = route.iter().map(|k| keys[*k].clone()).collect(); + writer.add_fibroute(prefixes[*prefix], route, true); + } + Change::DelRoute { prefix } => writer.del_fibroute(prefixes[*prefix]), + } + } + + /// The pools and the constants that index them agree. + #[test] + fn the_pools_are_the_size_the_generator_thinks() { + assert_eq!(nhop_keys().len(), usize::from(NUM_NHOPS)); + assert_eq!(prefixes().len(), usize::from(NUM_PREFIXES)); + assert_eq!(entry_pool().len(), usize::from(NUM_ENTRIES)); + assert_eq!(nhop_keys()[DROP_KEY], NhopKey::with_drop()); + assert_eq!(prefixes()[ROOT_V4], Prefix::root_v4()); + assert_eq!(prefixes()[ROOT_V6], Prefix::root_v6()); + for probe in probes() { + assert!( + prefixes().iter().any(|p| p.covers_addr(&probe)), + "probe {probe} is covered by no prefix, not even a root" + ); + } + } + + /// After any sequence of changes, a fib answers every lookup the way the model says. + /// + /// This is the whole of the fib's read path against an independent account of its contents: + /// which prefix the lookup lands on, and which entries the route there offers. It covers the + /// group store's sharing too, since registering a group under a key two routes name has to + /// change what both of them offer. + #[test] + fn a_fib_answers_lookups_the_way_the_model_says() { + let keys = nhop_keys(); + let prefixes = prefixes(); + let probes = probes(); + let pool = entry_pool(); + + bolero::check!() + .with_generator(ChangeSequences) + .cloned() + .for_each(|changes: Vec| { + let (mut writer, _reader) = FibWriter::new(FibKey::from_vrfid(1)); + let mut model = Model::new(); + + for (step, change) in changes.iter().enumerate() { + apply_to_fib(&mut writer, change, &pool, &keys); + model.apply(change, &pool); + + let fib = writer.enter().unwrap_or_else(|| unreachable!()); + let at = || format!("at step {step} of {changes:?}"); + + assert_eq!(fib.len_groups(), model.groups.len(), "{}", at()); + + for probe in &probes { + let want = model + .lpm(probe, &prefixes) + .unwrap_or_else(|| panic!("model has no route for {probe} {}", at())); + + let (hit, route) = fib.lpm_with_prefix(probe); + assert_eq!(hit, prefixes[want], "for {probe} {}", at()); + + let got: Vec = route + .iter() + .flat_map(|group| group.entries().iter().cloned()) + .collect(); + assert_eq!(got, model.entries_for(want), "for {probe} {}", at()); + } + } + }); + } + + /// A lookup always lands on a route with at least one entry to execute. + /// + /// `Fib::lpm_entry_prefix` panics outright on a route with none -- "hit route without + /// fibgroups/entries. This is a bug." -- on the forwarding path, for every packet that reaches + /// it. The invariant that saves it is held jointly by three guards in three files: the store + /// refuses an empty group, the writer refuses a route with no next-hop keys, and + /// `FibRoute::from_nhopkeys` refuses a route naming a group that is not registered. Nothing + /// states the invariant they add up to, so state it here. + #[test] + fn every_route_a_lookup_reaches_has_an_entry_to_execute() { + let keys = nhop_keys(); + let probes = probes(); + let pool = entry_pool(); + + bolero::check!() + .with_generator(ChangeSequences) + .cloned() + .for_each(|changes: Vec| { + let (mut writer, _reader) = FibWriter::new(FibKey::from_vrfid(1)); + for change in &changes { + apply_to_fib(&mut writer, change, &pool, &keys); + } + + let fib = writer.enter().unwrap_or_else(|| unreachable!()); + for probe in &probes { + let (_, route) = fib.lpm_with_prefix(probe); + assert!(route.len() > 0, "no entry for {probe} after {changes:?}"); + // and the index arithmetic that picks among them is total over that range + for index in 0..route.len() { + let _ = route.get_fibentry(index); + } + } + }); + } + + /// A reader sees what the writer sees, once every change has been published. + #[test] + fn a_reader_and_a_writer_agree_after_publishing() { + let keys = nhop_keys(); + let probes = probes(); + let pool = entry_pool(); + + bolero::check!() + .with_generator(ChangeSequences) + .cloned() + .for_each(|changes: Vec| { + let (mut writer, reader) = FibWriter::new(FibKey::from_vrfid(1)); + for change in &changes { + apply_to_fib(&mut writer, change, &pool, &keys); + } + + for probe in &probes { + let (want_prefix, want_entries) = { + let fib = writer.enter().unwrap_or_else(|| unreachable!()); + let (prefix, route) = fib.lpm_with_prefix(probe); + let entries: Vec = route + .iter() + .flat_map(|group| group.entries().iter().cloned()) + .collect(); + (prefix, entries) + }; + + let (got_prefix, route) = reader + .lpm_route_with_prefix(*probe) + .unwrap_or_else(|| unreachable!()); + let got_entries: Vec = route + .iter() + .flat_map(|group| group.entries().iter().cloned()) + .collect(); + + assert_eq!(got_prefix, want_prefix, "for {probe} after {changes:?}"); + assert_eq!(got_entries, want_entries, "for {probe} after {changes:?}"); + } + }); + } +} From 49cf4fda10e4349533fbce23c1e5ec4c092c5eca Mon Sep 17 00:00:00 2001 From: Daniel Noland Date: Fri, 7 Aug 2026 13:18:08 -0600 Subject: [PATCH 22/33] fix(routing): Drop a fib's vni alias when the fib goes A fib is reachable from the `FibTable` by two keys: its own `FibKey::Id`, and optionally a `FibKey::Vni` aliasing the same entry. `FibTable::del_fib` removed only the key it was given, so dropping the alias was a matter of the caller passing the vni the fib happened to be registered under -- which `FibTableWriter::del_fib` duly took as an argument, and passed on. That works only as long as every caller's idea of the vni matches the table's. `VrfTable` does keep them in step: `set_vni` calls `unset_vni` first, so a fib is never aliased under two vnis at once, and `remove_vrf` passes `vrf.vni`. So this was latent rather than live. It is the same shape as the next-hop resolution loop, though: an invariant held by discipline at a distance, with nothing in the types holding it, and one careless caller away from a `FibKey::Vni` that reaches a fib whose writer has been destroyed. The table does not need to be told. Each `FibTableEntry` records the identity of the fib it points at, and an alias shares the entry, so `del_fib` can find its own aliases: self.entries.retain(|_, entry| entry.id != id); With that, the vni argument to `FibTableWriter::del_fib` carries no information the table lacks, so it is gone -- which is the point. Restoring the invariant while leaving the argument in place would have left the trap. Found by a model-based property over the table: every key it holds reaches a live fib, and reaches it under its own identity. The counterexample was two changes long -- add a fib with a vni, delete it without one -- and the property fails again if `del_fib` goes back to removing a single key. The second half of that property is worth stating separately, because nothing else checks it: the thread-local read-handle cache keys on the identity the table reports for a key rather than on the key asked for, so an alias reporting the wrong identity would have two threads caching handles to different fibs under one name. Co-Authored-By: Claude Opus 5 (1M context) Signed-off-by: Daniel Noland --- routing/src/fib/fibtable.rs | 233 ++++++++++++++++++++++++++++++++++-- routing/src/fib/test.rs | 4 +- routing/src/rib/vrftable.rs | 2 +- 3 files changed, 229 insertions(+), 10 deletions(-) diff --git a/routing/src/fib/fibtable.rs b/routing/src/fib/fibtable.rs index 5fbab2a74a..ef62d9c024 100644 --- a/routing/src/fib/fibtable.rs +++ b/routing/src/fib/fibtable.rs @@ -39,9 +39,14 @@ impl FibTable { self.entries.insert(id, entry); } /// Delete a `Fib`, by unregistering a `FibReaderFactory` for it + /// + /// Every key that reaches the fib goes, not just its own: a fib registered under a [`Vni`] is + /// reachable by that alias too, and an alias must not outlive the fib it names. Each entry + /// records the identity of the fib it points at, so the table can find its own aliases rather + /// than relying on the caller to remember which vni a fib was registered under. fn del_fib(&mut self, id: FibKey) { info!("Unregistering Fib with id {id} from the FibTable"); - self.entries.remove(&id); + self.entries.retain(|_, entry| entry.id != id); } /// Register an existing `Fib` with a given [`Vni`]. /// This allows looking up a Fib (`FibReaderFactory`) from a [`Vni`] @@ -144,12 +149,14 @@ impl FibTableWriter { self.0.append(FibTableChange::UnRegisterVni(vni)); self.0.publish(); } - pub fn del_fib(&mut self, vrfid: VrfId, vni: Option) { - let fibid = FibKey::from_vrfid(vrfid); - self.0.append(FibTableChange::Del(fibid)); - if let Some(vni) = vni { - self.0.append(FibTableChange::UnRegisterVni(vni)); - } + /// Remove the fib for `vrfid`, and with it every key that reached it. + /// + /// This used to take the fib's [`Vni`] so as to drop that alias as well, which made a leaked + /// alias a matter of the caller passing the right thing. [`FibTable::del_fib`] now finds the + /// aliases itself. + pub fn del_fib(&mut self, vrfid: VrfId) { + self.0 + .append(FibTableChange::Del(FibKey::from_vrfid(vrfid))); self.0.publish(); } } @@ -234,3 +241,215 @@ impl FibTableReader { Ok(FibReader::rc_from_rc_rhandle(rhandle)) } } + +/// Model-based properties over a [`FibTable`]. +/// +/// The table is a map, so most of it is uninteresting. The part that is not is the **vni alias**: a +/// fib is reachable both by its own [`FibKey::Id`] and, optionally, by a [`FibKey::Vni`] pointing +/// at the same entry. Nothing in the table ties the two together -- `del_fib` removes the alias +/// only because the caller passes the vni it was registered with -- so an alias outliving its fib +/// is the failure worth generating for. +#[cfg(test)] +mod fibtable_properties { + use super::*; + use crate::fib::fibtype::FibWriter; + use bolero::{Driver, ValueGenerator}; + use std::ops::Bound::Included; + + const NUM_VRFS: u8 = 3; + const NUM_VNIS: u8 = 2; + const MAX_CHANGES: u8 = 10; + + fn vrf_ids() -> Vec { + (0..u32::from(NUM_VRFS)).collect() + } + + fn vnis() -> Vec { + (1..=u32::from(NUM_VNIS)) + .map(|i| Vni::new_checked(100 * i).unwrap_or_else(|_| unreachable!())) + .collect() + } + + /// Every key a generated table could be looked up by. + fn keys() -> Vec { + vrf_ids() + .into_iter() + .map(FibKey::from_vrfid) + .chain(vnis().into_iter().map(FibKey::from_vni)) + .collect() + } + + /// One change, as [`FibTableWriter`] exposes them, over indices into the pools above. + #[derive(Debug, Clone)] + enum Change { + AddFib { vrf: usize, vni: Option }, + RegisterByVni { vrf: usize, vni: usize }, + UnregisterVni { vni: usize }, + DelFib { vrf: usize }, + } + + /// Draws sequences of [`Change`]s. + #[derive(Debug, Clone, Copy, Default)] + struct ChangeSequences; + + fn index(driver: &mut D, count: u8) -> Option { + driver + .gen_u8(Included(&0), Included(&(count - 1))) + .map(usize::from) + } + + impl ValueGenerator for ChangeSequences { + type Output = Vec; + + fn generate(&self, driver: &mut D) -> Option> { + let len = driver.gen_u8(Included(&0), Included(&MAX_CHANGES))?; + let mut out = Vec::with_capacity(usize::from(len)); + for _ in 0..len { + let change = match driver.gen_u8(Included(&0), Included(&3))? { + 0 => { + // A fib created with no vni and aliased later, or never, is the ordinary + // case, so it is worth drawing. Drawn as one index over `NUM_VNIS + 1` + // with the last meaning "none": a helper returning `Option>` + // cannot tell "no vni" from the driver running out of input. + let vrf = index(driver, NUM_VRFS)?; + let drawn = index(driver, NUM_VNIS + 1)?; + Change::AddFib { + vrf, + vni: (drawn < usize::from(NUM_VNIS)).then_some(drawn), + } + } + 1 => Change::RegisterByVni { + vrf: index(driver, NUM_VRFS)?, + vni: index(driver, NUM_VNIS)?, + }, + 2 => Change::UnregisterVni { + vni: index(driver, NUM_VNIS)?, + }, + _ => Change::DelFib { + vrf: index(driver, NUM_VRFS)?, + }, + }; + out.push(change); + } + Some(out) + } + } + + /// Which fib each key should reach, by vrf id. The table stripped of its left-right wrapping, + /// its `Arc` sharing and its reader factories. + type Model = BTreeMap; + + /// The writers behind a generated table, which the harness has to keep alive: a `FibReader` + /// whose `FibWriter` is gone cannot be entered, and that would look like an alias fault. + struct Fibs { + live: BTreeMap, + /// Writers displaced by a second `add_fib` for the same vrf. Nothing in the table points at + /// them any more, but they are not destroyed either, so they are parked rather than + /// dropped -- dropping one is not what production does on a replacement. + retired: Vec, + } + + fn apply(table: &mut FibTableWriter, fibs: &mut Fibs, model: &mut Model, change: &Change) { + let vrfs = vrf_ids(); + let all_vnis = vnis(); + match change { + Change::AddFib { vrf, vni } => { + let vrf = vrfs[*vrf]; + let vni = vni.map(|i| all_vnis[i]); + let writer = table.add_fib(vrf, vni); + if let Some(displaced) = fibs.live.insert(vrf, writer) { + fibs.retired.push(displaced); + } + model.insert(FibKey::from_vrfid(vrf), vrf); + if let Some(vni) = vni { + model.insert(FibKey::from_vni(vni), vrf); + } + } + Change::RegisterByVni { vrf, vni } => { + let vrf = vrfs[*vrf]; + let vni = all_vnis[*vni]; + table.register_fib_by_vni(vrf, vni); + // the table refuses to alias a fib it does not hold + if model.contains_key(&FibKey::from_vrfid(vrf)) { + model.insert(FibKey::from_vni(vni), vrf); + } + } + Change::UnregisterVni { vni } => { + let vni = all_vnis[*vni]; + table.unregister_vni(vni); + model.remove(&FibKey::from_vni(vni)); + } + Change::DelFib { vrf } => { + let vrf = vrfs[*vrf]; + table.del_fib(vrf); + // every key that reached this fib goes, alias included + model.retain(|_, named| *named != vrf); + // production destroys the fib once the table no longer names it, which is what + // makes a leaked alias observable: it would hand out a reader that cannot be + // entered + if let Some(writer) = fibs.live.remove(&vrf) { + writer.destroy(); + } + } + } + } + + /// The pools and the constants that index them agree. + #[test] + fn the_pools_are_the_size_the_generator_thinks() { + assert_eq!(vrf_ids().len(), usize::from(NUM_VRFS)); + assert_eq!(vnis().len(), usize::from(NUM_VNIS)); + assert_eq!(keys().len(), usize::from(NUM_VRFS + NUM_VNIS)); + } + + /// Every key a fib table holds reaches a live fib, and reaches it under its own identity. + /// + /// Two things at once, and the second is the point. A `FibKey::Vni` is an alias for a + /// `FibKey::Id`, and the thread-local read-handle cache keys on the identity the table reports + /// for a key, not on the key asked for -- so an alias reporting the wrong identity would have + /// two threads caching handles to different fibs under one name. Nothing else checks that the + /// alias and the entry it aliases stay in step. + #[test] + fn every_key_in_a_fib_table_reaches_the_fib_it_names() { + let keys = keys(); + + bolero::check!() + .with_generator(ChangeSequences) + .cloned() + .for_each(|changes: Vec| { + let (mut table, _reader) = FibTableWriter::new(); + let mut fibs = Fibs { + live: BTreeMap::new(), + retired: Vec::new(), + }; + let mut model = Model::new(); + + for (step, change) in changes.iter().enumerate() { + apply(&mut table, &mut fibs, &mut model, change); + + let at = || format!("at step {step} of {changes:?}"); + let held = table.enter().unwrap_or_else(|| unreachable!()); + + assert_eq!(held.len(), model.len(), "{}", at()); + + for key in &keys { + let Some(reader) = held.get_fib(*key) else { + assert!(!model.contains_key(key), "{key} missing {}", at()); + continue; + }; + let want = *model + .get(key) + .unwrap_or_else(|| panic!("{key} unexpected {}", at())); + + assert!(reader.is_valid(), "{key} reaches a dead fib {}", at()); + assert_eq!( + reader.get_id(), + Some(FibKey::from_vrfid(want)), + "{key} reaches the wrong fib {}", + at() + ); + } + } + }); + } +} diff --git a/routing/src/fib/test.rs b/routing/src/fib/test.rs index bac97ec8ca..e2e9ec8ca1 100644 --- a/routing/src/fib/test.rs +++ b/routing/src/fib/test.rs @@ -367,7 +367,7 @@ mod tests { } if updates.is_multiple_of(50) && fibw.is_some() { - fibtw.del_fib(vrfid, None); + fibtw.del_fib(vrfid); if let Some(fib) = fibw.take() { // fib is destroyed here fib.destroy(); @@ -519,7 +519,7 @@ mod concurrency_tests { loop { let fibw = fibtw.add_fib(vrfid, None); thread::sleep(Duration::from_millis(5)); - fibtw.del_fib(vrfid, None); + fibtw.del_fib(vrfid); fibw.destroy(); iterations += 1; if iterations == MAX_ITERATIONS { diff --git a/routing/src/rib/vrftable.rs b/routing/src/rib/vrftable.rs index d3035239ec..e12e6ddbc6 100644 --- a/routing/src/rib/vrftable.rs +++ b/routing/src/rib/vrftable.rs @@ -189,7 +189,7 @@ impl VrfTable { // delete the corresponding fib if let Some(fibw) = vrf.fibw.take() { debug!("Deleting Fib for vrf {vrfid} from the FibTable"); - self.fibtablew.del_fib(vrfid, vrf.vni); + self.fibtablew.del_fib(vrfid); fibw.destroy(); } From 4f1aba09d33e01a0e0102304b101d9e483409d2e Mon Sep 17 00:00:00 2001 From: Daniel Noland Date: Fri, 7 Aug 2026 13:28:17 -0600 Subject: [PATCH 23/33] fix(routing): Refuse to remove the default vrf `VrfTable::remove_vrf` took any `VrfId` and removed it, the default vrf included. `get_default_vrf` and `get_default_vrf_mut` then reach their `unreachable!()` -- they treat the default vrf's existence as given, and it is given everywhere except here. `Vrf::set_status` already says the default vrf cannot be deleted, and enforces half of it: it refuses to move the default vrf out of `Active`, so `remove_deleted_vrfs` and `remove_deleting_vrfs` never pick it up. Both production callers of `remove_vrf` sit behind that same `can_be_deleted()` check, and `Cpi`'s delete branches on `DEFAULT_VRFID` before it gets there. So, as with the fib alias, this was latent rather than live -- an invariant held by discipline at three call sites, stated in a comment on a fourth function, and enforced nowhere a caller has to look. Found by a model-based property over the vrf table, on a one-change counterexample. That property is the wider point of this commit. The vrf table is where four key spaces have to agree -- `by_id`, `by_vni`, each `Vrf`'s own `vni` field, and the fib table's `FibKey::Id` and `FibKey::Vni` spaces -- and nothing holds them together but its methods doing the right number of things in the right order. The model is one map, from vrf id to the vni and status it carries, and all four views are checked against it and so against each other: - `by_id` holds what the model says, each vrf carrying what the model says - `by_vni` is exactly the inverse of the vnis the vrfs carry, with no stale entry left by a removal and none missing after a vni was set - the fib table holds a fib per vrf, aliased by vni where there is one, and every key reaches a live fib under the right identity - the default vrf is present and active - `check_vni`, the in-tree half of this oracle, agrees Verified by breaking five separate updates: dropping the `by_vni` removal from `unset_vni`, the fib aliasing from `set_vni`, the `unset_vni` call that makes `set_vni` release the vrf's previous vni, the `by_vni` removal from `remove_vrf`, and the new default-vrf guard. Each fails the property, at four different assertions. Two hundred thousand cases pass. `VrfStatus` gains a derived `Debug` so a mismatch names the status it found. Co-Authored-By: Claude Opus 5 (1M context) Signed-off-by: Daniel Noland --- routing/src/rib/vrf.rs | 2 +- routing/src/rib/vrftable.rs | 327 ++++++++++++++++++++++++++++++++++++ 2 files changed, 328 insertions(+), 1 deletion(-) diff --git a/routing/src/rib/vrf.rs b/routing/src/rib/vrf.rs index 6739a9d915..2ad0ca4899 100644 --- a/routing/src/rib/vrf.rs +++ b/routing/src/rib/vrf.rs @@ -113,7 +113,7 @@ impl ShimNhop { } } -#[derive(Copy, Clone, PartialEq)] +#[derive(Copy, Clone, Debug, PartialEq)] #[allow(unused)] pub enum VrfStatus { Active, diff --git a/routing/src/rib/vrftable.rs b/routing/src/rib/vrftable.rs index e12e6ddbc6..6221334a96 100644 --- a/routing/src/rib/vrftable.rs +++ b/routing/src/rib/vrftable.rs @@ -176,6 +176,17 @@ impl VrfTable { vrfid: VrfId, iftablew: &mut IfTableWriter, ) -> Result<(), RouterError> { + // The default vrf is not removable. `Vrf::set_status` says as much and keeps the default + // vrf `Active` so that the sweeps below never pick it up, but nothing stopped a caller + // naming it here -- and `get_default_vrf` treats the default vrf's existence as given, + // with an `unreachable!()` rather than an error. + if vrfid == Vrf::DEFAULT_VRFID { + error!("Refusing to remove the default vrf"); + return Err(RouterError::Internal( + "Bug: the default vrf cannot be removed", + )); + } + // remove the vrf from the vrf table debug!("Removing VRF {vrfid}..."); let Some(mut vrf) = self.by_id.remove(&vrfid) else { @@ -937,3 +948,319 @@ mod tests { test_vrf_fibgroup(build_test_vrf_nhops_partially_resolved()); } } + +/// Model-based properties over a [`VrfTable`]. +/// +/// The vrf table is where four key spaces have to agree: `by_id`, `by_vni`, each [`Vrf`]'s own +/// `vni` field, and the fib table's two -- `FibKey::Id` and the `FibKey::Vni` alias. Nothing holds +/// them together but the table's own methods doing the right number of things in the right order, +/// so what is generated here is sequences of those methods, and what is checked is that all four +/// still describe the same set of vrfs afterwards. +#[cfg(test)] +mod vrftable_properties { + use super::*; + use crate::interfaces::iftablerw::IfTableWriter; + use crate::rib::vrf::VrfStatus; + use bolero::{Driver, ValueGenerator}; + use std::collections::BTreeMap; + use std::ops::Bound::Included; + + const NUM_VRFS: u8 = 3; + const NUM_VNIS: u8 = 2; + const NUM_STATUSES: u8 = 3; + const MAX_CHANGES: u8 = 12; + + /// Vrf ids a generated table may hold, the default among them: `remove_vrf` takes any id, and + /// whether it should take that one is exactly the question worth generating for. + fn vrf_ids() -> Vec { + (0..u32::from(NUM_VRFS)).collect() + } + + fn vnis() -> Vec { + (1..=u32::from(NUM_VNIS)) + .map(|i| Vni::new_checked(100 * i).unwrap_or_else(|_| unreachable!())) + .collect() + } + + fn statuses() -> Vec { + vec![VrfStatus::Active, VrfStatus::Deleting, VrfStatus::Deleted] + } + + /// One change, as [`VrfTable`] exposes them, over indices into the pools above. + #[derive(Debug, Clone)] + enum Change { + AddVrf { vrf: usize, vni: Option }, + SetVni { vrf: usize, vni: usize }, + UnsetVni { vrf: usize }, + RemoveVrf { vrf: usize }, + SetStatus { vrf: usize, status: usize }, + RemoveDeleted, + RemoveDeleting, + } + + /// Draws sequences of [`Change`]s. + #[derive(Debug, Clone, Copy, Default)] + struct ChangeSequences; + + fn index(driver: &mut D, count: u8) -> Option { + driver + .gen_u8(Included(&0), Included(&(count - 1))) + .map(usize::from) + } + + impl ValueGenerator for ChangeSequences { + type Output = Vec; + + fn generate(&self, driver: &mut D) -> Option> { + let len = driver.gen_u8(Included(&0), Included(&MAX_CHANGES))?; + let mut out = Vec::with_capacity(usize::from(len)); + for _ in 0..len { + let change = match driver.gen_u8(Included(&0), Included(&6))? { + 0 => { + // `NUM_VNIS` means "created without a vni", which is the ordinary case. + // One draw rather than an `Option>`, which cannot be told apart + // from the driver running out of input. + let vrf = index(driver, NUM_VRFS)?; + let drawn = index(driver, NUM_VNIS + 1)?; + Change::AddVrf { + vrf, + vni: (drawn < usize::from(NUM_VNIS)).then_some(drawn), + } + } + 1 => Change::SetVni { + vrf: index(driver, NUM_VRFS)?, + vni: index(driver, NUM_VNIS)?, + }, + 2 => Change::UnsetVni { + vrf: index(driver, NUM_VRFS)?, + }, + 3 => Change::RemoveVrf { + vrf: index(driver, NUM_VRFS)?, + }, + 4 => Change::SetStatus { + vrf: index(driver, NUM_VRFS)?, + status: index(driver, NUM_STATUSES)?, + }, + 5 => Change::RemoveDeleted, + _ => Change::RemoveDeleting, + }; + out.push(change); + } + Some(out) + } + } + + /// Which vrfs exist, and for each the vni it carries and the status it is in. + /// + /// One map, from which all four of the table's views are derivable -- which is the point. The + /// table keeps them as four separate structures updated by hand; if any method updates three + /// of them, this says which one it missed. + type Model = BTreeMap, VrfStatus)>; + + fn owner_of(model: &Model, vni: Vni) -> Option { + model + .iter() + .find_map(|(id, (carried, _))| (*carried == Some(vni)).then_some(*id)) + } + + fn fresh_model() -> Model { + Model::from([(Vrf::DEFAULT_VRFID, (None, VrfStatus::Active))]) + } + + fn apply(table: &mut VrfTable, iftw: &mut IfTableWriter, model: &mut Model, change: &Change) { + let ids = vrf_ids(); + let all_vnis = vnis(); + match change { + Change::AddVrf { vrf, vni } => { + let id = ids[*vrf]; + let vni = vni.map(|i| all_vnis[i]); + let config = RouterVrfConfig::new(id, &format!("vrf{id}")).set_vni(vni); + let _ = table.add_vrf(&config); + // refused if the id is taken, or if the vni is + if model.contains_key(&id) || vni.is_some_and(|v| owner_of(model, v).is_some()) { + return; + } + model.insert(id, (vni, VrfStatus::Active)); + } + Change::SetVni { vrf, vni } => { + let id = ids[*vrf]; + let vni = all_vnis[*vni]; + let _ = table.set_vni(id, vni); + match owner_of(model, vni) { + // another vrf holds it: refused. The same vrf already holds it: nothing to do + Some(_) => (), + // otherwise the vrf drops whatever vni it had and takes this one -- but only + // if it exists at all + None => { + if let Some(entry) = model.get_mut(&id) { + entry.0 = Some(vni); + } + } + } + } + Change::UnsetVni { vrf } => { + let id = ids[*vrf]; + let _ = table.unset_vni(id); + if let Some(entry) = model.get_mut(&id) { + entry.0 = None; + } + } + Change::RemoveVrf { vrf } => { + let id = ids[*vrf]; + let _ = table.remove_vrf(id, iftw); + // the default vrf is refused + if id != Vrf::DEFAULT_VRFID { + model.remove(&id); + } + } + Change::SetStatus { vrf, status } => { + let id = ids[*vrf]; + let status = statuses()[*status]; + if let Ok(vrf) = table.get_vrf_mut(id) { + vrf.set_status(status); + } + // the default vrf's status is fixed: it is what keeps the sweeps below from + // deleting it + if id != Vrf::DEFAULT_VRFID + && let Some(entry) = model.get_mut(&id) + { + entry.1 = status; + } + } + Change::RemoveDeleted => { + table.remove_deleted_vrfs(iftw); + model.retain(|_, (_, status)| *status != VrfStatus::Deleted); + } + Change::RemoveDeleting => { + table.remove_deleting_vrfs(iftw); + model.retain(|_, (_, status)| *status != VrfStatus::Deleting); + } + } + } + + /// Check every view of the table against the one model, and against each other. + fn check(table: &VrfTable, model: &Model, at: &str) { + let ids = vrf_ids(); + let all_vnis = vnis(); + + // 1. by_id holds exactly the vrfs the model says, each carrying what the model says + assert_eq!(table.len(), model.len(), "vrf count {at}"); + for id in &ids { + let Ok(vrf) = table.get_vrf(*id) else { + assert!(!model.contains_key(id), "vrf {id} missing {at}"); + continue; + }; + let (vni, status) = model + .get(id) + .unwrap_or_else(|| panic!("vrf {id} unexpected {at}")); + assert_eq!(vrf.vrfid, *id, "vrf {id} filed under the wrong key {at}"); + assert_eq!(vrf.vni, *vni, "vrf {id} vni {at}"); + assert_eq!(vrf.status, *status, "vrf {id} status {at}"); + } + + // 2. by_vni is exactly the inverse of the vnis the vrfs carry -- no stale entry left by a + // removal, and none missing after a vni was set + assert_eq!( + table.by_vni.len(), + model.values().filter(|(vni, _)| vni.is_some()).count(), + "vni index size {at}" + ); + for vni in &all_vnis { + assert_eq!( + table.get_vrfid_by_vni(*vni).ok(), + owner_of(model, *vni), + "vni {vni} index {at}" + ); + assert_eq!( + table.get_vrf_by_vni(*vni).map(|vrf| vrf.vrfid).ok(), + owner_of(model, *vni), + "vni {vni} lookup {at}" + ); + } + + // 3. the fib table holds a fib for every vrf, under its id and under its vni if it has + // one, and every one of those keys reaches a live fib with the right identity + let fibs = table.fibtablew.enter().unwrap_or_else(|| unreachable!()); + let expected_keys = model.len() + model.values().filter(|(v, _)| v.is_some()).count(); + assert_eq!(fibs.len(), expected_keys, "fib table size {at}"); + for id in &ids { + let key = FibKey::from_vrfid(*id); + let Some(fib) = fibs.get_fib(key) else { + assert!(!model.contains_key(id), "no fib for vrf {id} {at}"); + continue; + }; + assert!(fib.is_valid(), "fib for vrf {id} is dead {at}"); + assert_eq!( + fib.get_id(), + Some(key), + "fib for vrf {id} is not its own {at}" + ); + } + for vni in &all_vnis { + let Some(fib) = fibs.get_fib(FibKey::from_vni(*vni)) else { + assert!(owner_of(model, *vni).is_none(), "no fib for vni {vni} {at}"); + continue; + }; + let owner = owner_of(model, *vni) + .unwrap_or_else(|| panic!("fib aliased by vni {vni} with no owner {at}")); + assert!(fib.is_valid(), "fib aliased by vni {vni} is dead {at}"); + assert_eq!( + fib.get_id(), + Some(FibKey::from_vrfid(owner)), + "vni {vni} reaches the wrong fib {at}" + ); + } + drop(fibs); + + // 4. the default vrf is always there and always active. `get_default_vrf` treats that as + // given, with an `unreachable!()` rather than an error + assert!(table.contains(Vrf::DEFAULT_VRFID), "no default vrf {at}"); + assert_eq!( + table.get_default_vrf().status, + VrfStatus::Active, + "default vrf not active {at}" + ); + + // 5. the table's own consistency check agrees. `check_vni` is the in-tree half of this + // oracle; it should pass for a vrf with a vni and fail for one without + for id in &ids { + let Some((vni, _)) = model.get(id) else { + continue; + }; + assert_eq!( + table.check_vni(*id).is_ok(), + vni.is_some(), + "check_vni disagrees for vrf {id} {at}" + ); + } + } + + /// The pools and the constants that index them agree. + #[test] + fn the_pools_are_the_size_the_generator_thinks() { + assert_eq!(vrf_ids().len(), usize::from(NUM_VRFS)); + assert_eq!(vnis().len(), usize::from(NUM_VNIS)); + assert_eq!(statuses().len(), usize::from(NUM_STATUSES)); + assert_eq!(vrf_ids()[0], Vrf::DEFAULT_VRFID); + } + + /// After any sequence of changes, every view of the vrf table still describes the same vrfs. + #[test] + fn a_vrf_tables_four_views_stay_in_step() { + bolero::check!() + .with_generator(ChangeSequences) + .cloned() + .for_each(|changes: Vec| { + let (fibtw, _fibtr) = FibTableWriter::new(); + let (mut iftw, _iftr) = IfTableWriter::new(); + let mut table = VrfTable::new(fibtw); + let mut model = fresh_model(); + + check(&table, &model, "on a fresh table"); + for (step, change) in changes.iter().enumerate() { + apply(&mut table, &mut iftw, &mut model, change); + check(&table, &model, &format!("at step {step} of {changes:?}")); + } + }); + } +} From 55305b8492412005a661ed3954ad0a04d7d19cd9 Mon Sep 17 00:00:00 2001 From: Daniel Noland Date: Fri, 7 Aug 2026 13:39:16 -0600 Subject: [PATCH 24/33] fix(frrmi): Frame a message by what arrived, not by what was asked for `IoBuffer` exists to cope with partial reads and writes on a stream socket. On the read side it got the distinction wrong, and the result is a live bug. `Frrmi::recv` sizes the read buffer for the read it is *about to attempt*: readb.buffer.resize(readb.used + len, 0); so after that call `buffer.len()` is what we hoped to have and `used` is what we actually got. `msg_len`, `next_read_len` and `is_ready` all consulted `buffer.len()`. Three consequences, in ascending order of how much they cost: **A header and a body arriving in separate reads killed the connection.** Read the 16-octet header, ask for the body, get `WouldBlock`: `buffer.len()` is now 17 for a one-octet body, so on the next pass `next_read_len` computed `1 - (17 - 16) = 0`, decided the message was complete, and called `recv` for zero octets. `read` into an empty buffer returns `Ok(0)`, which this code reads as end-of-stream -- so the frrmi raised `FrrmiPeerLeft`, dropped the socket and restarted, on the ordinary case of a response that does not arrive in one piece. It then retried the config, so the symptom is a reconnect loop rather than a stall, which is presumably why it has gone unnoticed. **A half-arrived header read as a complete one.** With fewer than 16 octets received, `buffer[0..8]` is partly the zeros `resize` wrote, so the announced length came out too small -- zero, if the received prefix of the length field happened to be zero, which is every short read of a header whose body length is a multiple of 256. `next_read_len` then returned 0 and `deserialize` sliced `buffer[16..used]` with `used` below 16, which panics outright in the routing thread. **The announced length was unbounded.** It comes off the wire and `recv` resizes to it, so a confused or hostile frr-agent announcing `u64::MAX` made `readb.used + len` overflow -- and a merely large announcement would have been a request to allocate that many octets. So: the three read-side predicates now count `used`, `IoBuffer::len` says in a comment that it means something only on the write side, `is_ready` subtracts 16 from `used` instead of adding it to a peer-supplied length, and a message longer than `MAX_MSG_LEN` is refused with `DecodeFailure`. Responses from the agent are a status word or an error message, so 16 MiB is generous by orders of magnitude; the bound is there to cap the allocation, not to constrain the protocol. Found by a round-trip property whose oracle is the message itself: serialize it, deliver its octets in generated chunks, and require what comes back out to be what went in. The first counterexample was `chunks: [16]` -- the header in one write and the body in the next. A second property runs the chunking across several messages on one connection, so a write may straddle a message boundary or carry two at once. Each of the three fixes was confirmed load-bearing by reverting it: reading `msg_len` off `buffer.len()` fails with a subtract overflow, counting `next_read_len` off it reproduces the original `Peer left`, and removing the length bound fails the absurd-length test with an add overflow. Co-Authored-By: Claude Opus 5 (1M context) Signed-off-by: Daniel Noland --- routing/src/frr/frrmi.rs | 271 +++++++++++++++++++++++++++++++++++---- 1 file changed, 245 insertions(+), 26 deletions(-) diff --git a/routing/src/frr/frrmi.rs b/routing/src/frr/frrmi.rs index 50f9ad802d..c8311e71e2 100644 --- a/routing/src/frr/frrmi.rs +++ b/routing/src/frr/frrmi.rs @@ -327,6 +327,11 @@ impl Frrmi { return Err(FrrErr::NotConnected); }; loop { + if let Some(announced) = self.readb.oversized() { + error!("Frr-agent announced a {announced}-octet message: refusing it"); + self.readb.clear(); + return Err(FrrErr::DecodeFailure); + } let pending = self.readb.next_read_len(); debug!("Recv data (read:{} pending:{pending})", self.readb.used); match Self::recv(sock, &mut self.readb, pending) { @@ -400,6 +405,14 @@ struct IoBuffer { used: usize, } impl IoBuffer { + /// Octets of `|length|genid|` ahead of every message body. + const HEADER_LEN: usize = 16; + + /// The largest message body we will accept off the wire. Responses from the frr-agent are a + /// status word or an error message, so this is generous by orders of magnitude; it is here to + /// bound the allocation, not to constrain the protocol. + const MAX_MSG_LEN: usize = 16 * 1024 * 1024; + #[must_use] #[allow(unused)] pub fn new() -> Self { @@ -412,6 +425,10 @@ impl IoBuffer { self.buffer.clear(); self.used = 0; } + /// The size of the buffer. Meaningful on the **write** side only, where `serialize` fills it: + /// there, `buffer.len()` is the size of the message and `used` is how much of it has gone out. + /// On the read side `recv` resizes the buffer to the size of the read it is about to attempt, so + /// `buffer.len()` says what we hoped for and only `used` says what arrived. #[must_use] fn len(&self) -> usize { self.buffer.len() @@ -426,37 +443,45 @@ impl IoBuffer { self.extend(msg); } - /// Tell the length that a message (encoded as |length|genid|data|) must have. - /// If less than 8 octets have been read it is not possible to know how big the message is yet. + /// Tell the length that a message (encoded as |length|genid|data|) must have, once the whole + /// header has been received. `None` until then. + /// + /// Off `used`, not off `buffer.len()`: see [`IoBuffer::len`]. Reading the length out of a + /// buffer sized for a read that has not happened yet takes whatever `resize` zero-filled for + /// the tail of the header, so a half-arrived header reads as a complete one announcing a + /// shorter message. #[must_use] fn msg_len(&self) -> Option { - if self.buffer.len() < 8 { - None - } else { - let len_buf = &self.buffer[0..8] - .try_into() - .unwrap_or_else(|_| unreachable!()); - - #[allow(clippy::cast_possible_truncation)] - let msg_len = u64::from_ne_bytes(*len_buf) as usize; - Some(msg_len) + if self.used < Self::HEADER_LEN { + return None; } + let len_buf: &[u8; 8] = &self.buffer[0..8] + .try_into() + .unwrap_or_else(|_| unreachable!()); + + #[allow(clippy::cast_possible_truncation)] + let msg_len = u64::from_ne_bytes(*len_buf) as usize; + Some(msg_len) + } + + /// The announced message length, if it is one we refuse to accept. + /// + /// The length prefix comes off the wire and `recv` resizes the read buffer to it, so without a + /// bound a confused or hostile frr-agent could ask us to allocate up to `u64::MAX`. + #[must_use] + fn oversized(&self) -> Option { + self.msg_len().filter(|len| *len > Self::MAX_MSG_LEN) } - /// Tell the number of octets that should be read next according to the contents of the read buffer - /// to get a message or be able to determine its length. - /// If less than 16 octets have been received, this returns the number needed to have exactly 16. - /// Else, we return the number of octets that are pending to have the complete message. + + /// Tell the number of octets that should be read next according to what has been received so + /// far, to get a message or be able to determine its length. + /// Until the whole header has arrived, this returns the number needed to complete it. + /// After that, the number of octets still pending to have the complete message. #[must_use] fn next_read_len(&self) -> usize { - if self.len() < 16 { - 16 - self.len() - } else { - let msg_len = self.msg_len().unwrap_or_else(|| unreachable!()); - if msg_len > (self.len() - 16) { - msg_len - (self.len() - 16) - } else { - 0 - } + match self.msg_len() { + None => Self::HEADER_LEN - self.used, + Some(msg_len) => msg_len.saturating_sub(self.used - Self::HEADER_LEN), } } @@ -464,7 +489,9 @@ impl IoBuffer { #[must_use] fn is_ready(&self) -> bool { match self.msg_len() { - Some(m) => self.len() == m + 16, + // subtracting rather than adding 16: the announced length is peer-supplied, and + // `m + 16` overflows for one close enough to `usize::MAX` + Some(msg_len) => self.used - Self::HEADER_LEN == msg_len, None => false, } } @@ -488,3 +515,195 @@ impl IoBuffer { Ok(FrrmiResponse { genid, data }) } } + +/// Properties over the frrmi wire framing. +/// +/// `IoBuffer` exists to cope with partial reads and writes on a stream socket -- so the property +/// worth having is that the message survives *any* division of its octets into reads. The oracle is +/// the message itself: serialize, deliver the bytes in generated chunks, and what comes back out +/// must be what went in. +#[cfg(test)] +mod framing_properties { + use super::*; + use bolero::{Driver, ValueGenerator}; + use std::ops::Bound::Included; + + const MAX_BODY: u8 = 20; + const MAX_CHUNK: u8 = 24; + const MAX_CHUNKS: u8 = 8; + const MAX_MESSAGES: u8 = 4; + + /// A message, and the way the peer's octets happen to arrive. + #[derive(Debug, Clone)] + struct Delivery { + genid: GenId, + /// The response body. Empty is worth generating: a zero-length body makes every octet of + /// the length prefix zero, which is the value a half-read header is indistinguishable from. + body: String, + /// Sizes of the successive writes the peer makes. Applied in order and clamped to what is + /// left; anything still unsent after the list runs out goes in one final write. + chunks: Vec, + } + + /// Draws [`Delivery`]s. + #[derive(Debug, Clone, Copy, Default)] + struct Deliveries; + + impl ValueGenerator for Deliveries { + type Output = Delivery; + + fn generate(&self, driver: &mut D) -> Option { + let genid = GenId::from(driver.gen_u8(Included(&0), Included(&3))?); + let body_len = usize::from(driver.gen_u8(Included(&0), Included(&MAX_BODY))?); + let body: String = (0..body_len) + .map(|i| char::from(b'a' + u8::try_from(i % 26).unwrap_or_else(|_| unreachable!()))) + .collect(); + + let count = driver.gen_u8(Included(&0), Included(&MAX_CHUNKS))?; + let mut chunks = Vec::with_capacity(usize::from(count)); + for _ in 0..count { + chunks.push(usize::from( + driver.gen_u8(Included(&1), Included(&MAX_CHUNK))?, + )); + } + Some(Delivery { + genid, + body, + chunks, + }) + } + } + + /// A `Frrmi` reading from one end of a socket pair, with the peer's end alongside it. + fn connected_pair() -> (UnixStream, Frrmi) { + let (peer, ours) = UnixStream::pair().unwrap_or_else(|e| unreachable!("{e}")); + let frrmi = Frrmi { + sock: Some(ours), + ..Frrmi::default() + }; + (peer, frrmi) + } + + /// Draws a sequence of [`Delivery`]s to send down one connection. + #[derive(Debug, Clone, Copy, Default)] + struct Streams; + + impl ValueGenerator for Streams { + type Output = Vec; + + fn generate(&self, driver: &mut D) -> Option> { + let count = driver.gen_u8(Included(&1), Included(&MAX_MESSAGES))?; + let mut out = Vec::with_capacity(usize::from(count)); + for _ in 0..count { + out.push(Deliveries.generate(driver)?); + } + Some(out) + } + } + + /// A message survives being delivered in any sequence of chunks. + #[test] + fn a_message_survives_any_division_into_reads() { + bolero::check!() + .with_generator(Deliveries) + .cloned() + .for_each(|delivery: Delivery| { + let mut wire = IoBuffer::new(); + wire.serialize(delivery.genid, delivery.body.as_bytes()); + let bytes = wire.buffer; + + let (mut peer, mut frrmi) = connected_pair(); + + let mut sent = 0; + let mut got = None; + let mut sizes = delivery.chunks.iter().copied(); + while sent < bytes.len() { + let take = sizes.next().unwrap_or(usize::MAX).min(bytes.len() - sent); + peer.write_all(&bytes[sent..sent + take]) + .unwrap_or_else(|e| unreachable!("{e}")); + sent += take; + + match frrmi.recv_msg() { + Ok(Some(response)) => { + assert!(got.is_none(), "two messages from one, for {delivery:?}"); + got = Some(response); + } + Ok(None) => (), + Err(e) => panic!("recv failed with {e} for {delivery:?}"), + } + } + + let response = got.unwrap_or_else(|| panic!("no message, for {delivery:?}")); + assert_eq!(response.genid, delivery.genid, "genid for {delivery:?}"); + assert_eq!(response.data, delivery.body, "body for {delivery:?}"); + }); + } + + /// A connection carries one message after another, however the octets are divided. + /// + /// The chunking runs over the whole stream rather than over each message, so a single write may + /// straddle a message boundary or carry several messages at once. `deserialize` clears the read + /// buffer, and this is what says nothing is left in it to confuse the message after. + #[test] + fn a_connection_carries_one_message_after_another() { + bolero::check!() + .with_generator(Streams) + .cloned() + .for_each(|deliveries: Vec| { + let mut bytes = Vec::new(); + for delivery in &deliveries { + let mut wire = IoBuffer::new(); + wire.serialize(delivery.genid, delivery.body.as_bytes()); + bytes.extend_from_slice(&wire.buffer); + } + + let (mut peer, mut frrmi) = connected_pair(); + + // one chunk list over the whole stream, taken from the deliveries in turn + let mut sizes = deliveries.iter().flat_map(|d| d.chunks.iter().copied()); + let mut got: Vec<(GenId, String)> = Vec::new(); + let mut sent = 0; + while sent < bytes.len() { + let take = sizes.next().unwrap_or(usize::MAX).min(bytes.len() - sent); + peer.write_all(&bytes[sent..sent + take]) + .unwrap_or_else(|e| unreachable!("{e}")); + sent += take; + + // drain: a single write may have completed more than one message + loop { + match frrmi.recv_msg() { + Ok(Some(response)) => got.push((response.genid, response.data)), + Ok(None) => break, + Err(e) => panic!("recv failed with {e} for {deliveries:?}"), + } + } + } + + let want: Vec<(GenId, String)> = deliveries + .iter() + .map(|d| (d.genid, d.body.clone())) + .collect(); + assert_eq!(got, want, "for {deliveries:?}"); + }); + } + + /// A message longer than we will accept is refused, not allocated for. + /// + /// The length prefix is whatever the peer put on the wire, and `recv` resizes the read buffer to + /// it. Announcing `u64::MAX` used to make that resize overflow its own length arithmetic. + #[test] + fn an_absurd_announced_length_is_refused() { + let mut header = Vec::new(); + header.extend_from_slice(&u64::MAX.to_ne_bytes()); + header.extend_from_slice(&0i64.to_ne_bytes()); + + let (mut peer, mut frrmi) = connected_pair(); + peer.write_all(&header) + .unwrap_or_else(|e| unreachable!("{e}")); + + assert!( + matches!(frrmi.recv_msg(), Err(FrrErr::DecodeFailure)), + "an absurd length must be refused" + ); + } +} From 8132aa75dda56aee44c0ef55b3c743f6825e7f4f Mon Sep 17 00:00:00 2001 From: Daniel Noland Date: Fri, 7 Aug 2026 13:52:23 -0600 Subject: [PATCH 25/33] fix(routing): Install a route with no next-hops as a drop, in the rib A route with no next-hops used to be accepted by `Vrf::add_route_complete` and put in the route trie, while `FibWriter::add_fibroute` refused it -- it rejects an empty key list. The rib and the fib then disagreed about the forwarding table, and disagreed in the worst direction: a packet matching that prefix falls through to a shorter one in the fib and is forwarded somewhere else, rather than dropped. Resolving via a default is how a routing loop starts. `add_route_rpc` knew this. It ended with: // If no next-hop was received with the route (or we could not successfully // process any), install the route anyway with an action drop. This is // better than not installing the route as that could break consistency // (e.g. resolving via a default) and cause a loop. if nhops.is_empty() { nhops.push(RouteNhop::default()); } Correct, well reasoned, and in the wrong place: one layer up, in a different module from the function whose contract it was upholding. It is the only production path into `add_route_complete`, so this was latent -- the fourth time on this branch that an invariant turned out to be held by a caller rather than by the function that depends on it. So the substitution moves into `Vrf::nhops_or_drop`, used by both `add_route_complete` and `add_route`. `add_route_rpc` keeps its warnings, since only that layer can tell "the control plane sent no next-hops" from "none of the ones it sent could be processed", but it no longer has to remember to inject anything. Found by a model-based property over the vrf's route table. Two structures move together on every route change -- the tries, and the `NhopStore` the routes hold `Rc`s into -- and a third, the vrf's `Fib`, is written through on the same calls. The model is one map, prefix to (next-hop keys, stale), and everything is checked against it: - the tries hold exactly the model's prefixes, each route naming the model's next-hop keys in order, with the model's stale flag - the next-hop store holds exactly the keys the routes name. One too many is a leak that keeps a stale fib group alive; one too few and a route names something nothing will resolve - `lpm` resolves for every address and lands where the model says, which is what keeps `lpm_v4`/`lpm_v6` and `check_deletion` off their `unreachable!()`s - the fib describes the same prefixes as the vrf Verified by breaking four separate things: removing the new substitution, not deregistering a replaced route's next-hops, not reinstalling a deleted default route, and letting `set_stale` mark the preset drop routes. Each fails a different one of the four checks above. Two hundred thousand cases pass. Co-Authored-By: Claude Opus 5 (1M context) Signed-off-by: Daniel Noland --- routing/src/rib/vrf.rs | 364 +++++++++++++++++++++++++++++++- routing/src/router/rpc_adapt.rs | 10 +- 2 files changed, 366 insertions(+), 8 deletions(-) diff --git a/routing/src/rib/vrf.rs b/routing/src/rib/vrf.rs index 2ad0ca4899..46b6dcb815 100644 --- a/routing/src/rib/vrf.rs +++ b/routing/src/rib/vrf.rs @@ -4,10 +4,11 @@ //! VRF module to store Ipv4 and Ipv6 routing tables use bitflags::bitflags; +use std::borrow::Cow; use std::hash::Hash; use std::net::IpAddr; use std::rc::{Rc, Weak}; -use tracing::debug; +use tracing::{debug, warn}; #[cfg(test)] use common::cliprovider::Frame; @@ -392,6 +393,27 @@ impl Vrf { } } + ///////////////////////////////////////////////////////////////////////// + /// The next-hops to install for a route, substituting an explicit drop if there are none. + /// + /// A route with no next-hops cannot forward anything, and leaving it out of the table + /// altogether is worse than installing it as a drop: the rib would keep it while the fib + /// declined it -- `FibWriter::add_fibroute` refuses a route with no next-hop keys -- so a + /// packet matching it would fall through to a shorter prefix in the fib and be forwarded + /// somewhere else rather than dropped. Resolving via a default is how a routing loop starts. + /// + /// `add_route_rpc` used to do this, for the one path that could produce an empty list. Doing + /// it here means no caller can skip it. + ///////////////////////////////////////////////////////////////////////// + fn nhops_or_drop<'a>(prefix: &Prefix, nhops: &'a [RouteNhop]) -> Cow<'a, [RouteNhop]> { + if nhops.is_empty() { + warn!("Route to {prefix} has no next-hop: will install it with action drop"); + Cow::Owned(vec![RouteNhop::default()]) + } else { + Cow::Borrowed(nhops) + } + } + ///////////////////////////////////////////////////////////////////////// // Route Insertion ///////////////////////////////////////////////////////////////////////// @@ -403,7 +425,7 @@ impl Vrf { vrf0: Option<&Vrf>, ) { // register next-hops and let the route keep references to the shared nexthops created/found - route.s_nhops = self.register_shared_nhops(nhops); + route.s_nhops = self.register_shared_nhops(&Self::nhops_or_drop(prefix, nhops)); // resolve the new route next-hops. This is only for testing. In prod code, // this method is only used for drop routes which require no resolution. @@ -464,7 +486,7 @@ impl Vrf { rstore: &RmacStore, ) { // register next-hops and let the route keep references to the shared nexthops created/found - route.s_nhops = self.register_shared_nhops(nhops); + route.s_nhops = self.register_shared_nhops(&Self::nhops_or_drop(prefix, nhops)); let rvrf = vrf0.unwrap_or(self); @@ -1140,3 +1162,339 @@ pub mod tests { } } + +/// Model-based properties over a [`Vrf`]'s route table. +/// +/// Two structures move together on every route change: the route tries, and the [`NhopStore`] whose +/// entries the routes hold `Rc`s into. A route that comes or goes has to leave the store holding +/// exactly the next-hops the remaining routes name -- one too many is a leak that keeps a stale +/// fib group alive, one too few and a route names a next-hop nothing will resolve. A third, the +/// vrf's `Fib`, is written through on the same calls and has to end up describing the same +/// prefixes. +#[cfg(test)] +mod vrf_properties { + use super::*; + use crate::fib::fibtype::{FibKey, FibWriter}; + use crate::rib::nexthop::NhopKey; + use bolero::{Driver, ValueGenerator}; + use std::collections::{BTreeMap, BTreeSet}; + use std::ops::Bound::Included; + use std::str::FromStr; + + const NUM_PREFIXES: u8 = 8; + const NUM_NHOPS: u8 = 3; + const MAX_CHANGES: u8 = 10; + const MAX_NHOPS_PER_ROUTE: u8 = 3; + + /// `0.0.0.0/0` and `::/0`, at these indices in [`prefixes`]. A vrf always carries a route for + /// both: `Vrf::lpm` has no answer otherwise and says so with an `unreachable!()`, and + /// `check_deletion` reaches for them by name. + const ROOT_V4: usize = 0; + const ROOT_V6: usize = 5; + + /// The prefixes a generated vrf may carry routes for. Nested, and both roots. + fn prefixes() -> Vec { + [ + "0.0.0.0/0", + "10.0.0.0/8", + "10.1.0.0/16", + "10.1.2.0/24", + "10.1.2.3/32", + "::/0", + "2001:db8::/32", + "2001:db8:1::/48", + ] + .iter() + .map(|p| Prefix::from_str(p).unwrap_or_else(|_| unreachable!())) + .collect() + } + + /// Addresses to look up: inside each level of the nesting, and outside all of them. + fn probes() -> Vec { + [ + "9.9.9.9", + "10.9.9.9", + "10.1.9.9", + "10.1.2.9", + "10.1.2.3", + "2000::1", + "2001:db8::1", + "2001:db8:1::1", + ] + .iter() + .map(|a| IpAddr::from_str(a).unwrap_or_else(|_| unreachable!())) + .collect() + } + + /// The next-hops a generated route may use. Small, so that routes share them: sharing is what + /// makes the store's reference counting do any work. + fn nhops() -> Vec { + vec![ + tests::build_test_nhop(Some("10.0.0.1"), Some(1), 0, None), + tests::build_test_nhop(Some("10.0.0.2"), None, 0, None), + tests::build_test_nhop(None, Some(3), 0, None), + ] + } + + /// One change, over indices into the pools above. + #[derive(Debug, Clone)] + enum Change { + /// A route. `nhops` may be empty: `add_route_complete` takes whatever the control plane + /// hands it, and a route with nowhere to go is the interesting end of that. + AddRoute { + prefix: usize, + nhops: Vec, + }, + DelRoute { + prefix: usize, + }, + SetStale { + value: bool, + }, + RemoveStale, + } + + /// Draws sequences of [`Change`]s. + #[derive(Debug, Clone, Copy, Default)] + struct ChangeSequences; + + fn index(driver: &mut D, count: u8) -> Option { + driver + .gen_u8(Included(&0), Included(&(count - 1))) + .map(usize::from) + } + + impl ValueGenerator for ChangeSequences { + type Output = Vec; + + fn generate(&self, driver: &mut D) -> Option> { + let len = driver.gen_u8(Included(&0), Included(&MAX_CHANGES))?; + let mut out = Vec::with_capacity(usize::from(len)); + for _ in 0..len { + let change = match driver.gen_u8(Included(&0), Included(&3))? { + 0 => { + let prefix = index(driver, NUM_PREFIXES)?; + let count = driver.gen_u8(Included(&0), Included(&MAX_NHOPS_PER_ROUTE))?; + let mut nhops = Vec::with_capacity(usize::from(count)); + for _ in 0..count { + nhops.push(index(driver, NUM_NHOPS)?); + } + Change::AddRoute { prefix, nhops } + } + 1 => Change::DelRoute { + prefix: index(driver, NUM_PREFIXES)?, + }, + 2 => Change::SetStale { + value: driver.produce::()?, + }, + _ => Change::RemoveStale, + }; + out.push(change); + } + Some(out) + } + } + + /// Which prefixes carry routes, and for each the next-hops it names and whether it is stale. + /// + /// The two root routes are the preset drop routes a vrf is born with; they are never removed, + /// only reset, and `set_stale` skips them. + #[derive(Debug, Clone)] + struct Model { + /// prefix index -> the next-hop keys of its route in order, and whether it is stale. + /// + /// Keys rather than pool indices, because the preset drop route names one the generator + /// cannot ask for, and because a root route holding a generated route is not the same thing + /// as a root route holding the preset one. + routes: BTreeMap, bool)>, + } + + impl Model { + /// The route a vrf installs for each root on creation: drop, so that a lookup always has an + /// answer. + fn preset() -> Vec { + vec![NhopKey::with_drop()] + } + + fn new() -> Self { + Self { + routes: BTreeMap::from([ + (ROOT_V4, (Self::preset(), false)), + (ROOT_V6, (Self::preset(), false)), + ]), + } + } + + fn is_root(prefix: usize) -> bool { + prefix == ROOT_V4 || prefix == ROOT_V6 + } + + /// Every next-hop key the routes name -- the set the store should hold, no more and no less. + fn referenced(&self) -> BTreeSet { + self.routes + .values() + .flat_map(|(nhops, _)| nhops.iter().cloned()) + .collect() + } + + /// The longest prefix carrying a route that covers `addr`. + fn lpm(&self, addr: &IpAddr, pool: &[Prefix]) -> Option { + self.routes + .keys() + .copied() + .filter(|i| pool[*i].covers_addr(addr)) + .max_by_key(|i| pool[*i].length()) + } + + fn apply(&mut self, change: &Change, pool: &[RouteNhop]) { + match change { + Change::AddRoute { prefix, nhops } => { + // a route with no next-hops is installed as a drop rather than left out + let keys = if nhops.is_empty() { + Self::preset() + } else { + nhops.iter().map(|i| pool[*i].key.clone()).collect() + }; + self.routes.insert(*prefix, (keys, false)); + } + Change::DelRoute { prefix } => { + if Self::is_root(*prefix) { + // a root route is reset to the preset drop route, not removed + self.routes.insert(*prefix, (Self::preset(), false)); + } else { + self.routes.remove(prefix); + } + } + Change::SetStale { value } => { + // `Vrf::set_stale` skips a route whose prefix is a root, and separately one + // that is still a preset drop route. Generated routes are never the latter, so + // here the root test is the whole of it. + for (prefix, (_, stale)) in &mut self.routes { + if !Self::is_root(*prefix) { + *stale = *value; + } + } + } + Change::RemoveStale => { + let stale: Vec = self + .routes + .iter() + .filter_map(|(prefix, (_, stale))| stale.then_some(*prefix)) + .collect(); + for prefix in stale { + self.apply(&Change::DelRoute { prefix }, pool); + } + } + } + } + } + + fn apply_to_vrf(vrf: &mut Vrf, rstore: &RmacStore, change: &Change, pool: &[RouteNhop]) { + let prefixes = prefixes(); + match change { + Change::AddRoute { prefix, nhops } => { + let route = tests::build_test_route(RouteOrigin::Bgp, 20, 100); + let nhops: Vec = nhops.iter().map(|i| pool[*i].clone()).collect(); + vrf.add_route_complete(&prefixes[*prefix], route, &nhops, None, rstore); + } + Change::DelRoute { prefix } => vrf.del_route(prefixes[*prefix], None, rstore), + Change::SetStale { value } => vrf.set_stale(*value), + Change::RemoveStale => vrf.remove_stale_routes(None, rstore), + } + } + + /// Check the vrf, its next-hop store and its fib against the one model. + fn check(vrf: &Vrf, model: &Model, at: &str) { + let prefixes = prefixes(); + let probes = probes(); + + // 1. the route tries hold exactly the prefixes the model says + let held: BTreeSet = (0..prefixes.len()) + .filter(|i| vrf.get_route(prefixes[*i]).is_some()) + .collect(); + let want: BTreeSet = model.routes.keys().copied().collect(); + assert_eq!(held, want, "route set {at}"); + assert_eq!( + vrf.len_v4() + vrf.len_v6(), + model.routes.len(), + "route count {at}" + ); + + // 2. each route names the next-hops the model says, in order + for (prefix, (nhops, stale)) in &model.routes { + let route = vrf + .get_route(prefixes[*prefix]) + .unwrap_or_else(|| panic!("no route for {prefix} {at}")); + let got: Vec = route.s_nhops.iter().map(|s| s.rc.key.clone()).collect(); + assert_eq!(got, *nhops, "next-hops of {prefix} {at}"); + assert_eq!(route.is_stale(), *stale, "stale flag of {prefix} {at}"); + } + + // 3. the next-hop store holds exactly the next-hops the routes name. One too many is a + // leak that keeps a stale fib group alive; one too few and a route names something + // nothing will resolve + let stored: BTreeSet = vrf.nhstore.iter().map(|rc| rc.key.clone()).collect(); + assert_eq!(stored, model.referenced(), "next-hop store {at}"); + + // 4. lpm resolves for every address, and lands where the model says + for probe in &probes { + let want = model + .lpm(probe, &prefixes) + .unwrap_or_else(|| panic!("model has no route for {probe} {at}")); + let (hit, _) = vrf.lpm(*probe); + assert_eq!(hit, prefixes[want], "lpm for {probe} {at}"); + } + + // 5. the fib describes the same prefixes as the vrf. Otherwise a packet matching a route + // the fib never heard about falls through to a shorter prefix and is forwarded + // somewhere else rather than dropped + let fibw = vrf.fibw.as_ref().unwrap_or_else(|| unreachable!()); + let fib = fibw.enter().unwrap_or_else(|| unreachable!()); + let mut want_v4 = BTreeSet::new(); + let mut want_v6 = BTreeSet::new(); + for prefix in model.routes.keys() { + match prefixes[*prefix] { + Prefix::IPV4(p) => want_v4.insert(p), + Prefix::IPV6(p) => want_v6.insert(p), + }; + } + let fib_v4: BTreeSet = fib.iter_v4().map(|(prefix, _)| prefix).collect(); + let fib_v6: BTreeSet = fib.iter_v6().map(|(prefix, _)| prefix).collect(); + assert_eq!(fib_v4, want_v4, "fib ipv4 prefixes {at}"); + assert_eq!(fib_v6, want_v6, "fib ipv6 prefixes {at}"); + } + + /// The pools and the constants that index them agree. + #[test] + fn the_pools_are_the_size_the_generator_thinks() { + assert_eq!(prefixes().len(), usize::from(NUM_PREFIXES)); + assert_eq!(nhops().len(), usize::from(NUM_NHOPS)); + assert_eq!(prefixes()[ROOT_V4], Prefix::root_v4()); + assert_eq!(prefixes()[ROOT_V6], Prefix::root_v6()); + } + + /// After any sequence of route changes, the vrf, its next-hop store and its fib agree. + #[test] + fn a_vrfs_routes_and_next_hops_stay_in_step() { + let pool = nhops(); + let rstore = RmacStore::new(); + + bolero::check!() + .with_generator(ChangeSequences) + .cloned() + .for_each(|changes: Vec| { + let config = RouterVrfConfig::new(1, "test"); + let mut vrf = Vrf::new(&config); + let (fibw, _fibr) = FibWriter::new(FibKey::from_vrfid(1)); + vrf.set_fibw(fibw); + let mut model = Model::new(); + + check(&vrf, &model, "on a fresh vrf"); + for (step, change) in changes.iter().enumerate() { + apply_to_vrf(&mut vrf, &rstore, change, &pool); + model.apply(change, &pool); + check(&vrf, &model, &format!("at step {step} of {changes:?}")); + } + }); + } +} diff --git a/routing/src/router/rpc_adapt.rs b/routing/src/router/rpc_adapt.rs index ff2e39bbc9..03bfbba33c 100644 --- a/routing/src/router/rpc_adapt.rs +++ b/routing/src/router/rpc_adapt.rs @@ -222,12 +222,12 @@ impl Vrf { } } - // If no next-hop was received with the route (or we could not successfully process any), - // install the route anyway with an action drop. This is better than not installing the - // route as that could break consistency (e.g. resolving via a default) and cause a loop. + // If no next-hop was received with the route, or none of them could be processed, the + // route is still installed -- with an action drop, which `Vrf::nhops_or_drop` substitutes. + // Not installing it would break consistency (e.g. resolving via a default) and cause a + // loop. Warn here rather than there, since only this layer can tell the two cases apart. if nhops.is_empty() { - warn!("Route to {prefix} from RPC would have no next-hop. Will inject DROP next-hop"); - nhops.push(RouteNhop::default()); + warn!("Route to {prefix} from RPC has no usable next-hop: will be a DROP route"); } // N.B. route and next-hops are passed separately From 1f6984e893f91a5748d6c138dcdb4d5acc9a6881 Mon Sep 17 00:00:00 2001 From: Daniel Noland Date: Fri, 7 Aug 2026 14:10:01 -0600 Subject: [PATCH 26/33] test(routing): Cover the vrf deletion check `Vrf::check_deletion` sits entirely behind `status == VrfStatus::Deleting`, and the property added in e36afa56d only ever held an `Active` vrf -- so the whole body, both of its `unreachable!()`s included, never ran. A coverage pass turned that up in a file the same commit had just reported at 95%. The transition matters: it is what moves a vrf from `Deleting` to `Deleted` once the only routes left are the two preset drop ones, and so it is what decides whether `VrfTable::remove_deleting_vrfs` ever picks the vrf up. The generator now draws status moves, and the model tracks the status and performs the same transition on route deletion. That needed one more thing in the model: whether a route is still the *preset* drop route, which is not the same as naming the drop next-hop. A generated route for a root prefix with no next-hops names it too, but carries a real origin, distance and metric, so `Route::is_preset_drop_route` says no -- and that is the question `check_deletion` asks. The check now asserts on it directly, which also reaches the conjuncts of `is_preset_drop_route` that short-circuiting had hidden. Production coverage of `rib/vrf.rs` goes 89.8% -> 93.9% (production lines only; `#[cfg(test)]` spans excluded, since llvm-cov counts test code as covered and a third of this crate's instrumented lines now are test code). Verified by breaking `check_deletion` two ways: dropping its `Deleting` precondition, and requiring only one root to be a preset drop route. Both fail the property. Co-Authored-By: Claude Opus 5 (1M context) Signed-off-by: Daniel Noland --- routing/src/rib/vrf.rs | 112 ++++++++++++++++++++++++++++++++--------- 1 file changed, 87 insertions(+), 25 deletions(-) diff --git a/routing/src/rib/vrf.rs b/routing/src/rib/vrf.rs index 46b6dcb815..8abb46186d 100644 --- a/routing/src/rib/vrf.rs +++ b/routing/src/rib/vrf.rs @@ -1185,6 +1185,7 @@ mod vrf_properties { const NUM_NHOPS: u8 = 3; const MAX_CHANGES: u8 = 10; const MAX_NHOPS_PER_ROUTE: u8 = 3; + const NUM_STATUSES: u8 = 3; /// `0.0.0.0/0` and `::/0`, at these indices in [`prefixes`]. A vrf always carries a route for /// both: `Vrf::lpm` has no answer otherwise and says so with an `unreachable!()`, and @@ -1226,6 +1227,10 @@ mod vrf_properties { .collect() } + fn statuses() -> Vec { + vec![VrfStatus::Active, VrfStatus::Deleting, VrfStatus::Deleted] + } + /// The next-hops a generated route may use. Small, so that routes share them: sharing is what /// makes the store's reference counting do any work. fn nhops() -> Vec { @@ -1252,6 +1257,12 @@ mod vrf_properties { value: bool, }, RemoveStale, + /// Move the vrf's status. Worth generating because `del_route` calls `check_deletion`, + /// whose whole body sits behind `status == Deleting` -- so without this, the transition + /// that `VrfTable::remove_deleting_vrfs` depends on never runs at all. + SetStatus { + status: usize, + }, } /// Draws sequences of [`Change`]s. @@ -1271,7 +1282,7 @@ mod vrf_properties { let len = driver.gen_u8(Included(&0), Included(&MAX_CHANGES))?; let mut out = Vec::with_capacity(usize::from(len)); for _ in 0..len { - let change = match driver.gen_u8(Included(&0), Included(&3))? { + let change = match driver.gen_u8(Included(&0), Included(&4))? { 0 => { let prefix = index(driver, NUM_PREFIXES)?; let count = driver.gen_u8(Included(&0), Included(&MAX_NHOPS_PER_ROUTE))?; @@ -1287,7 +1298,10 @@ mod vrf_properties { 2 => Change::SetStale { value: driver.produce::()?, }, - _ => Change::RemoveStale, + 3 => Change::RemoveStale, + _ => Change::SetStatus { + status: index(driver, NUM_STATUSES)?, + }, }; out.push(change); } @@ -1299,29 +1313,42 @@ mod vrf_properties { /// /// The two root routes are the preset drop routes a vrf is born with; they are never removed, /// only reset, and `set_stale` skips them. + /// What the model believes about one route. + #[derive(Debug, Clone, PartialEq)] + struct RouteState { + /// The next-hop keys it names, in order. Keys rather than pool indices, because the preset + /// drop route names one the generator cannot ask for. + nhops: Vec, + stale: bool, + /// Whether this is still the drop route a vrf installs for each root on creation. + /// + /// Distinct from "names the drop next-hop": a generated route for a root prefix with no + /// next-hops ends up naming it too, but carries a real origin, distance and metric, so + /// `Route::is_preset_drop_route` says no -- and `check_deletion` asks that question. + preset: bool, + } + #[derive(Debug, Clone)] struct Model { - /// prefix index -> the next-hop keys of its route in order, and whether it is stale. - /// - /// Keys rather than pool indices, because the preset drop route names one the generator - /// cannot ask for, and because a root route holding a generated route is not the same thing - /// as a root route holding the preset one. - routes: BTreeMap, bool)>, + routes: BTreeMap, + status: VrfStatus, } impl Model { /// The route a vrf installs for each root on creation: drop, so that a lookup always has an /// answer. - fn preset() -> Vec { - vec![NhopKey::with_drop()] + fn preset() -> RouteState { + RouteState { + nhops: vec![NhopKey::with_drop()], + stale: false, + preset: true, + } } fn new() -> Self { Self { - routes: BTreeMap::from([ - (ROOT_V4, (Self::preset(), false)), - (ROOT_V6, (Self::preset(), false)), - ]), + routes: BTreeMap::from([(ROOT_V4, Self::preset()), (ROOT_V6, Self::preset())]), + status: VrfStatus::Active, } } @@ -1333,10 +1360,22 @@ mod vrf_properties { fn referenced(&self) -> BTreeSet { self.routes .values() - .flat_map(|(nhops, _)| nhops.iter().cloned()) + .flat_map(|route| route.nhops.iter().cloned()) .collect() } + /// `Vrf::check_deletion`, which `del_route` calls: a vrf on its way out becomes deletable + /// once the only routes left are the two preset drop ones. + fn check_deletion(&mut self) { + let only_presets = self.routes.len() == 2 + && [ROOT_V4, ROOT_V6] + .iter() + .all(|root| self.routes.get(root).is_some_and(|route| route.preset)); + if self.status == VrfStatus::Deleting && only_presets { + self.status = VrfStatus::Deleted; + } + } + /// The longest prefix carrying a route that covers `addr`. fn lpm(&self, addr: &IpAddr, pool: &[Prefix]) -> Option { self.routes @@ -1350,28 +1389,36 @@ mod vrf_properties { match change { Change::AddRoute { prefix, nhops } => { // a route with no next-hops is installed as a drop rather than left out - let keys = if nhops.is_empty() { - Self::preset() + let nhops = if nhops.is_empty() { + vec![NhopKey::with_drop()] } else { nhops.iter().map(|i| pool[*i].key.clone()).collect() }; - self.routes.insert(*prefix, (keys, false)); + self.routes.insert( + *prefix, + RouteState { + nhops, + stale: false, + preset: false, + }, + ); } Change::DelRoute { prefix } => { if Self::is_root(*prefix) { // a root route is reset to the preset drop route, not removed - self.routes.insert(*prefix, (Self::preset(), false)); + self.routes.insert(*prefix, Self::preset()); } else { self.routes.remove(prefix); } + self.check_deletion(); } Change::SetStale { value } => { // `Vrf::set_stale` skips a route whose prefix is a root, and separately one // that is still a preset drop route. Generated routes are never the latter, so // here the root test is the whole of it. - for (prefix, (_, stale)) in &mut self.routes { + for (prefix, route) in &mut self.routes { if !Self::is_root(*prefix) { - *stale = *value; + route.stale = *value; } } } @@ -1379,12 +1426,16 @@ mod vrf_properties { let stale: Vec = self .routes .iter() - .filter_map(|(prefix, (_, stale))| stale.then_some(*prefix)) + .filter_map(|(prefix, route)| route.stale.then_some(*prefix)) .collect(); for prefix in stale { self.apply(&Change::DelRoute { prefix }, pool); } } + Change::SetStatus { status } => { + // the vrf under test is not the default one, so the move always takes + self.status = statuses()[*status]; + } } } } @@ -1400,6 +1451,7 @@ mod vrf_properties { Change::DelRoute { prefix } => vrf.del_route(prefixes[*prefix], None, rstore), Change::SetStale { value } => vrf.set_stale(*value), Change::RemoveStale => vrf.remove_stale_routes(None, rstore), + Change::SetStatus { status } => vrf.set_status(statuses()[*status]), } } @@ -1421,15 +1473,24 @@ mod vrf_properties { ); // 2. each route names the next-hops the model says, in order - for (prefix, (nhops, stale)) in &model.routes { + for (prefix, want) in &model.routes { let route = vrf .get_route(prefixes[*prefix]) .unwrap_or_else(|| panic!("no route for {prefix} {at}")); let got: Vec = route.s_nhops.iter().map(|s| s.rc.key.clone()).collect(); - assert_eq!(got, *nhops, "next-hops of {prefix} {at}"); - assert_eq!(route.is_stale(), *stale, "stale flag of {prefix} {at}"); + assert_eq!(got, want.nhops, "next-hops of {prefix} {at}"); + assert_eq!(route.is_stale(), want.stale, "stale flag of {prefix} {at}"); + assert_eq!( + route.is_preset_drop_route(), + want.preset, + "preset-drop-route of {prefix} {at}" + ); } + // the status, and with it `check_deletion`: a vrf on its way out becomes deletable exactly + // when the only routes left are the two preset drop ones + assert_eq!(vrf.status, model.status, "status {at}"); + // 3. the next-hop store holds exactly the next-hops the routes name. One too many is a // leak that keeps a stale fib group alive; one too few and a route names something // nothing will resolve @@ -1469,6 +1530,7 @@ mod vrf_properties { fn the_pools_are_the_size_the_generator_thinks() { assert_eq!(prefixes().len(), usize::from(NUM_PREFIXES)); assert_eq!(nhops().len(), usize::from(NUM_NHOPS)); + assert_eq!(statuses().len(), usize::from(NUM_STATUSES)); assert_eq!(prefixes()[ROOT_V4], Prefix::root_v4()); assert_eq!(prefixes()[ROOT_V6], Prefix::root_v6()); } From 97ca8019f8a3214a23d2bd9c5eb11cf1f5445bf4 Mon Sep 17 00:00:00 2001 From: Daniel Noland Date: Fri, 7 Aug 2026 14:30:34 -0600 Subject: [PATCH 27/33] test(routing): Property-test resolution across vrfs `VrfTable::refresh_non_default_fibs` and `refresh_fibs_by_vni` hand the default vrf to every other vrf as its resolution vrf. That is how a next-hop in an overlay vrf reaches an interface the underlay knows about, and it was the one part of next-hop resolution nothing had exercised: those two functions, `set_stale`, `remove_stale_routes` and `values_mut_except_default` were all at zero coverage, and every generated next-hop graph so far had lived inside a single vrf. Three properties over a generated topology -- vrfs with and without vnis, directly connected routes in the default vrf, recursive routes in the others: - a refresh resolves every other vrf's next-hops through the default vrf. The interface has to come from the default vrf's route, and the address has to stay that of the next-hop being resolved -- unless the resolver has an address of its own, in which case that one wins. The oracle is a longest-prefix match over the generated route list, which asks no next-hop anything. - `refresh_fibs_by_vni` refreshes the vrfs whose vni is named and leaves the rest untouched, checked by changing the underlay between a full refresh and a selective one and comparing against a snapshot. - marking everything stale and sweeping leaves every vrf holding only its two preset drop routes, the default vrf included -- it is swept separately from the rest, since it is their resolution vrf. All three also assert the rib-to-fib contract over the whole table: every entry in every fib is one the forwarder can execute. `FibEntry::is_valid` is the written-down half of that and `rib2fib` filters on it, but the drop injected for an empty group bypasses the filter, and nothing had checked the table at once. Verified by breaking three things: resolving each vrf against itself rather than the default vrf, dropping the vni filter, and skipping the default vrf in the stale sweep. Each fails a different property. The address half of `EgressObject::merge`'s rule -- first interface, last address -- is now pinned end to end, in the situation `rib2fib` describes it for: a next-hop supplies the layer-2 target "unless a next-hop deeper in the resolution chain provides one of its own". That needed the generator to draw underlay next-hops both with and without an on-link address of their own; without that, inverting the rule changes nothing observable. The interface half is not observable this way, and cannot be: the next-hop being resolved has no interface -- that is why it is being resolved -- and one that has an interface is not resolved further, so a chain never holds two. "First non-none" and "last some" therefore agree on every chain `lazy_resolve` will build. It stays covered by `squash_properties` at the unit level, which is where the distinction is visible. Production coverage of `rib/vrftable.rs` goes 73.1% -> 87.0%, and `routing/src` as a whole 58.8% -> 60.1% (production lines only; `#[cfg(test)]` spans excluded). Co-Authored-By: Claude Opus 5 (1M context) Signed-off-by: Daniel Noland --- routing/src/rib/vrftable.rs | 468 ++++++++++++++++++++++++++++++++++++ 1 file changed, 468 insertions(+) diff --git a/routing/src/rib/vrftable.rs b/routing/src/rib/vrftable.rs index 6221334a96..b1110fdecf 100644 --- a/routing/src/rib/vrftable.rs +++ b/routing/src/rib/vrftable.rs @@ -1264,3 +1264,471 @@ mod vrftable_properties { }); } } + +/// Properties over resolution **across** vrfs. +/// +/// `VrfTable::refresh_non_default_fibs` and `refresh_fibs_by_vni` hand the default vrf to every +/// other vrf as its resolution vrf, which is how a next-hop in an overlay vrf reaches an interface +/// the underlay knows about. Nothing exercised that path: both functions, `set_stale`, +/// `remove_stale_routes` and `values_mut_except_default` were at zero coverage, and every generated +/// next-hop graph so far has lived inside a single vrf. +#[cfg(test)] +mod crossvrf_properties { + use super::*; + use crate::fib::fibobjects::{EgressObject, FibEntry, PktInstruction}; + use crate::rib::vrf::tests::{build_test_nhop, build_test_route, mk_addr}; + use crate::rib::vrf::{Route, RouteNhop, RouteOrigin}; + use bolero::{Driver, ValueGenerator}; + use lpm::prefix::Prefix; + use net::interface::InterfaceIndex; + use std::collections::BTreeMap; + use std::net::IpAddr; + use std::ops::Bound::Included; + use std::str::FromStr; + + const NUM_VRFS: u8 = 2; // non-default vrfs + const NUM_VNIS: u8 = 2; + const NUM_UNDERLAY: u8 = 2; + const NUM_OVERLAY: u8 = 2; + const NUM_IFINDEXES: u8 = 3; + const NUM_VIAS: u8 = 3; + const MAX_ROUTES: u8 = 3; + + /// Non-default vrf ids. The default vrf is 0 and `VrfTable::new` makes it. + fn vrf_ids() -> Vec { + (1..=u32::from(NUM_VRFS)).collect() + } + + fn vnis() -> Vec { + (1..=u32::from(NUM_VNIS)) + .map(|i| Vni::new_checked(100 * i).unwrap_or_else(|_| unreachable!())) + .collect() + } + + /// Prefixes the default vrf carries routes for. Nested, so a longest match has to be chosen. + fn underlay() -> Vec { + ["7.0.0.0/8", "7.1.0.0/16"] + .iter() + .map(|p| Prefix::from_str(p).unwrap_or_else(|_| unreachable!())) + .collect() + } + + /// Prefixes the other vrfs carry routes for. + fn overlay() -> Vec { + ["10.0.0.0/8", "10.1.0.0/16"] + .iter() + .map(|p| Prefix::from_str(p).unwrap_or_else(|_| unreachable!())) + .collect() + } + + fn ifindexes() -> Vec { + (1..=u32::from(NUM_IFINDEXES)).collect() + } + + /// On-link addresses a directly connected underlay next-hop may carry, one per interface. + /// + /// Whether the resolving next-hop has an address of its own is what decides which address ends + /// up in the fib, and `rib2fib` is explicit that the deeper one wins: the next-hop being + /// resolved supplies the layer-2 target "unless a next-hop deeper in the resolution chain + /// provides one of its own". Generating both cases is what lets this property tell + /// `EgressObject::merge`'s rule -- first ifindex, last address -- from its inverse. + fn onlink_addrs() -> Vec { + (1..=NUM_IFINDEXES) + .map(|i| mk_addr(&format!("7.200.0.{i}"))) + .collect() + } + + /// Addresses an overlay route may point via. `8.0.0.1` is covered by no underlay prefix, so it + /// reaches only the default vrf's root -- whose preset next-hop is a drop. + fn vias() -> Vec { + ["7.0.0.1", "7.1.0.1", "8.0.0.1"] + .iter() + .map(|a| mk_addr(a)) + .collect() + } + + /// A generated arrangement of vrfs and routes. + #[derive(Debug, Clone)] + struct Topology { + /// Whether each non-default vrf carries a vni. Vrf `i` gets vni `i`, so two vrfs can never + /// ask for the same one -- `add_vrf` rightly refuses that, and it is the `VrfTable` + /// property's business rather than this one's. + vrfs: Vec, + /// Routes in the default vrf, as (underlay prefix index, ifindex index, on-link). Directly + /// connected: the next-hop carries an interface, and an address of its own when `on-link`. + underlay: Vec<(usize, usize, bool)>, + /// Routes in the other vrfs, as (vrf index, overlay prefix index, via index). Recursive: + /// the next-hop carries an address and no interface, so it has to be resolved. + overlay: Vec<(usize, usize, usize)>, + /// One more default-vrf route, applied after the first refresh, so that a later refresh has + /// something to notice. + later: Option<(usize, usize, bool)>, + /// Vnis to pass to `refresh_fibs_by_vni`. + selected: Vec, + } + + /// Draws [`Topology`]s. + #[derive(Debug, Clone, Copy, Default)] + struct Topologies; + + fn index(driver: &mut D, count: u8) -> Option { + driver + .gen_u8(Included(&0), Included(&(count - 1))) + .map(usize::from) + } + + impl ValueGenerator for Topologies { + type Output = Topology; + + fn generate(&self, driver: &mut D) -> Option { + let mut vrfs = Vec::with_capacity(usize::from(NUM_VRFS)); + for _ in 0..NUM_VRFS { + vrfs.push(driver.produce::()?); + } + + let count = driver.gen_u8(Included(&0), Included(&MAX_ROUTES))?; + let mut underlay = Vec::with_capacity(usize::from(count)); + for _ in 0..count { + underlay.push(( + index(driver, NUM_UNDERLAY)?, + index(driver, NUM_IFINDEXES)?, + driver.produce::()?, + )); + } + + let count = driver.gen_u8(Included(&0), Included(&MAX_ROUTES))?; + let mut overlay = Vec::with_capacity(usize::from(count)); + for _ in 0..count { + overlay.push(( + index(driver, NUM_VRFS)?, + index(driver, NUM_OVERLAY)?, + index(driver, NUM_VIAS)?, + )); + } + + let later = if driver.produce::()? { + Some(( + index(driver, NUM_UNDERLAY)?, + index(driver, NUM_IFINDEXES)?, + driver.produce::()?, + )) + } else { + None + }; + + let count = driver.gen_u8(Included(&0), Included(&NUM_VNIS))?; + let mut selected = Vec::with_capacity(usize::from(count)); + for _ in 0..count { + selected.push(index(driver, NUM_VNIS)?); + } + + Some(Topology { + vrfs, + underlay, + overlay, + later, + selected, + }) + } + } + + /// The default vrf's routes, as the model sees them: prefix index -> (ifindex index, on-link). + /// A later route for a prefix replaces an earlier one, as the trie does. + type Underlay = BTreeMap; + + /// The other vrfs' routes: (vrf index, overlay prefix index) -> via index. A map for the same + /// reason: the generator may name one prefix twice, and only the last route survives. + type Overlay = BTreeMap<(usize, usize), usize>; + + /// The interface the default vrf leads to for `via`, or `None` if `via` reaches only the root -- + /// whose preset next-hop is a drop, so nothing is reachable through it. + /// + /// A longest-prefix match over the generated route list, asking no next-hop anything. + fn resolves_to(model: &Underlay, via: usize) -> Option<(u32, bool)> { + let address = vias()[via]; + let prefixes = underlay(); + model + .iter() + .filter(|(prefix, _)| prefixes[**prefix].covers_addr(&address)) + .max_by_key(|(prefix, _)| prefixes[**prefix].length()) + .map(|(_, (ifindex, onlink))| (ifindexes()[*ifindex], *onlink)) + } + + /// What the fib should hold for an overlay route pointing via `via`. + /// + /// One entry, and the shape of it is the whole point of resolving recursively: the interface + /// comes from the *resolver* in the default vrf, and the address stays that of the next-hop that + /// needed resolving. `EgressObject::merge` keeping the first ifindex and the last address is + /// what produces that, and `rib2fib` explains why it must -- otherwise the egress stage would + /// resolve the packet's own destination at layer 2, which is only right when it is on-link. + fn expected_entry(model: &Underlay, via: usize) -> FibEntry { + match resolves_to(model, via) { + // the interface comes from the resolver; the address is the resolver's own if it has + // one, and otherwise stays that of the next-hop that needed resolving + Some((ifindex, onlink)) => { + let index = usize::try_from(ifindex).unwrap_or_else(|_| unreachable!()) - 1; + let address = if onlink { + onlink_addrs()[index] + } else { + vias()[via] + }; + FibEntry::with_inst(PktInstruction::Egress(EgressObject::new( + InterfaceIndex::try_new(ifindex).ok(), + Some(address), + None, + ))) + } + None => FibEntry::drop_fibentry(), + } + } + + fn underlay_route(ifindex: usize, onlink: bool) -> (Route, Vec) { + let address = onlink.then(|| onlink_addrs()[ifindex].to_string()); + ( + build_test_route(RouteOrigin::Connected, 0, 0), + vec![build_test_nhop( + address.as_deref(), + Some(ifindexes()[ifindex]), + 0, + None, + )], + ) + } + + fn overlay_route(via: usize) -> (Route, Vec) { + ( + build_test_route(RouteOrigin::Bgp, 20, 100), + vec![build_test_nhop( + Some(&vias()[via].to_string()), + None, + 0, + None, + )], + ) + } + + /// Build the table described by `topology`, without refreshing anything yet. + fn realize(topology: &Topology, rstore: &RmacStore) -> (VrfTable, Underlay, Overlay) { + let (fibtw, _fibtr) = FibTableWriter::new(); + let mut table = VrfTable::new(fibtw); + let ids = vrf_ids(); + let all_vnis = vnis(); + + for (vrf, has_vni) in topology.vrfs.iter().enumerate() { + let config = RouterVrfConfig::new(ids[vrf], &format!("vrf{vrf}")) + .set_vni(has_vni.then(|| all_vnis[vrf])); + table + .add_vrf(&config) + .unwrap_or_else(|e| unreachable!("{e}")); + } + + let mut model = Underlay::new(); + for (prefix, ifindex, onlink) in &topology.underlay { + let (route, nhops) = underlay_route(*ifindex, *onlink); + let vrf0 = table + .get_vrf_mut(Vrf::DEFAULT_VRFID) + .unwrap_or_else(|e| unreachable!("{e}")); + vrf0.add_route_complete(&underlay()[*prefix], route, &nhops, None, rstore); + model.insert(*prefix, (*ifindex, *onlink)); + } + + let mut overlay_model = Overlay::new(); + for (vrf, prefix, via) in &topology.overlay { + let (route, nhops) = overlay_route(*via); + // deliberately inserted with no resolution vrf, so the route starts resolved against + // its own vrf -- where the address reaches nothing. Only the table-level refresh can + // put it right, which is what makes the refresh load-bearing here. + let target = table + .get_vrf_mut(ids[*vrf]) + .unwrap_or_else(|e| unreachable!("{e}")); + target.add_route_complete(&overlay()[*prefix], route, &nhops, None, rstore); + overlay_model.insert((*vrf, *prefix), *via); + } + + (table, model, overlay_model) + } + + /// The fib entries a vrf offers for one overlay prefix, or `None` if it has no such route. + fn fib_entries(table: &VrfTable, vrfid: VrfId, prefix: Prefix) -> Option> { + let vrf = table.get_vrf(vrfid).unwrap_or_else(|e| unreachable!("{e}")); + let fibw = vrf.fibw.as_ref().unwrap_or_else(|| unreachable!()); + let fib = fibw.enter().unwrap_or_else(|| unreachable!()); + let Prefix::IPV4(wanted) = prefix else { + unreachable!() + }; + fib.iter_v4().find(|(p, _)| *p == wanted).map(|(_, route)| { + route + .iter() + .flat_map(|group| group.entries().iter().cloned()) + .collect() + }) + } + + /// Every entry in every fib of every vrf is one the forwarder can execute. + /// + /// This is the rib-to-fib contract: `FibEntry::is_valid` is the written-down half of it, and + /// `rib2fib` filters on it -- but the drop injected for an empty group bypasses that filter, and + /// nothing checked the whole table at once. + fn every_entry_is_executable(table: &VrfTable, at: &str) { + for vrf in table.values() { + let fibw = vrf.fibw.as_ref().unwrap_or_else(|| unreachable!()); + let fib = fibw.enter().unwrap_or_else(|| unreachable!()); + for (prefix, route) in fib.iter_v4() { + for group in route.iter() { + assert!(!group.is_empty(), "empty group for {prefix} {at}"); + for entry in group.iter() { + assert!( + entry.is_valid(), + "vrf {} offers unusable {entry:?} for {prefix} {at}", + vrf.vrfid + ); + } + } + } + } + } + + /// The pools and the constants that index them agree. + #[test] + fn the_pools_are_the_size_the_generator_thinks() { + assert_eq!(vrf_ids().len(), usize::from(NUM_VRFS)); + assert_eq!(vnis().len(), usize::from(NUM_VNIS)); + assert_eq!(underlay().len(), usize::from(NUM_UNDERLAY)); + assert_eq!(overlay().len(), usize::from(NUM_OVERLAY)); + assert_eq!(ifindexes().len(), usize::from(NUM_IFINDEXES)); + assert_eq!(vias().len(), usize::from(NUM_VIAS)); + // vrf `i` takes vni `i`, so there must be at least as many vnis as vrfs + const { assert!(NUM_VNIS >= NUM_VRFS) }; + // the last via must be reachable through no underlay prefix, so that the unresolvable case + // is generated + assert_eq!(onlink_addrs().len(), usize::from(NUM_IFINDEXES)); + let all: Underlay = (0..usize::from(NUM_UNDERLAY)) + .map(|p| (p, (0, false))) + .collect(); + assert!(resolves_to(&all, usize::from(NUM_VIAS) - 1).is_none()); + } + + /// A refresh resolves every other vrf's next-hops through the default vrf. + /// + /// The interface has to come from the default vrf's route, and the address has to stay that of + /// the next-hop being resolved. + #[test] + fn a_refresh_resolves_other_vrfs_through_the_default_one() { + let rstore = RmacStore::new(); + bolero::check!() + .with_generator(Topologies) + .cloned() + .for_each(|topology: Topology| { + let (mut table, model, routes) = realize(&topology, &rstore); + table.refresh_non_default_fibs(&rstore); + every_entry_is_executable(&table, "after a refresh"); + + let ids = vrf_ids(); + for ((vrf, prefix), via) in &routes { + let got = fib_entries(&table, ids[*vrf], overlay()[*prefix]) + .unwrap_or_else(|| panic!("no fib route for {prefix} in vrf {vrf}")); + assert_eq!( + got, + vec![expected_entry(&model, *via)], + "vrf {vrf} prefix {prefix} via {via}, for {topology:?}" + ); + } + }); + } + + /// `refresh_fibs_by_vni` refreshes the vrfs whose vni is named, and leaves the rest alone. + #[test] + fn refreshing_by_vni_touches_only_those_vnis() { + let rstore = RmacStore::new(); + bolero::check!() + .with_generator(Topologies) + .cloned() + .for_each(|topology: Topology| { + let Some(later) = topology.later else { return }; + let (mut table, mut model, routes) = realize(&topology, &rstore); + table.refresh_non_default_fibs(&rstore); + + let ids = vrf_ids(); + let all_vnis = vnis(); + + // what each overlay route offers before anything changes + let before: BTreeMap<(usize, usize), Option>> = routes + .keys() + .map(|(vrf, prefix)| { + ( + (*vrf, *prefix), + fib_entries(&table, ids[*vrf], overlay()[*prefix]), + ) + }) + .collect(); + + // change the underlay, then refresh only the named vnis + let (prefix, ifindex, onlink) = later; + let (route, nhops) = underlay_route(ifindex, onlink); + let vrf0 = table + .get_vrf_mut(Vrf::DEFAULT_VRFID) + .unwrap_or_else(|e| unreachable!("{e}")); + vrf0.add_route_complete(&underlay()[prefix], route, &nhops, None, &rstore); + model.insert(prefix, (ifindex, onlink)); + + let selected: Vec = topology.selected.iter().map(|i| all_vnis[*i]).collect(); + table.refresh_fibs_by_vni(&selected, &rstore); + every_entry_is_executable(&table, "after refreshing by vni"); + + for ((vrf, prefix), via) in &routes { + let got = fib_entries(&table, ids[*vrf], overlay()[*prefix]); + let vni = table + .get_vrf(ids[*vrf]) + .unwrap_or_else(|e| unreachable!("{e}")) + .vni; + if vni.is_some_and(|vni| selected.contains(&vni)) { + assert_eq!( + got, + Some(vec![expected_entry(&model, *via)]), + "refreshed vrf {vrf} prefix {prefix}, for {topology:?}" + ); + } else { + assert_eq!( + got, + before[&(*vrf, *prefix)], + "untouched vrf {vrf} prefix {prefix}, for {topology:?}" + ); + } + } + }); + } + + /// Marking everything stale and sweeping leaves every vrf with only its preset drop routes. + /// + /// The default vrf is swept separately from the rest, since it is the resolution vrf for them. + #[test] + fn a_stale_sweep_empties_every_vrf() { + let rstore = RmacStore::new(); + bolero::check!() + .with_generator(Topologies) + .cloned() + .for_each(|topology: Topology| { + let (mut table, _model, _routes) = realize(&topology, &rstore); + table.refresh_non_default_fibs(&rstore); + + table.set_stale(true); + table.remove_stale_routes(&rstore); + + for vrf in table.values() { + assert_eq!(vrf.len_v4(), 1, "vrf {} kept ipv4 routes", vrf.vrfid); + assert_eq!(vrf.len_v6(), 1, "vrf {} kept ipv6 routes", vrf.vrfid); + for prefix in [Prefix::root_v4(), Prefix::root_v6()] { + let route = vrf + .get_route(prefix) + .unwrap_or_else(|| panic!("vrf {} lost {prefix}", vrf.vrfid)); + assert!( + route.is_preset_drop_route(), + "vrf {} left {prefix} as something other than the preset drop route", + vrf.vrfid + ); + } + } + every_entry_is_executable(&table, "after a stale sweep"); + }); + } +} From 7d23fcf7628db0467eb8020ab5e6e34650e8309d Mon Sep 17 00:00:00 2001 From: Daniel Noland Date: Fri, 7 Aug 2026 14:46:23 -0600 Subject: [PATCH 28/33] fix(routing): Keep the interface name out of the next-hop key `RouteNhop::from_rpc_nhop` looked the interface name up from `ifindex` against the interface table and put it in the `NhopKey`. The interface table is populated out of band from the routes, so the same next-hop off the wire keyed one way before we learned about its interface and another way after -- two `Nhop`s, two fib groups, two resolutions, for one next-hop. The file already argues against exactly this, thirty lines above, about the other derived field it could have put in the key: // Note: dmac is not set in nhops, because it may not be known when the // next-hop is added and the encapsulation is part of the next-hop key // which should be immutable for keying purposes. `ifname` is that: derived from `ifindex`, not known when the next-hop arrives, and part of the key. It also distinguishes nothing -- `NhopKey` is documented as holding "the properties that make a shared next-hop unique", and a name derived from an index that is already in the key is not one of them. So the lookup is gone, and with it the interface table argument to `from_rpc_nhop` and `add_route_rpc`. That is the part worth having: the invariant is now carried by the signature rather than by a test, since the conversion has no interface table to depend on. `IfTableWriter::as_reader` goes too -- the CPI route path was its only caller, which is itself evidence the dependency was only ever for the name. The cost is that `ifname` is now never populated in production. It reaches only a per-packet `debug!` in the forwarder, which already prints the ifindex, so nothing observable is lost -- but the field, and `EgressObject`'s copy of it, are vestigial. Removing them properly means touching `NhopKey`, `EgressObject`, `EgressObject::merge` and its property; worth doing, but as its own change. Found while giving `router/rpc_adapt.rs` its first tests. It translates `IpRoute`s and next-hops arriving from FRR over the CPI into routing state -- external input, the only production path into `Vrf::add_route_complete`, and it was at zero coverage. Four properties, with the wire message as the oracle: - a next-hop is refused for exactly the four reasons it should be (interface index zero, a vni that is not one, a vxlan next-hop that does not say which vtep to send to, and a forwarding next-hop with neither interface nor address), and otherwise yields the key the message describes - a route is installed with the origin, distance, metric and surviving next-hops the message describes -- or a drop next-hop if none survived - a prefix that cannot be parsed installs nothing - deleting the route the message names removes it and leaves the next-hop store holding only what the root routes need Verified by breaking six things: accepting interface index zero, keeping the ifindex on a vxlan next-hop, accepting a vxlan next-hop with no vtep address, dropping the connected-host-becomes-local rule, accepting a forwarding next-hop with nowhere to send, and installing a route for an unparseable prefix. Each fails. `router/rpc_adapt.rs` goes 0% -> 88.7% production coverage, `rib/rib2fib.rs` 90.7% -> 96.9% (the rpc path reaches its local-route branch), and `routing/src` as a whole 60.1% -> 61.4%. Co-Authored-By: Claude Opus 5 (1M context) Signed-off-by: Daniel Noland --- routing/src/interfaces/iftablerw.rs | 4 - routing/src/rib/nexthop.rs | 6 + routing/src/router/cpi.rs | 5 +- routing/src/router/rpc_adapt.rs | 420 ++++++++++++++++++++++++++-- 4 files changed, 404 insertions(+), 31 deletions(-) diff --git a/routing/src/interfaces/iftablerw.rs b/routing/src/interfaces/iftablerw.rs index 29f231098e..db9c641424 100644 --- a/routing/src/interfaces/iftablerw.rs +++ b/routing/src/interfaces/iftablerw.rs @@ -80,10 +80,6 @@ impl IfTableWriter { (IfTableWriter(w), IfTableReader(r)) } #[must_use] - pub fn as_reader(&self) -> IfTableReader { - IfTableReader::new(self.0.clone()) - } - #[must_use] pub fn enter(&self) -> Option> { self.0.enter() } diff --git a/routing/src/rib/nexthop.rs b/routing/src/rib/nexthop.rs index 0d48ebe2fd..4196ffccc1 100644 --- a/routing/src/rib/nexthop.rs +++ b/routing/src/rib/nexthop.rs @@ -60,6 +60,12 @@ pub struct NhopKey { pub ifindex: Option, pub encap: Option, pub fwaction: FwAction, + /// The name of `ifindex`'s interface, for diagnostics only. + /// + /// Not populated when a next-hop is learned over the CPI, and it must not be: the name has to + /// be looked up against the interface table, which is populated out of band, so a key carrying + /// one would differ before and after we learn about the interface -- two next-hops where there + /// is one. See `RouteNhop::from_rpc_nhop`, and the same argument for a vxlan dmac above it. pub ifname: Option, } diff --git a/routing/src/router/cpi.rs b/routing/src/router/cpi.rs index 68d9d0a220..b8fe075dda 100644 --- a/routing/src/router/cpi.rs +++ b/routing/src/router/cpi.rs @@ -209,14 +209,13 @@ impl RpcOperation for IpRoute { fn add(&self, db: &mut Self::ObjectStore) -> RpcResultCode { let rmac_store = &db.rmac_store; let vrftable = &mut db.vrftable; - let iftabler = &db.iftw.as_reader(); if self.vrfid == Vrf::DEFAULT_VRFID { let Ok(vrf0) = vrftable.get_vrf_mut(self.vrfid) else { error!("Unable to find default VRF!"); return RpcResultCode::Failure; }; - vrf0.add_route_rpc(self, None, rmac_store, iftabler); + vrf0.add_route_rpc(self, None, rmac_store); vrftable.refresh_non_default_fibs(rmac_store); } else { // this assumes that we always resolve non-default vrfs with the default vrf @@ -229,7 +228,7 @@ impl RpcOperation for IpRoute { error!("Unable to get vrf with id {}", self.vrfid); return RpcResultCode::Failure; }; - vrf.add_route_rpc(self, Some(vrf0), rmac_store, iftabler); + vrf.add_route_rpc(self, Some(vrf0), rmac_store); } RpcResultCode::Ok } diff --git a/routing/src/router/rpc_adapt.rs b/routing/src/router/rpc_adapt.rs index 03bfbba33c..b735b27522 100644 --- a/routing/src/router/rpc_adapt.rs +++ b/routing/src/router/rpc_adapt.rs @@ -11,7 +11,6 @@ use crate::errors::RouterError; use crate::evpn::{RmacEntry, RmacStore}; -use crate::interfaces::iftablerw::IfTableReader; use crate::rib::encapsulation::{Encapsulation, VxlanEncapsulation}; use crate::rib::nexthop::{FwAction, NhopKey}; use crate::rib::vrf::{Route, RouteFlags, RouteNhop, RouteOrigin, Vrf}; @@ -100,11 +99,7 @@ impl TryFrom<&Rmac> for RmacEntry { impl RouteNhop { #[tracing::instrument(level = "debug")] - fn from_rpc_nhop( - nh: &NextHop, - origin: RouteOrigin, - iftabler: &IfTableReader, - ) -> Result { + fn from_rpc_nhop(nh: &NextHop, origin: RouteOrigin) -> Result { let mut ifindex = nh .ifindex .map(|i| match InterfaceIndex::try_new(i) { @@ -129,22 +124,21 @@ impl RouteNhop { None => None, }; - // lookup interface name - let ifname = match ifindex { - None => None, - Some(k) => iftabler - .enter() - .and_then(|iftable| iftable.get_interface(k).map(|iface| iface.name.clone())), - }; - - // build key for this next hop + // build key for this next hop. + // + // No interface name: it would have to be looked up from `ifindex` against the interface + // table, which is populated out of band, so the same next-hop off the wire would key + // differently before and after we learn about the interface -- two `Nhop`s, two fib groups, + // for one next-hop. This is the same reasoning that keeps a vxlan dmac out of the key, + // written down above: a next-hop key has to be immutable for keying purposes, so nothing + // derived from mutable state outside it belongs in one. let key = NhopKey::new( origin, nh.address, ifindex, encap, FwAction::from(nh.fwaction), - ifname, + None, ); // validate next hop from its key @@ -180,13 +174,7 @@ impl Route { } impl Vrf { - pub fn add_route_rpc( - &mut self, - iproute: &IpRoute, - vrf0: Option<&Vrf>, - rstore: &RmacStore, - iftabler: &IfTableReader, - ) { + pub fn add_route_rpc(&mut self, iproute: &IpRoute, vrf0: Option<&Vrf>, rstore: &RmacStore) { let prefix = match Prefix::try_from((iproute.prefix, iproute.prefix_len)) { Ok(p) => p, Err(e) => { @@ -216,7 +204,7 @@ impl Vrf { let route = Route::from_iproute(&prefix, iproute); let mut nhops = Vec::with_capacity(iproute.nhops.len()); for nhop in &iproute.nhops { - match RouteNhop::from_rpc_nhop(nhop, route.origin, iftabler) { + match RouteNhop::from_rpc_nhop(nhop, route.origin) { Ok(nh) => nhops.push(nh), Err(e) => error!("Omitting next-hop {nhop} in route to {prefix}: {e}"), } @@ -245,3 +233,387 @@ impl Vrf { self.del_route(prefix, vrf0, rstore); } } + +/// Properties over the translation from control-plane messages into routing state. +/// +/// This is where the routing stack parses input it does not control: `IpRoute`s and their next-hops +/// arrive from FRR over the CPI, and everything downstream is built from whatever this module makes +/// of them. It had no test coverage at all. +/// +/// The oracle throughout is the wire message: what the key should hold, and which next-hops should +/// be refused, worked out from the fields rather than by rerunning the conversion. +#[cfg(test)] +mod rpc_properties { + use super::*; + use crate::fib::fibtype::{FibKey, FibWriter}; + use crate::rib::vrf::RouterVrfConfig; + use bolero::{Driver, ValueGenerator}; + use dplane_rpc::proto::{Ifindex, MaskLen, VrfId}; + use std::net::IpAddr; + use std::ops::Bound::Included; + use std::str::FromStr; + + const NUM_PREFIXES: u8 = 5; + const NUM_ADDRESSES: u8 = 3; + const NUM_IFINDEXES: u8 = 4; + const NUM_VNIS: u8 = 3; + const NUM_RTYPES: u8 = 7; + const MAX_NHOPS: u8 = 3; + + /// The prefix of the last entry in [`prefixes`] cannot be built, so the "message names a prefix + /// we cannot parse" path is generated. + const BAD_PREFIX: usize = 4; + + /// `(address, mask length)` pairs as they arrive on the wire, including one that is not a + /// prefix at all. + fn prefixes() -> Vec<(IpAddr, MaskLen)> { + vec![ + ( + IpAddr::from_str("10.0.0.0").unwrap_or_else(|_| unreachable!()), + 8, + ), + ( + IpAddr::from_str("10.1.0.0").unwrap_or_else(|_| unreachable!()), + 16, + ), + // a host prefix, which turns a connected route into a local one + ( + IpAddr::from_str("10.1.2.3").unwrap_or_else(|_| unreachable!()), + 32, + ), + ( + IpAddr::from_str("2001:db8::").unwrap_or_else(|_| unreachable!()), + 32, + ), + // 33 bits of an ipv4 address: no such prefix + ( + IpAddr::from_str("10.0.0.0").unwrap_or_else(|_| unreachable!()), + 33, + ), + ] + } + + /// Next-hop addresses. `None` is on the wire too, for an interface-only next-hop or a drop. + fn addresses() -> Vec> { + vec![ + None, + Some(IpAddr::from_str("10.0.0.1").unwrap_or_else(|_| unreachable!())), + Some(IpAddr::from_str("7.0.0.1").unwrap_or_else(|_| unreachable!())), + ] + } + + /// Next-hop interface indices, as raw wire values: absent, the invalid zero, one the interface + /// table knows, and one it does not. + fn ifindexes() -> Vec> { + vec![None, Some(0), Some(2), Some(99)] + } + + /// Encapsulation vnis: absent, the invalid zero, and a usable one. + fn vnis() -> Vec> { + vec![None, Some(0), Some(3000)] + } + + fn rtypes() -> Vec { + vec![ + RouteType::Local, + RouteType::Connected, + RouteType::Static, + RouteType::Ospf, + RouteType::Isis, + RouteType::Bgp, + RouteType::Other, + ] + } + + /// One next-hop as it arrives, over indices into the pools above. + #[derive(Debug, Clone)] + struct NhopSpec { + drop: bool, + address: usize, + ifindex: usize, + vni: usize, + vrfid: VrfId, + } + + /// One route as it arrives. + #[derive(Debug, Clone)] + struct RouteSpec { + prefix: usize, + rtype: usize, + distance: u8, + metric: u32, + nhops: Vec, + } + + /// Draws [`RouteSpec`]s. + #[derive(Debug, Clone, Copy, Default)] + struct Routes; + + fn index(driver: &mut D, count: u8) -> Option { + driver + .gen_u8(Included(&0), Included(&(count - 1))) + .map(usize::from) + } + + impl ValueGenerator for Routes { + type Output = RouteSpec; + + fn generate(&self, driver: &mut D) -> Option { + let prefix = index(driver, NUM_PREFIXES)?; + let rtype = index(driver, NUM_RTYPES)?; + let distance = driver.produce::()?; + let metric = driver.produce::()?; + // deliberately able to draw none: a route with no next-hops is on the wire, and the + // comment in `add_route_rpc` is about exactly that + let count = driver.gen_u8(Included(&0), Included(&MAX_NHOPS))?; + let mut nhops = Vec::with_capacity(usize::from(count)); + for _ in 0..count { + nhops.push(NhopSpec { + drop: driver.produce::()?, + address: index(driver, NUM_ADDRESSES)?, + ifindex: index(driver, NUM_IFINDEXES)?, + vni: index(driver, NUM_VNIS)?, + vrfid: 0, + }); + } + Some(RouteSpec { + prefix, + rtype, + distance, + metric, + nhops, + }) + } + } + + fn wire_nhop(spec: &NhopSpec) -> NextHop { + NextHop { + fwaction: if spec.drop { + ForwardAction::Drop + } else { + ForwardAction::Forward + }, + address: addresses()[spec.address], + ifindex: ifindexes()[spec.ifindex], + vrfid: spec.vrfid, + encap: vnis()[spec.vni].map(|vni| NextHopEncap::VXLAN(VxlanEncap { vni })), + } + } + + fn wire_route(spec: &RouteSpec) -> IpRoute { + let (prefix, prefix_len) = prefixes()[spec.prefix]; + IpRoute { + prefix, + prefix_len, + vrfid: 0, + tableid: 254, + rtype: rtypes()[spec.rtype], + distance: spec.distance, + metric: spec.metric, + nhops: spec.nhops.iter().map(wire_nhop).collect(), + } + } + + /// The origin the route should be recorded with. + /// + /// The pairs are written out rather than deferred to `From`, so that a wrong pairing + /// is visible. The one rule that is not a pairing: a *connected* route to a single host is the + /// address of one of our own interfaces, so it is recorded as `Local` -- which is what makes + /// `build_pkt_instructions` emit a local-delivery instruction instead of an egress. + fn expected_origin(rtype: RouteType, prefix: &Prefix) -> RouteOrigin { + if rtype == RouteType::Connected && prefix.is_host() { + return RouteOrigin::Local; + } + match rtype { + RouteType::Local => RouteOrigin::Local, + RouteType::Connected => RouteOrigin::Connected, + RouteType::Static => RouteOrigin::Static, + RouteType::Ospf => RouteOrigin::Ospf, + RouteType::Isis => RouteOrigin::Isis, + RouteType::Bgp => RouteOrigin::Bgp, + RouteType::Other => RouteOrigin::Other, + } + } + + /// The key the next-hop should produce, or `None` if it should be refused. + /// + /// Four reasons to refuse, each worked out from the wire fields: interface index zero, a vni + /// that is not one, a vxlan next-hop that does not say which vtep to send to, and a forwarding + /// next-hop with neither an interface nor an address -- which is nowhere to send anything. + fn expected_key(spec: &NhopSpec, origin: RouteOrigin) -> Option { + let raw = ifindexes()[spec.ifindex]; + if raw == Some(0) { + return None; + } + let address = addresses()[spec.address]; + + let encap = match vnis()[spec.vni] { + None => None, + Some(vni) => Some(Encapsulation::Vxlan(VxlanEncapsulation { + vni: Vni::new_checked(vni).ok()?, + remote: address?, + dmac: None, + })), + }; + + // an encapsulated next-hop is reached by the underlay, so whatever interface the message + // named for it is ignored + let ifindex = if encap.is_some() { + None + } else { + raw.and_then(|i| InterfaceIndex::try_new(i).ok()) + }; + + let fwaction = if spec.drop { + FwAction::Drop + } else { + FwAction::Forward + }; + if fwaction == FwAction::Forward && ifindex.is_none() && address.is_none() { + return None; + } + + // no interface name: the conversion has no interface table to look one up in, which is + // what keeps one wire next-hop from keying two ways + Some(NhopKey::new( + origin, address, ifindex, encap, fwaction, None, + )) + } + + fn test_vrf() -> Vrf { + let config = RouterVrfConfig::new(1, "test"); + let mut vrf = Vrf::new(&config); + let (fibw, _fibr) = FibWriter::new(FibKey::from_vrfid(1)); + vrf.set_fibw(fibw); + vrf + } + + /// The pools and the constants that index them agree, and each refusal is reachable. + #[test] + fn the_pools_are_the_size_the_generator_thinks() { + assert_eq!(prefixes().len(), usize::from(NUM_PREFIXES)); + assert_eq!(addresses().len(), usize::from(NUM_ADDRESSES)); + assert_eq!(ifindexes().len(), usize::from(NUM_IFINDEXES)); + assert_eq!(vnis().len(), usize::from(NUM_VNIS)); + assert_eq!(rtypes().len(), usize::from(NUM_RTYPES)); + + let (prefix, len) = prefixes()[BAD_PREFIX]; + assert!( + Prefix::try_from((prefix, len)).is_err(), + "the bad prefix must not parse" + ); + assert!( + Vni::new_checked(0).is_err(), + "vni zero must not be a valid vni" + ); + assert!( + InterfaceIndex::try_new(0).is_err(), + "interface index zero must not be valid" + ); + // the delete property relies on no pool prefix being a root, since a root route is reset + // rather than removed + for (address, len) in prefixes() { + assert!(Prefix::try_from((address, len)).is_ok_and(|p| !p.is_root()) || len > 32); + } + } + + /// A next-hop off the wire is refused for exactly the reasons it should be, and otherwise + /// yields the key the message describes. + #[test] + fn a_wire_next_hop_becomes_the_key_the_message_describes() { + bolero::check!() + .with_generator(Routes) + .cloned() + .for_each(|spec: RouteSpec| { + for origin in [RouteOrigin::Local, RouteOrigin::Bgp, RouteOrigin::Connected] { + for nhop in &spec.nhops { + let got = RouteNhop::from_rpc_nhop(&wire_nhop(nhop), origin); + match expected_key(nhop, origin) { + Some(want) => { + let got = got.unwrap_or_else(|e| { + panic!("refused {nhop:?} with {e}, expected {want:?}") + }); + assert_eq!(got.key, want, "for {nhop:?} origin {origin:?}"); + assert_eq!(got.vrfid, nhop.vrfid, "vrfid for {nhop:?}"); + } + None => assert!(got.is_err(), "accepted {nhop:?}, expected refusal"), + } + } + } + }); + } + + /// A route off the wire is installed as the message describes, with the next-hops that survived + /// translation -- or a drop next-hop if none did. + #[test] + fn a_wire_route_is_installed_as_the_message_describes() { + bolero::check!() + .with_generator(Routes) + .cloned() + .for_each(|spec: RouteSpec| { + let rstore = RmacStore::new(); + let mut vrf = test_vrf(); + vrf.add_route_rpc(&wire_route(&spec), None, &rstore); + + let (raw, len) = prefixes()[spec.prefix]; + let Ok(prefix) = Prefix::try_from((raw, len)) else { + // a prefix we cannot parse installs nothing: only the two preset root routes + assert_eq!(vrf.len_v4() + vrf.len_v6(), 2, "for {spec:?}"); + return; + }; + + let origin = expected_origin(rtypes()[spec.rtype], &prefix); + let route = vrf + .get_route(prefix) + .unwrap_or_else(|| panic!("no route for {prefix}, for {spec:?}")); + + assert_eq!(route.origin, origin, "origin for {spec:?}"); + assert_eq!(route.distance, spec.distance, "distance for {spec:?}"); + assert_eq!(route.metric, spec.metric, "metric for {spec:?}"); + + let mut want: Vec = spec + .nhops + .iter() + .filter_map(|nhop| expected_key(nhop, origin)) + .collect(); + if want.is_empty() { + // nothing usable: the route is still installed, as a drop + want.push(NhopKey::with_drop()); + } + let got: Vec = route.s_nhops.iter().map(|s| s.rc.key.clone()).collect(); + assert_eq!(got, want, "next-hops for {spec:?}"); + }); + } + + /// Deleting the route the message names removes it; a prefix we cannot parse removes nothing. + #[test] + fn a_wire_delete_removes_what_the_message_names() { + bolero::check!() + .with_generator(Routes) + .cloned() + .for_each(|spec: RouteSpec| { + let rstore = RmacStore::new(); + let mut vrf = test_vrf(); + let route = wire_route(&spec); + vrf.add_route_rpc(&route, None, &rstore); + vrf.del_route_rpc(&route, None, &rstore); + + let (raw, len) = prefixes()[spec.prefix]; + if let Ok(prefix) = Prefix::try_from((raw, len)) { + assert!( + vrf.get_route(prefix).is_none(), + "route to {prefix} survived deletion, for {spec:?}" + ); + } + // no pool prefix is a root, so nothing but the two preset root routes is left + assert_eq!(vrf.len_v4() + vrf.len_v6(), 2, "for {spec:?}"); + // and the next-hop store is left holding only what the root routes name + let keys: Vec = vrf.nhstore.iter().map(|rc| rc.key.clone()).collect(); + assert_eq!( + keys, + vec![NhopKey::with_drop()], + "leftover next-hops for {spec:?}" + ); + }); + } +} From 85442109d4785b701f5bd7e2c895f296e1410d95 Mon Sep 17 00:00:00 2001 From: Daniel Noland Date: Fri, 7 Aug 2026 14:56:51 -0600 Subject: [PATCH 29/33] test(routing): Property-test the control-plane operations `router/cpi.rs` was at 1.8%. It is the layer above `rpc_adapt`: it picks the vrf, decides what a lookup failure means, and chains an operation's effects through the rest of the database. Seven properties over a whole `RoutingDb`, with an underlay route, an interface table and one overlay vrf on a vni. The one worth having is the evpn data path, end to end. An overlay route resolves through the underlay whether or not a router mac is known, but the encapsulation cannot be completed without one -- so until the mac arrives the only safe entry is a drop, and `Rmac::add` refreshing the fibs on that vni is what turns it into an encapsulate-then-egress with the right vni, remote and dmac. Breaking `VxlanEncapsulation::resolve` so it succeeds without a mac shows what that guards: the fib would offer [Encap(Vxlan { vni: 3000, remote: 7.0.0.1, dmac: None }), Egress(...)] which is a vxlan packet with no destination mac, put on the wire. Alongside it, and for the same reason, every entry is checked to be executable *and* to carry any `Drop` first. Resolution does produce `[Drop, Egress]` -- for an encapsulation that could not be completed -- and that is only safe because `packet_exec_instructions` stops at the first instruction that finishes the packet. Nothing said so; now something does. The rest: - withdrawing a router mac leaves the route forwarding. `Rmac::del` marks the entry stale rather than removing it and `resolve` accepts a stale one, so traffic keeps flowing to a mac that may have moved rather than the vni blackholing while the control plane catches up. Deliberate, and undocumented outside a one-line comment. - a route for a vrf we do not have fails on add, and is forgiven on delete until a config has been applied. The asymmetry is deliberate: a delete for a vrf we never had is a route we do not have either, so failing it would leave frr retrying something already true. - deleting the last route of a vrf on its way out takes the vrf with it, which is where `Vrf::check_deletion` and `VrfTable::remove_vrf` meet. - an interface address is refused unless both its mask and its interface index are usable, and lands in the interface table when it is not. - `nonlocal_nhop` spots a route whose next-hops live in another vrf. Verified by breaking five things: vxlan resolution succeeding without a mac, an rmac not refreshing its vni's fibs, an unknown vrf never being forgiven on delete, a deletable vrf being left behind, and an interface address skipping its mask check. Each fails. Production coverage: `router/cpi.rs` 1.8% -> 29.5% (the rest is the mio event loop and the socket plumbing, which needs a different harness), `routingdb.rs` 57.9% -> 84.2%, `interfaces/iftable.rs` 64.7% -> 81.4%, `iftablerw.rs` 58.1% -> 71.0%, `rib/vrftable.rs` 87.0% -> 91.7%, `rpc_adapt.rs` 88.7% -> 94.3%, and `routing/src` as a whole 61.4% -> 64.4%. Co-Authored-By: Claude Opus 5 (1M context) Signed-off-by: Daniel Noland --- routing/src/router/cpi.rs | 387 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 387 insertions(+) diff --git a/routing/src/router/cpi.rs b/routing/src/router/cpi.rs index b8fe075dda..ddec4dc920 100644 --- a/routing/src/router/cpi.rs +++ b/routing/src/router/cpi.rs @@ -497,3 +497,390 @@ pub fn process_cpi_data(rio: &mut Rio, peer: &SocketAddr, data: &mut Bytes, db: } } } + +/// Properties over the control-plane operations, driven through a whole [`RoutingDb`]. +/// +/// This is the layer above `rpc_adapt`: it picks the vrf, decides what a lookup failure means, and +/// chains an operation's effects into the rest of the database -- an rmac arriving refreshes the +/// fibs of the vrfs on its vni, and deleting a route can delete the vrf with it. Almost none of it +/// was covered. +#[cfg(test)] +mod cpi_properties { + use super::*; + use crate::atable::atablerw::AtableWriter; + use crate::config::RouterConfig; + use crate::evpn::RmacStore; + use crate::fib::fibobjects::{FibEntry, PktInstruction}; + use crate::fib::fibtable::FibTableWriter; + use crate::interfaces::iftablerw::IfTableWriter; + use crate::interfaces::tests::build_test_iftable; + use crate::rib::encapsulation::Encapsulation; + use crate::rib::vrf::tests::{build_test_nhop, build_test_route}; + use crate::rib::vrf::{RouteOrigin, RouterVrfConfig, VrfStatus}; + use bolero::{Driver, ValueGenerator}; + use dplane_rpc::msg::{ForwardAction, NextHop, VxlanEncap}; + use dplane_rpc::objects::MacAddress; + use lpm::prefix::Prefix; + use net::eth::mac::Mac; + use net::vxlan::Vni; + use std::net::IpAddr; + use std::ops::Bound::Included; + use std::str::FromStr; + + /// The vrf the overlay routes live in, and the vni it is reachable by. + const OVERLAY_VRF: VrfId = 7; + const OVERLAY_VNI: u32 = 3000; + /// The interface the underlay route egresses on. + const UNDERLAY_IFINDEX: u32 = 2; + + const NUM_VTEPS: u8 = 2; + const NUM_MACS: u8 = 2; + + fn addr(a: &str) -> IpAddr { + IpAddr::from_str(a).unwrap_or_else(|_| unreachable!()) + } + + /// Remote vtep addresses an overlay next-hop may point at. Both are covered by the underlay + /// route installed below, so reachability is never the reason a route fails to resolve. + fn vteps() -> Vec { + vec![addr("7.0.0.1"), addr("7.0.0.2")] + } + + fn macs() -> Vec<[u8; 6]> { + vec![ + [0x00, 0xaa, 0x00, 0x00, 0x00, 0x01], + [0x00, 0xbb, 0x00, 0x00, 0x00, 0x02], + ] + } + + /// A database with an interface table, an underlay route in the default vrf towards the vteps, + /// and one overlay vrf on [`OVERLAY_VNI`]. + fn fabric() -> RoutingDb { + let (fibtw, _fibtr) = FibTableWriter::new(); + let (iftw, _iftr) = IfTableWriter::new_with_data(build_test_iftable()); + let (_atw, atabler) = AtableWriter::new(); + let mut db = RoutingDb::new(fibtw, iftw, atabler); + + // the underlay: 7.0.0.0/8 out of an interface, so a vtep address resolves + let vrf0 = db + .vrftable + .get_vrf_mut(Vrf::DEFAULT_VRFID) + .unwrap_or_else(|e| unreachable!("{e}")); + vrf0.add_route_complete( + &Prefix::from_str("7.0.0.0/8").unwrap_or_else(|_| unreachable!()), + build_test_route(RouteOrigin::Connected, 0, 0), + &[build_test_nhop(None, Some(UNDERLAY_IFINDEX), 0, None)], + None, + &RmacStore::new(), + ); + + let vni = Vni::new_checked(OVERLAY_VNI).unwrap_or_else(|_| unreachable!()); + let config = RouterVrfConfig::new(OVERLAY_VRF, "overlay").set_vni(Some(vni)); + db.vrftable + .add_vrf(&config) + .unwrap_or_else(|e| unreachable!("{e}")); + db + } + + /// An overlay route: prefix reachable by vxlan to `vtep` on [`OVERLAY_VNI`]. + fn overlay_route(vrfid: VrfId, prefix: &str, vtep: IpAddr) -> IpRoute { + let (address, len) = prefix.split_once('/').unwrap_or_else(|| unreachable!()); + IpRoute { + prefix: addr(address), + prefix_len: len.parse().unwrap_or_else(|_| unreachable!()), + vrfid, + tableid: 254, + rtype: RouteType::Bgp, + distance: 20, + metric: 100, + nhops: vec![NextHop { + fwaction: ForwardAction::Forward, + address: Some(vtep), + ifindex: None, + vrfid, + encap: Some(NextHopEncap::VXLAN(VxlanEncap { vni: OVERLAY_VNI })), + }], + } + } + + fn rmac_msg(vtep: IpAddr, mac: [u8; 6]) -> Rmac { + Rmac { + address: vtep, + mac: MacAddress::new(mac), + vni: OVERLAY_VNI, + } + } + + /// The entries a vrf's fib offers for one prefix. + fn fib_entries(db: &RoutingDb, vrfid: VrfId, prefix: &str) -> Vec { + let prefix = Prefix::from_str(prefix).unwrap_or_else(|_| unreachable!()); + let Prefix::IPV4(wanted) = prefix else { + unreachable!() + }; + let vrf = db + .vrftable + .get_vrf(vrfid) + .unwrap_or_else(|e| unreachable!("{e}")); + let fibw = vrf.fibw.as_ref().unwrap_or_else(|| unreachable!()); + let fib = fibw.enter().unwrap_or_else(|| unreachable!()); + fib.iter_v4() + .find(|(p, _)| *p == wanted) + .map(|(_, route)| { + route + .iter() + .flat_map(|group| group.entries().iter().cloned()) + .collect() + }) + .unwrap_or_default() + } + + /// Every entry is one the forwarder can execute, and any `Drop` in it comes before anything + /// that would act on the packet. + /// + /// The second half is why an entry like `[Drop, Egress]` -- which resolution does produce, for + /// an encapsulation that could not be completed -- is safe: the forwarder stops at the first + /// instruction that finishes the packet, so a `Drop` reached first means nothing after it runs. + fn entries_are_well_formed(entries: &[FibEntry], at: &str) { + for entry in entries { + assert!(entry.is_valid(), "unusable {entry:?} {at}"); + let drop_at = entry + .iter() + .position(|inst| matches!(inst, PktInstruction::Drop)); + if let Some(index) = drop_at { + assert_eq!(index, 0, "a drop is not first in {entry:?} {at}"); + } + } + } + + /// Draws a `(vtep, mac)` pair. + #[derive(Debug, Clone, Copy, Default)] + struct Fabrics; + + impl ValueGenerator for Fabrics { + type Output = (usize, usize); + + fn generate(&self, driver: &mut D) -> Option<(usize, usize)> { + let vtep = driver.gen_u8(Included(&0), Included(&(NUM_VTEPS - 1)))?; + let mac = driver.gen_u8(Included(&0), Included(&(NUM_MACS - 1)))?; + Some((usize::from(vtep), usize::from(mac))) + } + } + + /// The pools and the constants that index them agree. + #[test] + fn the_pools_are_the_size_the_generator_thinks() { + assert_eq!(vteps().len(), usize::from(NUM_VTEPS)); + assert_eq!(macs().len(), usize::from(NUM_MACS)); + // every vtep must be reachable through the underlay route, so a failure to resolve is only + // ever about the router mac + let underlay = Prefix::from_str("7.0.0.0/8").unwrap_or_else(|_| unreachable!()); + for vtep in vteps() { + assert!(underlay.covers_addr(&vtep), "{vtep} is not in the underlay"); + } + } + + /// An overlay route drops until the router mac for its vtep arrives, and encapsulates after. + /// + /// This is the evpn data path end to end: the route resolves through the underlay either way, + /// but the encapsulation cannot be completed without a router mac, so until one arrives the + /// only safe thing is to drop. When `Rmac::add` stores one it refreshes the fibs on that vni, + /// which is what turns the entry into an encapsulate-then-egress. + #[test] + fn an_overlay_route_drops_until_its_router_mac_arrives() { + bolero::check!().with_generator(Fabrics).cloned().for_each( + |(vtep, mac): (usize, usize)| { + let mut db = fabric(); + let vtep = vteps()[vtep]; + let prefix = "10.0.0.0/24"; + + assert_eq!( + overlay_route(OVERLAY_VRF, prefix, vtep).add(&mut db), + RpcResultCode::Ok + ); + + // no router mac yet: the encapsulation cannot be completed + let before = fib_entries(&db, OVERLAY_VRF, prefix); + entries_are_well_formed(&before, "before the rmac"); + assert!( + before + .iter() + .all(|entry| matches!(entry.iter().next(), Some(PktInstruction::Drop))), + "an overlay route with no router mac must drop, got {before:?}" + ); + + assert_eq!(rmac_msg(vtep, macs()[mac]).add(&mut db), RpcResultCode::Ok); + + let after = fib_entries(&db, OVERLAY_VRF, prefix); + entries_are_well_formed(&after, "after the rmac"); + let expected_mac = Mac::from(macs()[mac]); + for entry in &after { + let mut instructions = entry.iter(); + match instructions.next() { + Some(PktInstruction::Encap(Encapsulation::Vxlan(vxlan))) => { + assert_eq!(vxlan.vni.as_u32(), OVERLAY_VNI, "vni in {entry:?}"); + assert_eq!(vxlan.remote, vtep, "remote in {entry:?}"); + assert_eq!(vxlan.dmac, Some(expected_mac), "dmac in {entry:?}"); + } + other => panic!("expected an encapsulation first, got {other:?}"), + } + match instructions.next() { + Some(PktInstruction::Egress(egress)) => { + assert_eq!( + egress.ifindex().map(InterfaceIndex::to_u32), + Some(UNDERLAY_IFINDEX), + "egress interface in {entry:?}" + ); + assert_eq!( + *egress.address(), + Some(vtep), + "egress address in {entry:?}" + ); + } + other => panic!("expected an egress second, got {other:?}"), + } + assert!(instructions.next().is_none(), "extra work in {entry:?}"); + } + }, + ); + } + + /// Withdrawing a router mac does not stop traffic. + /// + /// `Rmac::del` marks the entry stale rather than removing it, and `VxlanEncapsulation::resolve` + /// accepts a stale one -- "ok if we found a mac, even if the entry is stale". Forwarding to a + /// mac that may have moved beats blackholing the vni while the control plane catches up. + #[test] + fn withdrawing_a_router_mac_leaves_the_route_forwarding() { + bolero::check!().with_generator(Fabrics).cloned().for_each( + |(vtep, mac): (usize, usize)| { + let mut db = fabric(); + let vtep = vteps()[vtep]; + let prefix = "10.0.0.0/24"; + let rmac = rmac_msg(vtep, macs()[mac]); + + overlay_route(OVERLAY_VRF, prefix, vtep).add(&mut db); + rmac.add(&mut db); + let before = fib_entries(&db, OVERLAY_VRF, prefix); + + assert_eq!(rmac.del(&mut db), RpcResultCode::Ok); + // the fib is only rebuilt on the next refresh, so ask for one + db.vrftable.refresh_non_default_fibs(&db.rmac_store); + + let after = fib_entries(&db, OVERLAY_VRF, prefix); + entries_are_well_formed(&after, "after withdrawing the rmac"); + assert_eq!(after, before, "withdrawing a router mac changed the fib"); + }, + ); + } + + /// A route for a vrf we do not have fails on add, and succeeds on delete until we have a config. + /// + /// The asymmetry is deliberate: a delete for a vrf we never had is a route we do not have + /// either, so reporting failure would leave frr retrying something already true. Once a config + /// has been applied there is no such excuse, and the same lookup is a real failure. + #[test] + fn an_unknown_vrf_fails_on_add_and_forgives_on_delete() { + let missing = OVERLAY_VRF + 1; + let prefix = "10.9.0.0/24"; + let vtep = vteps()[0]; + + let mut db = fabric(); + assert_eq!( + overlay_route(missing, prefix, vtep).add(&mut db), + RpcResultCode::Failure + ); + assert_eq!( + overlay_route(missing, prefix, vtep).del(&mut db), + RpcResultCode::Ok, + "a delete for an unknown vrf is forgiven while we have no config" + ); + + db.set_config(RouterConfig::new(1)); + assert!(db.have_config()); + assert_eq!( + overlay_route(missing, prefix, vtep).del(&mut db), + RpcResultCode::Failure, + "once a config is applied the same lookup is a real failure" + ); + } + + /// Deleting the last route of a vrf on its way out takes the vrf with it. + #[test] + fn deleting_the_last_route_of_a_dying_vrf_removes_it() { + let mut db = fabric(); + let prefix = "10.0.0.0/24"; + let vtep = vteps()[0]; + let route = overlay_route(OVERLAY_VRF, prefix, vtep); + route.add(&mut db); + + // while the vrf is active, deleting its routes leaves it in place + assert_eq!(route.del(&mut db), RpcResultCode::Ok); + assert!(db.vrftable.contains(OVERLAY_VRF)); + + route.add(&mut db); + db.vrftable + .get_vrf_mut(OVERLAY_VRF) + .unwrap_or_else(|e| unreachable!("{e}")) + .set_status(VrfStatus::Deleting); + + // now the same delete empties it, which makes it deletable, which removes it + assert_eq!(route.del(&mut db), RpcResultCode::Ok); + assert!( + !db.vrftable.contains(OVERLAY_VRF), + "a vrf that became deletable was left behind" + ); + } + + /// An interface address is refused unless both the mask and the interface index are usable. + #[test] + fn an_interface_address_is_refused_unless_it_is_usable() { + let cases = [ + // (ifindex, mask, expected) + (UNDERLAY_IFINDEX, 24, RpcResultCode::Ok), + (UNDERLAY_IFINDEX, 0, RpcResultCode::InvalidRequest), + (UNDERLAY_IFINDEX, 33, RpcResultCode::InvalidRequest), + (0, 24, RpcResultCode::InvalidRequest), + ]; + for (ifindex, mask, want) in cases { + let mut db = fabric(); + let message = IfAddress { + ifname: "eth0".to_string(), + address: addr("10.0.0.1"), + mask_len: mask, + ifindex, + vrfid: Vrf::DEFAULT_VRFID, + }; + let present = |db: &RoutingDb| { + let iftable = db.iftw.enter().unwrap_or_else(|| unreachable!()); + let Ok(index) = InterfaceIndex::try_new(ifindex) else { + return false; + }; + iftable + .get_interface(index) + .is_some_and(|iface| !iface.addresses.is_empty()) + }; + + assert_eq!(message.add(&mut db), want, "adding {message}"); + assert_eq!( + present(&db), + want == RpcResultCode::Ok, + "after adding {message}" + ); + + assert_eq!(message.del(&mut db), want, "deleting {message}"); + assert!(!present(&db), "the address survived its own deletion"); + } + } + + /// `nonlocal_nhop` spots a route whose next-hops live in another vrf. + #[test] + fn a_next_hop_in_another_vrf_is_nonlocal() { + let vtep = vteps()[0]; + let mut route = overlay_route(OVERLAY_VRF, "10.0.0.0/24", vtep); + assert!(!nonlocal_nhop(&route), "its own vrf is not nonlocal"); + route.nhops[0].vrfid = Vrf::DEFAULT_VRFID; + assert!(nonlocal_nhop(&route), "another vrf is nonlocal"); + route.nhops.clear(); + assert!(!nonlocal_nhop(&route), "no next-hops, nothing nonlocal"); + } +} From c04dec2b50f189e376db8035b5bb07b32a08f710 Mon Sep 17 00:00:00 2001 From: Daniel Noland Date: Fri, 7 Aug 2026 15:07:20 -0600 Subject: [PATCH 30/33] test(routing): Property-test the FRR config renderers `bfd`, `ospf` and `renderer/mod.rs` were at zero coverage and `vrf` at 19.6%. Each renderer had a test that printed its output and asserted nothing, so the code ran and no claim was ever made about it. A renderer's failure mode is not a crash. It is a BFD session, an OSPF area or a vni that never gets configured because a line was not emitted, or that gets configured twice -- neither visible anywhere but in FRR's own state. A round-trip oracle would need an FRR parser and is not worth building. What is worth checking is weaker and catches both: **every value the config holds appears in the output, and no value it does not hold appears.** Eight properties. The one the others cannot replace is `everything_configured_reaches_the_output`: a sub-renderer that works perfectly is no use if the top level never calls it, and only a whole-`InternalConfig` property sees that. Deleting either `render_vrfs_ospf` or the BFD peers from `InternalConfig::render` fails it and nothing else. The rest pin the rules that are not simply "render what is set": - a BFD source address is emitted only for a *multihop* peer. A single-hop peer with a source silently loses it, which is deliberate -- FRR has nowhere to put it -- and was written down only as a parenthesis in a comment. - a BFD section is not emitted at all when there are no peers, so an empty list does not leave a bare `bfd` / `exit` pair in the config. - the default vrf renders *without* a `vrf ` / `exit-vrf` wrapper: its configuration belongs at the top level, and wrapping it would put the underlay's static routes and vni into a vrf FRR does not have. - the four OSPF network keywords, written out independently so a transposed pair is visible. - rendering is deterministic. This one is about `frr-reload.py`, which diffs the output against what FRR is running: a rendering that varied would look like a configuration change every pass and reload FRR for nothing. Verified by breaking seven things: rendering a BFD source without multihop, emitting the BFD section when empty, transposing two OSPF network keywords, dropping an OSPF instance's vrf, dropping the OSPF and the BFD calls from the top-level renderer, and wrapping the default vrf. Each fails. One thing learned about the config model on the way, and recorded where the harness works around it: `VrfConfigTable` is a multi-index map with a *unique* index over `name`, `tableid`, `vni` and `vpc_id`, and an `Option` field's `None` counts as a value -- so it can hold at most one vrf without a vni, and at most one without a vpc id. Production satisfies that (the default vrf has neither, every other vrf is a vpc vrf and has both), but the types take `Option` and say nothing, and `add_vrf_config` calls a collision "a bug". Production coverage: `frr/renderer/bfd.rs`, `ospf.rs` and `mod.rs` 0% -> 100%, `vrf.rs` 19.6% -> 97.8%, `prefixlist.rs` 84.4% -> 95.6%, and `routing/src` as a whole 64.4% -> 66.5%. Co-Authored-By: Claude Opus 5 (1M context) Signed-off-by: Daniel Noland --- routing/src/frr/renderer/mod.rs | 528 ++++++++++++++++++++++++++++++++ 1 file changed, 528 insertions(+) diff --git a/routing/src/frr/renderer/mod.rs b/routing/src/frr/renderer/mod.rs index 64a7f89837..75de135fa6 100644 --- a/routing/src/frr/renderer/mod.rs +++ b/routing/src/frr/renderer/mod.rs @@ -66,3 +66,531 @@ impl Render for InternalConfig { cfg } } + +/// Properties over the FRR config renderers. +/// +/// A renderer's failure mode is not a crash: it is a BFD session, an OSPF area or a vni that never +/// gets configured because a line was not emitted, or that gets configured twice. Neither shows up +/// anywhere but in FRR's own state. Each renderer had one test that printed its output and asserted +/// nothing, so `ospf`, `bfd` and this module were at zero coverage. +/// +/// A round-trip oracle would need an FRR parser and is not worth building. What is worth checking is +/// weaker and still catches both failure modes: **every value the config holds appears in the +/// output, and no value it does not hold appears.** That is enough to catch an omitted field, a +/// field emitted when it should not be, and a top-level renderer that forgot to call a sub-renderer. +#[cfg(test)] +mod renderer_properties { + use super::*; + use bolero::{Driver, ValueGenerator}; + use config::external::overlay::vpc::VpcId; + use config::internal::device::DeviceConfig; + use config::internal::routing::bfd::{ + BFD_DETECT_MULTIPLIER, BFD_RECEIVE_INTERVAL_MS, BFD_TRANSMIT_INTERVAL_MS, BfdPeer, + }; + use config::internal::routing::ospf::{Ospf, OspfInterface, OspfNetwork}; + use config::internal::routing::vrf::{VrfConfig, VrfConfigTable}; + use net::route::RouteTableId; + use net::vxlan::Vni; + use std::net::{IpAddr, Ipv4Addr}; + use std::ops::Bound::Included; + use std::str::FromStr; + + const NUM_ADDRESSES: u8 = 3; + const NUM_NETWORKS: u8 = 4; + const MAX_PEERS: u8 = 3; + const MAX_VRFS: u8 = 3; + + fn addresses() -> Vec { + ["10.0.0.1", "10.0.0.2", "2001:db8::1"] + .iter() + .map(|a| IpAddr::from_str(a).unwrap_or_else(|_| unreachable!())) + .collect() + } + + fn networks() -> Vec { + vec![ + OspfNetwork::Broadcast, + OspfNetwork::NonBroadcast, + OspfNetwork::Point2Point, + OspfNetwork::Point2Multipoint, + ] + } + + /// The keyword FRR expects for each network type, written out here rather than taken from + /// `OspfNetwork::rendered`, so a wrong pairing is visible. + fn network_keyword(network: &OspfNetwork) -> &'static str { + match network { + OspfNetwork::Broadcast => "broadcast", + OspfNetwork::NonBroadcast => "non-broadcast", + OspfNetwork::Point2Point => "point-to-point", + OspfNetwork::Point2Multipoint => "point-to-multipoint", + } + } + + /// A BFD peer as the generator describes it. + #[derive(Debug, Clone, Copy)] + struct PeerSpec { + address: usize, + multihop: bool, + source: Option, + } + + /// A vrf as the generator describes it. + /// + /// Name, table id, vni and vpc id are all derived from its position rather than generated. + /// `VrfConfigTable` is a multi-index map with a *unique* index over each of them, and an + /// `Option` field's `None` counts as a value there -- so it can hold at most one vrf without a + /// vni, and at most one without a vpc id. Production satisfies that (the default vrf has + /// neither; every other vrf is a vpc vrf and has both), but the types do not say so, and + /// `add_vrf_config` calls a collision "a bug". So the harness gives every vrf its own, and the + /// "renders it only when it has one" cases are checked against `VrfConfig` directly below. + #[derive(Debug, Clone, Copy)] + struct VrfSpec { + ospf: Option, + } + + #[derive(Debug, Clone)] + struct Fabric { + genid: GenId, + peers: Vec, + vrfs: Vec, + } + + /// Draws [`Fabric`]s. + #[derive(Debug, Clone, Copy, Default)] + struct Fabrics; + + fn index(driver: &mut D, count: u8) -> Option { + driver + .gen_u8(Included(&0), Included(&(count - 1))) + .map(usize::from) + } + + /// An index into a pool of `count`, or `count` itself to mean "absent". + /// + /// One draw with a sentinel rather than an `Option>`, whose outer `None` -- the driver + /// running out of input -- cannot be told from the inner one. + fn maybe_index(driver: &mut D, count: u8) -> Option { + index(driver, count + 1) + } + + /// Read a [`maybe_index`] draw back as an option. + fn drawn(value: usize, count: u8) -> Option { + (value < usize::from(count)).then_some(value) + } + + impl ValueGenerator for Fabrics { + type Output = Fabric; + + fn generate(&self, driver: &mut D) -> Option { + let genid = GenId::from(driver.gen_u8(Included(&1), Included(&9))?); + + let count = driver.gen_u8(Included(&0), Included(&MAX_PEERS))?; + let mut peers = Vec::with_capacity(usize::from(count)); + for _ in 0..count { + peers.push(PeerSpec { + address: index(driver, NUM_ADDRESSES)?, + multihop: driver.produce::()?, + source: drawn(maybe_index(driver, NUM_ADDRESSES)?, NUM_ADDRESSES), + }); + } + + let count = driver.gen_u8(Included(&0), Included(&MAX_VRFS))?; + let mut vrfs = Vec::with_capacity(usize::from(count)); + for _ in 0..count { + vrfs.push(VrfSpec { + ospf: drawn(maybe_index(driver, NUM_ADDRESSES)?, NUM_ADDRESSES), + }); + } + + Some(Fabric { genid, peers, vrfs }) + } + } + + fn peer(spec: PeerSpec) -> BfdPeer { + BfdPeer::new(addresses()[spec.address]) + .set_multihop(spec.multihop) + .set_source(spec.source.map(|i| addresses()[i])) + } + + /// A router id for vrf `index`, distinct per vrf so it can be looked for in the output. + fn router_id(index: usize) -> Ipv4Addr { + Ipv4Addr::new( + 192, + 168, + 0, + u8::try_from(index + 1).unwrap_or_else(|_| unreachable!()), + ) + } + + fn vrf_name(index: usize) -> String { + format!("VPC-{index}") + } + + /// A vpc id for vrf `index`. + /// + /// Every non-default vrf needs one: `VrfConfigTable` holds a unique index over `vpc_id`, so two + /// vrfs without one collide. Real configs always have them -- non-default vrfs are vpc vrfs -- + /// but it is not obvious from the type, which takes an `Option`. + fn vpc_id(index: usize) -> VpcId { + VpcId::try_from(format!("vpc{index:02}").as_str()).unwrap_or_else(|_| unreachable!()) + } + + fn vni_for(index: usize) -> Vni { + Vni::new_checked(3000 + u32::try_from(index).unwrap_or_else(|_| unreachable!())) + .unwrap_or_else(|_| unreachable!()) + } + + fn internal_config(fabric: &Fabric) -> InternalConfig { + let mut config = InternalConfig::new("GW1", DeviceConfig::new()); + config.bfd_peers = fabric.peers.iter().copied().map(peer).collect(); + + let mut vrfs = VrfConfigTable::new(); + for (index, spec) in fabric.vrfs.iter().enumerate() { + let mut vrf = VrfConfig::new(&vrf_name(index), Some(vni_for(index)), false) + .set_table_id( + RouteTableId::try_from( + 100 + u32::try_from(index).unwrap_or_else(|_| unreachable!()), + ) + .unwrap_or_else(|_| unreachable!()), + ) + .set_vpc_id(vpc_id(index)); + if spec.ospf.is_some() { + let mut ospf = Ospf::new(router_id(index)); + ospf.set_vrf_name(vrf_name(index)); + vrf.set_ospf(ospf); + } + vrfs.add_vrf_config(vrf) + .unwrap_or_else(|e| unreachable!("{e}")); + } + config.vrfs = vrfs; + config + } + + /// How many times `needle` appears in `haystack`. + fn occurrences(haystack: &str, needle: &str) -> usize { + haystack.matches(needle).count() + } + + /// The pools and the constants that index them agree. + #[test] + fn the_pools_are_the_size_the_generator_thinks() { + assert_eq!(addresses().len(), usize::from(NUM_ADDRESSES)); + assert_eq!(networks().len(), usize::from(NUM_NETWORKS)); + // the derived names, table ids and vnis must be distinct, or `add_vrf_config` would refuse + // them and the harness would be testing its own collision handling + let count = usize::from(MAX_VRFS); + for derived in [ + (0..count).map(vrf_name).collect::>(), + (0..count).map(|i| format!("{:?}", vpc_id(i))).collect(), + (0..count).map(|i| vni_for(i).to_string()).collect(), + (0..count).map(|i| router_id(i).to_string()).collect(), + ] { + let mut sorted = derived.clone(); + sorted.sort(); + sorted.dedup(); + assert_eq!( + sorted.len(), + derived.len(), + "derived values must be distinct" + ); + } + } + + /// A BFD peer renders every field it carries, and none it does not. + /// + /// Note the one rule that is not "render what is set": a source address is emitted only for a + /// multihop peer. A single-hop peer with a source silently loses it, which is deliberate -- + /// FRR has nowhere to put it -- and worth having written down. + #[test] + fn a_bfd_peer_renders_the_fields_it_carries() { + bolero::check!() + .with_generator(Fabrics) + .cloned() + .for_each(|fabric: Fabric| { + for spec in &fabric.peers { + let peer = peer(*spec); + let text = peer.render(&()).to_string(); + + assert_eq!( + occurrences(&text, &format!(" peer {}", peer.address)), + 1, + "peer address once, for {spec:?} in {text}" + ); + assert_eq!( + occurrences(&text, " multihop"), + usize::from(spec.multihop), + "multihop iff set, for {spec:?} in {text}" + ); + + let source_shown = spec.multihop && spec.source.is_some(); + assert_eq!( + occurrences(&text, " source "), + usize::from(source_shown), + "a source is rendered only for a multihop peer, for {spec:?} in {text}" + ); + if source_shown { + let source = addresses()[spec.source.unwrap_or_else(|| unreachable!())]; + assert_eq!( + occurrences(&text, &format!(" source {source}")), + 1, + "the source that was set, for {spec:?} in {text}" + ); + } + + // and the timing parameters FRR needs to bring the session up at all + for line in [ + " no shutdown".to_string(), + format!(" detect-multiplier {BFD_DETECT_MULTIPLIER}"), + format!(" transmit-interval {BFD_TRANSMIT_INTERVAL_MS}"), + format!(" receive-interval {BFD_RECEIVE_INTERVAL_MS}"), + ] { + assert_eq!(occurrences(&text, &line), 1, "{line} once, in {text}"); + } + } + }); + } + + /// A BFD section appears only when there are peers, and holds each of them once. + #[test] + fn a_bfd_section_appears_only_for_peers_it_has() { + bolero::check!() + .with_generator(Fabrics) + .cloned() + .for_each(|fabric: Fabric| { + let peers: Vec = fabric.peers.iter().copied().map(peer).collect(); + let text = peers.render(&()).to_string(); + + if peers.is_empty() { + assert!( + !text.contains("bfd"), + "an empty peer list must render no bfd section, got {text}" + ); + return; + } + + assert_eq!( + occurrences(&text, "\nbfd\n"), + 1, + "one bfd section in {text}" + ); + assert_eq!(occurrences(&text, "\nexit\n"), 1, "one exit in {text}"); + for address in addresses() { + let wanted = peers.iter().filter(|p| p.address == address).count(); + assert_eq!( + occurrences(&text, &format!(" peer {address}")), + wanted, + "{address} appears once per peer that has it, in {text}" + ); + } + }); + } + + /// An OSPF instance renders its router id, and its vrf only when it has one. + #[test] + fn an_ospf_instance_renders_its_router_id_and_vrf() { + bolero::check!() + .with_generator(Fabrics) + .cloned() + .for_each(|fabric: Fabric| { + for (index, _) in fabric.vrfs.iter().enumerate() { + let id = router_id(index); + let name = vrf_name(index); + + let plain = Ospf::new(id).render(&()).to_string(); + assert_eq!( + occurrences(&plain, "router ospf\n"), + 1, + "an ospf instance with no vrf, in {plain}" + ); + assert_eq!( + occurrences(&plain, &format!(" ospf router-id {id}")), + 1, + "the router id, in {plain}" + ); + + let mut in_vrf = Ospf::new(id); + in_vrf.set_vrf_name(name.clone()); + let text = in_vrf.render(&()).to_string(); + assert_eq!( + occurrences(&text, &format!("router ospf vrf {name}")), + 1, + "an ospf instance in a vrf, in {text}" + ); + assert_eq!( + occurrences(&text, &format!(" ospf router-id {id}")), + 1, + "the router id, in {text}" + ); + } + }); + } + + /// An OSPF interface renders its area, and each option only when it is set. + #[test] + fn an_ospf_interface_renders_the_options_it_has() { + bolero::check!() + .with_generator(bolero::produce::<(u8, bool, Option, Option)>()) + .cloned() + .for_each( + |(area, passive, cost, network): (u8, bool, Option, Option)| { + let area = Ipv4Addr::new(0, 0, 0, area); + let network = + network.map(|n| networks()[usize::from(n) % networks().len()].clone()); + + let mut interface = OspfInterface::new(area).set_passive(passive); + if let Some(cost) = cost { + interface = interface.set_cost(cost); + } + if let Some(network) = network.clone() { + interface = interface.set_network(network); + } + let text = interface.render(&()).to_string(); + + assert_eq!( + occurrences(&text, &format!(" ip ospf area {area}")), + 1, + "the area, in {text}" + ); + assert_eq!( + occurrences(&text, " ip ospf passive"), + usize::from(passive), + "passive iff set, in {text}" + ); + assert_eq!( + occurrences(&text, " ip ospf cost "), + usize::from(cost.is_some()), + "cost iff set, in {text}" + ); + if let Some(cost) = cost { + assert_eq!(occurrences(&text, &format!(" ip ospf cost {cost}")), 1); + } + assert_eq!( + occurrences(&text, " ip ospf network "), + usize::from(network.is_some()), + "network iff set, in {text}" + ); + if let Some(network) = &network { + assert_eq!( + occurrences( + &text, + &format!(" ip ospf network {}", network_keyword(network)) + ), + 1, + "the network keyword FRR expects, in {text}" + ); + } + }, + ); + } + + /// Everything the config holds reaches the rendered output. + /// + /// This is the property the per-renderer ones cannot give: a sub-renderer that works perfectly is + /// no use if the top level never calls it, and an object silently missing from an FRR config is + /// a session or a vni that never comes up. + #[test] + fn everything_configured_reaches_the_output() { + bolero::check!() + .with_generator(Fabrics) + .cloned() + .for_each(|fabric: Fabric| { + let config = internal_config(&fabric); + let text = config.render(&fabric.genid).to_string(); + + assert_eq!( + occurrences(&text, &format!("! config for gen {}", fabric.genid)), + 1, + "the generation this config is for, in {text}" + ); + + for address in addresses() { + let wanted = fabric + .peers + .iter() + .filter(|p| addresses()[p.address] == address) + .count(); + assert_eq!( + occurrences(&text, &format!(" peer {address}")), + wanted, + "bfd peer {address}, in {text}" + ); + } + + for (index, spec) in fabric.vrfs.iter().enumerate() { + let name = vrf_name(index); + assert_eq!( + occurrences(&text, &format!("\nvrf {name}\n")), + 1, + "vrf {name} declared once, in {text}" + ); + assert_eq!( + occurrences(&text, &format!(" vni {}", vni_for(index))), + 1, + "vni of {name}, in {text}" + ); + assert_eq!( + occurrences(&text, &format!("router ospf vrf {name}")), + usize::from(spec.ospf.is_some()), + "ospf instance of {name} iff it has one, in {text}" + ); + assert_eq!( + occurrences(&text, &format!(" ospf router-id {}", router_id(index))), + usize::from(spec.ospf.is_some()), + "router id of {name} iff it has ospf, in {text}" + ); + } + }); + } + + /// A vrf renders its own name and vni only when it should. + /// + /// The default vrf is the interesting case: its configuration belongs at the top level of the + /// FRR config, so it must *not* be wrapped in `vrf ` / `exit-vrf`. Wrapping it would put + /// the underlay's static routes and vni into a vrf that FRR does not have. + #[test] + fn a_vrf_renders_its_wrapper_only_when_it_is_not_the_default() { + bolero::check!() + .with_generator(bolero::produce::<(bool, bool)>()) + .cloned() + .for_each(|(default, has_vni): (bool, bool)| { + let name = if default { "default" } else { "VPC-1" }; + let vni = has_vni.then(|| vni_for(0)); + let text = VrfConfig::new(name, vni, default).render(&()).to_string(); + + let wrapped = usize::from(!default); + assert_eq!( + occurrences(&text, &format!("\nvrf {name}\n")), + wrapped, + "a vrf declaration iff not the default, in {text}" + ); + assert_eq!( + occurrences(&text, "exit-vrf"), + wrapped, + "an exit-vrf iff not the default, in {text}" + ); + assert_eq!( + occurrences(&text, " vni "), + usize::from(has_vni), + "a vni iff it has one, in {text}" + ); + }); + } + + /// Rendering the same config twice gives the same text. + /// + /// Worth its own property because the output is handed to `frr-reload.py`, which diffs it against + /// what FRR is running. A rendering that varied -- an unordered table iterated, say -- would look + /// like a configuration change on every pass and reload FRR for nothing. + #[test] + fn rendering_is_deterministic() { + bolero::check!() + .with_generator(Fabrics) + .cloned() + .for_each(|fabric: Fabric| { + let once = internal_config(&fabric).render(&fabric.genid).to_string(); + let twice = internal_config(&fabric).render(&fabric.genid).to_string(); + assert_eq!(once, twice, "rendering is not deterministic"); + }); + } +} From 7dd61e723d1357c77624d0c27b9feb12b233b62c Mon Sep 17 00:00:00 2001 From: Daniel Noland Date: Fri, 7 Aug 2026 15:20:32 -0600 Subject: [PATCH 31/33] test(routing): Model-check the interface table and its attachments The interface table is a map, but two things about it are not. An interface's **attachment** names a vrf, which lives in a different structure and can be removed underneath it. Nothing in the types ties the two together: `VrfTable::remove_vrf` calling `detach_interfaces_from_vrf` is the whole of it, so the property checks the invariant that rests on it -- no interface is left attached to a vrf that has gone -- rather than trusting the one call site. That is the fourth structure on this branch where a reference outlives its referent only because one function remembers to clean up; the difference here is that the one function does. And a **reconfiguration has to leave the runtime state alone**. `mod_interface` replaces the name, description, type, admin state and mtu, and must not touch the addresses, the vrf attachment or the operational state -- none of which comes from the configuration being replaced, all of which is learned out of band. The model tracks both halves separately so that a reconfiguration touching the wrong one shows up. One asymmetry worth recording rather than fixing: an interface address for an interface the table does not hold is dropped, and the caller is not told. The error is raised inside `absorb_first`, where the only thing to do with it is log it, so `IfAddress::add` reports success. That is the same out-of-band-population hazard as the next-hop key in fa5398d01, but the consequence is much smaller: `Interface::addresses` is read only by the CLI and by the interface renderer, not by anything in the forwarding path. Noted in the harness where the model mirrors it. Verified by breaking six things: removing a vrf without detaching its interfaces, a reconfiguration clearing the addresses, a reconfiguration dropping the attachment, detach-from-vrf detaching every interface rather than that vrf's, attaching to a vrf that does not exist, and accepting a duplicate interface. Each fails. Production coverage: `interfaces/iftable.rs` 81.4% -> 94.1%, `iftablerw.rs` 71.0% -> 91.1%, `interface.rs` 75.9% -> 84.3%, and `routing/src` as a whole 66.5% -> 67.3%. Co-Authored-By: Claude Opus 5 (1M context) Signed-off-by: Daniel Noland --- routing/src/interfaces/iftablerw.rs | 462 ++++++++++++++++++++++++++++ 1 file changed, 462 insertions(+) diff --git a/routing/src/interfaces/iftablerw.rs b/routing/src/interfaces/iftablerw.rs index db9c641424..df2bf78ea0 100644 --- a/routing/src/interfaces/iftablerw.rs +++ b/routing/src/interfaces/iftablerw.rs @@ -213,3 +213,465 @@ impl IfTableReaderFactory { #[allow(unsafe_code)] unsafe impl Send for IfTableWriter {} + +/// Model-based properties over the interface table. +/// +/// The table is a map, but two things about it are not: an interface's *attachment* names a vrf, +/// which lives in a different structure and can be removed underneath it; and a reconfiguration has +/// to leave the runtime state -- addresses, attachment, operational state -- alone, since none of it +/// comes from the configuration that is being replaced. +#[cfg(test)] +mod iftable_properties { + use super::*; + use crate::fib::fibtable::FibTableWriter; + use crate::interfaces::interface::{Attachment, IfType}; + use crate::rib::vrf::{RouterVrfConfig, Vrf}; + use bolero::{Driver, ValueGenerator}; + use net::interface::address::IfAddr; + use std::collections::{BTreeMap, BTreeSet}; + use std::net::IpAddr; + use std::ops::Bound::Included; + use std::str::FromStr; + + const NUM_IFACES: u8 = 3; + const NUM_VRFS: u8 = 2; + const NUM_ADDRESSES: u8 = 2; + const NUM_STATES: u8 = 3; + const MAX_CHANGES: u8 = 12; + + /// Interface indices. `InterfaceIndex` is non-zero, so these start at one. + fn ifindexes() -> Vec { + (1..=u32::from(NUM_IFACES)) + .map(|i| InterfaceIndex::try_new(i).unwrap_or_else(|_| unreachable!())) + .collect() + } + + /// Non-default vrf ids. The default vrf is 0 and `VrfTable::new` makes it; it is left out so + /// that removing a vrf is always allowed. + fn vrf_ids() -> Vec { + (1..=u32::from(NUM_VRFS)).collect() + } + + fn addresses() -> Vec { + ["10.0.0.1", "10.0.0.2"] + .iter() + .map(|a| { + IfAddr::new(IpAddr::from_str(a).unwrap_or_else(|_| unreachable!()), 24) + .unwrap_or_else(|_| unreachable!()) + }) + .collect() + } + + fn states() -> Vec { + vec![IfState::Unknown, IfState::Down, IfState::Up] + } + + /// One change, over indices into the pools above. + #[derive(Debug, Clone)] + enum Change { + /// Add an interface. `renamed` picks between two names so that a modification is visible. + AddInterface { + iface: usize, + renamed: bool, + }, + ModInterface { + iface: usize, + renamed: bool, + }, + DelInterface { + iface: usize, + }, + AddAddress { + iface: usize, + address: usize, + }, + DelAddress { + iface: usize, + address: usize, + }, + SetOperState { + iface: usize, + state: usize, + }, + SetAdminState { + iface: usize, + state: usize, + }, + AttachToVrf { + iface: usize, + vrf: usize, + }, + Detach { + iface: usize, + }, + DetachVrf { + vrf: usize, + }, + AddVrf { + vrf: usize, + }, + RemoveVrf { + vrf: usize, + }, + } + + /// Draws sequences of [`Change`]s. + #[derive(Debug, Clone, Copy, Default)] + struct ChangeSequences; + + fn index(driver: &mut D, count: u8) -> Option { + driver + .gen_u8(Included(&0), Included(&(count - 1))) + .map(usize::from) + } + + impl ValueGenerator for ChangeSequences { + type Output = Vec; + + fn generate(&self, driver: &mut D) -> Option> { + let len = driver.gen_u8(Included(&0), Included(&MAX_CHANGES))?; + let mut out = Vec::with_capacity(usize::from(len)); + for _ in 0..len { + let iface = index(driver, NUM_IFACES)?; + let change = match driver.gen_u8(Included(&0), Included(&11))? { + 0 => Change::AddInterface { + iface, + renamed: driver.produce::()?, + }, + 1 => Change::ModInterface { + iface, + renamed: driver.produce::()?, + }, + 2 => Change::DelInterface { iface }, + 3 => Change::AddAddress { + iface, + address: index(driver, NUM_ADDRESSES)?, + }, + 4 => Change::DelAddress { + iface, + address: index(driver, NUM_ADDRESSES)?, + }, + 5 => Change::SetOperState { + iface, + state: index(driver, NUM_STATES)?, + }, + 6 => Change::SetAdminState { + iface, + state: index(driver, NUM_STATES)?, + }, + 7 => Change::AttachToVrf { + iface, + vrf: index(driver, NUM_VRFS)?, + }, + 8 => Change::Detach { iface }, + 9 => Change::DetachVrf { + vrf: index(driver, NUM_VRFS)?, + }, + 10 => Change::AddVrf { + vrf: index(driver, NUM_VRFS)?, + }, + _ => Change::RemoveVrf { + vrf: index(driver, NUM_VRFS)?, + }, + }; + out.push(change); + } + Some(out) + } + } + + fn name_of(iface: usize, renamed: bool) -> String { + if renamed { + format!("eth{iface}-renamed") + } else { + format!("eth{iface}") + } + } + + fn config_for(iface: usize, renamed: bool) -> RouterInterfaceConfig { + let mut config = RouterInterfaceConfig::new(&name_of(iface, renamed), ifindexes()[iface]); + config.set_iftype(IfType::Unknown); + config + } + + /// What the model believes about one interface. + #[derive(Debug, Clone, PartialEq)] + struct IfaceState { + name: String, + admin: IfState, + oper: IfState, + /// The vrf it is attached to, by index into [`vrf_ids`]. + attached: Option, + addresses: BTreeSet, + } + + #[derive(Debug, Clone)] + struct Model { + interfaces: BTreeMap, + vrfs: BTreeSet, + } + + impl Model { + fn new() -> Self { + Self { + interfaces: BTreeMap::new(), + vrfs: BTreeSet::new(), + } + } + } + + /// The table, the vrfs it attaches to, and the reader that sees both. + struct World { + iftw: IfTableWriter, + iftr: IfTableReader, + vrftable: VrfTable, + } + + fn world() -> World { + let (fibtw, _fibtr) = FibTableWriter::new(); + let (iftw, iftr) = IfTableWriter::new(); + World { + iftw, + iftr, + vrftable: VrfTable::new(fibtw), + } + } + + /// Add an interface, which is refused if one with that index is already there. + fn apply_add(world: &mut World, model: &mut Model, iface: usize, renamed: bool) { + let result = world.iftw.add_interface(config_for(iface, renamed)); + if model.interfaces.contains_key(&iface) { + assert!(result.is_err(), "a duplicate interface was accepted"); + return; + } + assert!(result.is_ok(), "a new interface was refused: {result:?}"); + model.interfaces.insert( + iface, + IfaceState { + name: name_of(iface, renamed), + admin: IfState::Up, + oper: IfState::Unknown, + attached: None, + addresses: BTreeSet::new(), + }, + ); + } + + /// Reconfigure an interface. + /// + /// The configuration is replaced; the runtime state is not. Addresses, the vrf attachment and + /// the operational state are all learned out of band, and a reconfiguration knows nothing about + /// any of them. + fn apply_mod(world: &mut World, model: &mut Model, iface: usize, renamed: bool) { + let result = world.iftw.mod_interface(config_for(iface, renamed)); + let Some(state) = model.interfaces.get_mut(&iface) else { + assert!(result.is_err(), "an unknown interface was modified"); + return; + }; + assert!(result.is_ok(), "a known interface was refused: {result:?}"); + state.name = name_of(iface, renamed); + state.admin = IfState::Up; + } + + /// Attach an interface to a vrf. Both halves have to be there: the interface, and a vrf with a + /// fib whose id can be named. + fn apply_attach(world: &mut World, model: &mut Model, iface: usize, vrf: usize) { + let result = + world + .iftw + .attach_interface_to_vrf(ifindexes()[iface], vrf_ids()[vrf], &world.vrftable); + let attachable = model.interfaces.contains_key(&iface) && model.vrfs.contains(&vrf); + assert_eq!(result.is_ok(), attachable, "attaching {iface} to {vrf}"); + if attachable { + model + .interfaces + .get_mut(&iface) + .unwrap_or_else(|| unreachable!()) + .attached = Some(vrf); + } + } + + /// Detach every interface attached to `vrf`. + fn detach_all_from(model: &mut Model, vrf: usize) { + for state in model.interfaces.values_mut() { + if state.attached == Some(vrf) { + state.attached = None; + } + } + } + + /// Remove a vrf, which detaches the interfaces that were attached to it. + fn apply_remove_vrf(world: &mut World, model: &mut Model, vrf: usize) { + let result = world.vrftable.remove_vrf(vrf_ids()[vrf], &mut world.iftw); + assert_eq!( + result.is_ok(), + model.vrfs.contains(&vrf), + "removing vrf {vrf}" + ); + if model.vrfs.remove(&vrf) { + detach_all_from(model, vrf); + } + } + + fn apply(world: &mut World, model: &mut Model, change: &Change) { + let ifaces = ifindexes(); + let vrfs = vrf_ids(); + match change { + Change::AddInterface { iface, renamed } => apply_add(world, model, *iface, *renamed), + Change::ModInterface { iface, renamed } => apply_mod(world, model, *iface, *renamed), + Change::AttachToVrf { iface, vrf } => apply_attach(world, model, *iface, *vrf), + Change::RemoveVrf { vrf } => apply_remove_vrf(world, model, *vrf), + Change::DelInterface { iface } => { + world.iftw.del_interface(ifaces[*iface]); + model.interfaces.remove(iface); + } + Change::AddAddress { iface, address } => { + world + .iftw + .add_ip_address(ifaces[*iface], addresses()[*address]); + // an address for an interface we do not have is dropped, and the caller is not + // told: the error is logged inside `absorb_first` and goes no further + if let Some(state) = model.interfaces.get_mut(iface) { + state.addresses.insert(*address); + } + } + Change::DelAddress { iface, address } => { + world + .iftw + .del_ip_address(ifaces[*iface], addresses()[*address]); + if let Some(state) = model.interfaces.get_mut(iface) { + state.addresses.remove(address); + } + } + Change::SetOperState { iface, state } => { + world + .iftw + .set_iface_oper_state(ifaces[*iface], states()[*state]); + if let Some(entry) = model.interfaces.get_mut(iface) { + entry.oper = states()[*state]; + } + } + Change::SetAdminState { iface, state } => { + world + .iftw + .set_iface_admin_state(ifaces[*iface], states()[*state]); + if let Some(entry) = model.interfaces.get_mut(iface) { + entry.admin = states()[*state]; + } + } + Change::Detach { iface } => { + world.iftw.detach_interface(ifaces[*iface]); + if let Some(state) = model.interfaces.get_mut(iface) { + state.attached = None; + } + } + Change::DetachVrf { vrf } => { + world.iftw.detach_interfaces_from_vrf(vrfs[*vrf]); + detach_all_from(model, *vrf); + } + Change::AddVrf { vrf } => { + let config = RouterVrfConfig::new(vrfs[*vrf], &format!("vrf{vrf}")); + let result = world.vrftable.add_vrf(&config); + assert_eq!( + result.is_ok(), + !model.vrfs.contains(vrf), + "adding vrf {vrf}" + ); + model.vrfs.insert(*vrf); + } + } + } + + fn check(world: &World, model: &Model, at: &str) { + let ifaces = ifindexes(); + let vrfs = vrf_ids(); + let addrs = addresses(); + + for view in [ + world.iftw.enter().unwrap_or_else(|| unreachable!()), + world.iftr.enter().unwrap_or_else(|| unreachable!()), + ] { + assert_eq!(view.len(), model.interfaces.len(), "interface count {at}"); + + for (index, ifindex) in ifaces.iter().enumerate() { + let Some(iface) = view.get_interface(*ifindex) else { + assert!( + !model.interfaces.contains_key(&index), + "interface {index} missing {at}" + ); + continue; + }; + let want = model + .interfaces + .get(&index) + .unwrap_or_else(|| panic!("interface {index} unexpected {at}")); + + assert_eq!(iface.ifindex, *ifindex, "filed under the wrong key {at}"); + assert_eq!(iface.name, want.name, "name of {index} {at}"); + assert_eq!(iface.admin_state, want.admin, "admin state of {index} {at}"); + assert_eq!(iface.oper_state, want.oper, "oper state of {index} {at}"); + + let held: BTreeSet = (0..addrs.len()) + .filter(|i| iface.addresses.contains(&addrs[*i])) + .collect(); + assert_eq!(held, want.addresses, "addresses of {index} {at}"); + assert_eq!( + iface.addresses.len(), + want.addresses.len(), + "stray addresses on {index} {at}" + ); + + match (&iface.attachment, want.attached) { + (None, None) => (), + (Some(Attachment::Vrf(key)), Some(vrf)) => { + assert_eq!(*key, FibKey::Id(vrfs[vrf]), "attachment of {index} {at}"); + } + (got, want) => { + panic!("attachment of {index} is {got:?}, expected {want:?} {at}") + } + } + + // and the vrf it names still exists. Nothing in the types ties an attachment to the + // life of the vrf it points at; `VrfTable::remove_vrf` detaching them is the whole + // of it + if let Some(Attachment::Vrf(FibKey::Id(vrfid))) = &iface.attachment { + assert!( + world.vrftable.contains(*vrfid), + "interface {index} is attached to vrf {vrfid}, which is gone {at}" + ); + } + } + } + } + + /// The pools and the constants that index them agree. + #[test] + fn the_pools_are_the_size_the_generator_thinks() { + assert_eq!(ifindexes().len(), usize::from(NUM_IFACES)); + assert_eq!(vrf_ids().len(), usize::from(NUM_VRFS)); + assert_eq!(addresses().len(), usize::from(NUM_ADDRESSES)); + assert_eq!(states().len(), usize::from(NUM_STATES)); + // the default vrf is excluded, so every vrf in the pool can be removed + assert!(!vrf_ids().contains(&Vrf::DEFAULT_VRFID)); + assert_ne!(name_of(0, false), name_of(0, true)); + } + + /// After any sequence of changes, the interface table holds what the model says -- and no + /// interface is left attached to a vrf that has gone. + #[test] + fn an_interface_tables_state_and_attachments_stay_in_step() { + bolero::check!() + .with_generator(ChangeSequences) + .cloned() + .for_each(|changes: Vec| { + let mut world = world(); + let mut model = Model::new(); + + check(&world, &model, "on a fresh table"); + for (step, change) in changes.iter().enumerate() { + apply(&mut world, &mut model, change); + check(&world, &model, &format!("at step {step} of {changes:?}")); + } + }); + } +} From 9df9cbe86c8bb89493200b54475c5811c4459cf3 Mon Sep 17 00:00:00 2001 From: Daniel Noland Date: Fri, 7 Aug 2026 15:31:17 -0600 Subject: [PATCH 32/33] test(routing): Property-test the adjacency table's publish discipline The adjacency table is a map and its contents are dull. What is not dull is the *publishing*. `add_adjacency`, `del_adjacency` and `clear` each take a `publish` flag, and `AtResolver::refresh_atable_from_proc` depends on it: every poll clears the whole table with `publish: false`, adds back every entry the kernel reported with `publish: false`, and publishes once at the end. So the property is not about the map, it is about what a reader may see: > a reader sees the table as of the last publish, and never an intermediate state If it could see an intermediate one, the egress stage would find an empty adjacency table on every ARP poll and have no destination mac for anything. Breaking `clear` so it publishes unconditionally shows exactly that: the counterexample is two changes long and the failure message is "the table emptied under a reader mid-refresh". The model therefore holds two states -- what the writer has appended, and what a reader is entitled to see -- and the generator carries the `publish` flag on every mutation. A second test spells the same claim out in the shape the resolver uses, since that is the sequence whose failure has the consequence. Also three tests for the one part of the resolver that does not need `/proc`: resolving the device name an ARP entry carries to an interface index. Every entry the kernel reports goes through it and an unresolvable one is dropped, so the distinction between "no such device" and "a device whose index we cannot represent" has to survive -- the first is `Ok(None)` and drops the entry quietly, the second is an error that says why. `InterfaceIndex` is non-zero and the kernel should never report zero, but the number comes from outside. Verified by breaking four things: `clear` publishing unconditionally, an adjacency keyed by address alone so two interfaces collide, `del_adjacency` losing its key, and an interface index of zero becoming a miss rather than an error. Each fails. `atable/atablerw.rs` 66.7% -> 80.0%; `adjacency.rs` and `resolver.rs` were already at 85.4% and 79.7% from the live test that reads `/proc`, and did not move -- the new tests make claims about lines that already ran. `routing/src` as a whole 67.3% -> 67.4%, which is the honest measure of how little coverage was left to win here. Co-Authored-By: Claude Opus 5 (1M context) Signed-off-by: Daniel Noland --- routing/src/atable/atablerw.rs | 295 +++++++++++++++++++++++++++++++++ routing/src/atable/resolver.rs | 74 +++++++++ 2 files changed, 369 insertions(+) diff --git a/routing/src/atable/atablerw.rs b/routing/src/atable/atablerw.rs index 3b4ae9eb75..c173b63caf 100644 --- a/routing/src/atable/atablerw.rs +++ b/routing/src/atable/atablerw.rs @@ -82,3 +82,298 @@ impl AtableReaderFactory { AtableReader(self.0.handle()) } } + +/// Model-based properties over the adjacency table and its left-right wrapper. +/// +/// The table is a map. What is not map-like is the *publishing*: `add_adjacency`, `del_adjacency` +/// and `clear` all take a `publish` flag, and `AtResolver::refresh_atable_from_proc` relies on it -- +/// it clears the table with `publish: false`, adds every entry it found with `publish: false`, and +/// publishes once at the end. So a reader must never observe the cleared-but-not-yet-repopulated +/// state. If it could, the egress stage would find an empty adjacency table on every refresh and +/// have no destination mac for anything. +#[cfg(test)] +mod atable_properties { + use super::*; + use bolero::{Driver, ValueGenerator}; + use net::eth::mac::Mac; + use std::collections::BTreeMap; + use std::net::IpAddr; + use std::ops::Bound::Included; + use std::str::FromStr; + + const NUM_IFACES: u8 = 2; + const NUM_ADDRESSES: u8 = 3; + const NUM_MACS: u8 = 2; + const MAX_CHANGES: u8 = 12; + + fn ifindexes() -> Vec { + (1..=u32::from(NUM_IFACES)) + .map(|i| InterfaceIndex::try_new(i).unwrap_or_else(|_| unreachable!())) + .collect() + } + + fn addresses() -> Vec { + ["10.0.0.1", "10.0.0.2", "10.0.0.3"] + .iter() + .map(|a| IpAddr::from_str(a).unwrap_or_else(|_| unreachable!())) + .collect() + } + + fn macs() -> Vec { + vec![ + Mac::from([0x00, 0xaa, 0x00, 0x00, 0x00, 0x01]), + Mac::from([0x00, 0xbb, 0x00, 0x00, 0x00, 0x02]), + ] + } + + /// One change, over indices into the pools above. Every mutation carries the writer's `publish` + /// flag, since whether a change is visible yet is the point. + #[derive(Debug, Clone)] + enum Change { + Add { + iface: usize, + address: usize, + mac: usize, + publish: bool, + }, + Del { + iface: usize, + address: usize, + publish: bool, + }, + Clear { + publish: bool, + }, + Publish, + } + + /// Draws sequences of [`Change`]s. + #[derive(Debug, Clone, Copy, Default)] + struct ChangeSequences; + + fn index(driver: &mut D, count: u8) -> Option { + driver + .gen_u8(Included(&0), Included(&(count - 1))) + .map(usize::from) + } + + impl ValueGenerator for ChangeSequences { + type Output = Vec; + + fn generate(&self, driver: &mut D) -> Option> { + let len = driver.gen_u8(Included(&0), Included(&MAX_CHANGES))?; + let mut out = Vec::with_capacity(usize::from(len)); + for _ in 0..len { + let change = match driver.gen_u8(Included(&0), Included(&3))? { + 0 => Change::Add { + iface: index(driver, NUM_IFACES)?, + address: index(driver, NUM_ADDRESSES)?, + mac: index(driver, NUM_MACS)?, + publish: driver.produce::()?, + }, + 1 => Change::Del { + iface: index(driver, NUM_IFACES)?, + address: index(driver, NUM_ADDRESSES)?, + publish: driver.produce::()?, + }, + 2 => Change::Clear { + publish: driver.produce::()?, + }, + _ => Change::Publish, + }; + out.push(change); + } + Some(out) + } + } + + /// The adjacencies, by `(interface, address)` index pair, to mac index. + type Entries = BTreeMap<(usize, usize), usize>; + + /// Two states, which is the whole point: what the writer has appended, and what a reader is + /// entitled to see. + #[derive(Debug, Clone, Default)] + struct Model { + appended: Entries, + published: Entries, + } + + impl Model { + fn publish(&mut self) { + self.published = self.appended.clone(); + } + + fn apply(&mut self, change: &Change) { + match change { + Change::Add { + iface, + address, + mac, + publish, + } => { + // an adjacency is keyed by the interface and address it carries, so re-learning + // one with a different mac replaces it + self.appended.insert((*iface, *address), *mac); + if *publish { + self.publish(); + } + } + Change::Del { + iface, + address, + publish, + } => { + self.appended.remove(&(*iface, *address)); + if *publish { + self.publish(); + } + } + Change::Clear { publish } => { + self.appended.clear(); + if *publish { + self.publish(); + } + } + Change::Publish => self.publish(), + } + } + } + + fn apply_to_table(writer: &mut AtableWriter, change: &Change) { + let ifaces = ifindexes(); + let addrs = addresses(); + match change { + Change::Add { + iface, + address, + mac, + publish, + } => writer.add_adjacency( + Adjacency::new(addrs[*address], ifaces[*iface], macs()[*mac]), + *publish, + ), + Change::Del { + iface, + address, + publish, + } => writer.del_adjacency(addrs[*address], ifaces[*iface], *publish), + Change::Clear { publish } => writer.clear(*publish), + Change::Publish => writer.publish(), + } + } + + /// Everything a reader can see of the table, as index pairs. + fn seen(table: &AdjacencyTable) -> Entries { + let ifaces = ifindexes(); + let addrs = addresses(); + let all = macs(); + let mut out = Entries::new(); + for (iface, ifindex) in ifaces.iter().enumerate() { + for (address, addr) in addrs.iter().enumerate() { + if let Some(adjacency) = table.get_adjacency(*addr, *ifindex) { + let mac = all + .iter() + .position(|m| *m == adjacency.get_mac()) + .unwrap_or_else(|| unreachable!()); + // and the adjacency agrees with the key it was found under + assert_eq!(adjacency.get_ifindex(), *ifindex, "adjacency ifindex"); + assert_eq!(adjacency.get_ip(), *addr, "adjacency address"); + out.insert((iface, address), mac); + } + } + } + assert_eq!( + out.len(), + table.len(), + "the table holds entries outside the pools" + ); + out + } + + /// The pools and the constants that index them agree. + #[test] + fn the_pools_are_the_size_the_generator_thinks() { + assert_eq!(ifindexes().len(), usize::from(NUM_IFACES)); + assert_eq!(addresses().len(), usize::from(NUM_ADDRESSES)); + assert_eq!(macs().len(), usize::from(NUM_MACS)); + // the macs must be distinguishable, or a replacement would not be visible + assert_ne!(macs()[0], macs()[1]); + } + + /// A reader sees the table as of the last publish, and never an intermediate state. + /// + /// The unpublished half is what `AtResolver::refresh_atable_from_proc` depends on: it clears and + /// repopulates the whole table without publishing, so between the two a reader must still see + /// the previous contents rather than nothing. + #[test] + fn a_reader_sees_the_table_as_of_the_last_publish() { + bolero::check!() + .with_generator(ChangeSequences) + .cloned() + .for_each(|changes: Vec| { + let (mut writer, reader) = AtableWriter::new(); + let mut model = Model::default(); + + for (step, change) in changes.iter().enumerate() { + apply_to_table(&mut writer, change); + model.apply(change); + let at = format!("at step {step} of {changes:?}"); + + let view = reader.enter().unwrap_or_else(|| unreachable!()); + assert_eq!(seen(&view), model.published, "reader {at}"); + } + + // and once everything is published, the reader sees everything appended + writer.publish(); + model.publish(); + let view = reader.enter().unwrap_or_else(|| unreachable!()); + assert_eq!(seen(&view), model.appended, "reader after a final publish"); + }); + } + + /// A refresh that clears and repopulates without publishing is invisible until it does. + /// + /// The same claim as above, spelled out in the shape the resolver actually uses, because that is + /// the sequence whose failure would empty the adjacency table under the egress stage. + #[test] + fn a_clear_and_repopulate_is_invisible_until_published() { + let (mut writer, reader) = AtableWriter::new(); + let ifindex = ifindexes()[0]; + let (old, new) = (addresses()[0], addresses()[1]); + + writer.add_adjacency(Adjacency::new(old, ifindex, macs()[0]), true); + assert!( + reader + .enter() + .unwrap_or_else(|| unreachable!()) + .get_adjacency(old, ifindex) + .is_some() + ); + + // a refresh: clear, repopulate, and only then publish + writer.clear(false); + writer.add_adjacency(Adjacency::new(new, ifindex, macs()[1]), false); + + let view = reader.enter().unwrap_or_else(|| unreachable!()); + assert!( + view.get_adjacency(old, ifindex).is_some(), + "the table emptied under a reader mid-refresh" + ); + assert!( + view.get_adjacency(new, ifindex).is_none(), + "an unpublished addition was visible" + ); + drop(view); + + writer.publish(); + let view = reader.enter().unwrap_or_else(|| unreachable!()); + assert!( + view.get_adjacency(old, ifindex).is_none(), + "the clear was lost" + ); + assert!( + view.get_adjacency(new, ifindex).is_some(), + "the addition was lost" + ); + } +} diff --git a/routing/src/atable/resolver.rs b/routing/src/atable/resolver.rs index c86f342e99..c97cfc1991 100644 --- a/routing/src/atable/resolver.rs +++ b/routing/src/atable/resolver.rs @@ -148,6 +148,80 @@ impl AtResolver { } } +/// Properties over the one part of the resolver that does not need `/proc`. +/// +/// `refresh_atable_from_proc` reads the kernel's ARP table and its interface list, so it cannot be +/// driven from a test. What can is the step between them: resolving the device name an ARP entry +/// carries to an interface index. Every ARP entry the kernel reports goes through it, and an entry +/// it cannot resolve is dropped -- so getting it wrong loses adjacencies silently. +#[cfg(test)] +mod resolver_properties { + use super::*; + use netdev::Interface; + + fn interface(index: u32, name: &str) -> Interface { + Interface { + index, + name: name.to_string(), + ..Interface::dummy() + } + } + + /// A device name resolves to the index of the interface that has it, and to nothing otherwise. + #[test] + fn a_device_name_resolves_to_its_own_interface() { + let interfaces = [interface(2, "eth0"), interface(3, "eth1")]; + + for (index, name) in [(2, "eth0"), (3, "eth1")] { + let found = get_interface_ifindex(&interfaces, name) + .unwrap_or_else(|e| unreachable!("{e}")) + .unwrap_or_else(|| panic!("{name} did not resolve")); + assert_eq!(found.to_u32(), index, "{name} resolved to the wrong index"); + } + + assert_eq!( + get_interface_ifindex(&interfaces, "eth2").unwrap_or_else(|e| unreachable!("{e}")), + None, + "an unknown device must resolve to nothing, not to something else" + ); + assert_eq!( + get_interface_ifindex(&[], "eth0").unwrap_or_else(|e| unreachable!("{e}")), + None, + "no interfaces, nothing to resolve to" + ); + } + + /// An interface index of zero is refused rather than turned into an adjacency. + /// + /// `InterfaceIndex` is non-zero, and the kernel should never report a zero index -- but the + /// resolver takes the number from outside, so the distinction between "no such device" and "a + /// device whose index we cannot represent" has to survive: the first is `Ok(None)` and drops the + /// entry quietly, the second is an error and says why. + #[test] + fn an_interface_index_of_zero_is_an_error_not_a_miss() { + let interfaces = [interface(0, "eth0")]; + assert!( + get_interface_ifindex(&interfaces, "eth0").is_err(), + "index zero must be an error" + ); + assert_eq!( + get_interface_ifindex(&interfaces, "eth1").unwrap_or_else(|e| unreachable!("{e}")), + None, + "a different name is still just a miss" + ); + } + + /// The first interface with a name wins, and a later one with the same name does not shadow it. + #[test] + fn a_repeated_device_name_resolves_to_the_first() { + let interfaces = [interface(2, "eth0"), interface(9, "eth0")]; + let found = get_interface_ifindex(&interfaces, "eth0") + .unwrap_or_else(|e| unreachable!("{e}")) + .unwrap_or_else(|| unreachable!()); + assert_eq!(found.to_u32(), 2); + } +} + #[cfg(test)] pub mod tests { use super::*; From de3a6b82beef85fdee62913602f27fe2ea82c258 Mon Sep 17 00:00:00 2001 From: Daniel Noland Date: Fri, 7 Aug 2026 00:21:05 -0600 Subject: [PATCH 33/33] fix(flow-entry): Add an insertion that will not displace a live flow Two packets of one new flow can reach a NAT stage at the same time. Packets of a 5-tuple usually land on one core, but nothing guarantees that, and each packet builds a pair of its own before inserting it. With a plain insert, whoever gets there second displaces the other's forward flow -- and only that half. The two reverse keys carry the allocations that made them, no two allocations agree, so the reverses never collide and the loser's is never displaced along with its partner. It stays in the table, live, mapping a translation whose allocation goes back to the pool as soon as the displaced forward half is collected. Return traffic for that public pair, once it has been handed out again, is then translated for whoever held it before. Two changes, either of which leaves a hole on its own. insert_if_absent stands aside when a live flow already holds the key, and reports that flow so the caller can go on with it. Arbitrating on one key is enough, because racing packets of a single flow share their forward key by construction: only whoever wins it inserts a reverse. A flow that is present but no longer live is displaced as before, since it is a corpse its timer has not swept yet and standing aside for one would drop a packet that could have replaced it. Displacing a flow now also invalidates the other half of its pair, wherever it happens. The race is not the only way to reach the orphan: the flow timer expires the two halves separately, so an expired forward half could be replaced by an ordinary insert while its partner was still live. That path needs no concurrency at all. Three tests, each of which fails against the code without its guard: a live flow keeps its key, a dead one does not, and displacing a flow takes its partner with it. Co-Authored-By: Claude Opus 5 (1M context) Signed-off-by: Daniel Noland Rebased onto a `related_pair` that is fallible and requires exactly one half of the pair to carry `INITIATOR`; both are invariants main gained after this was written. The test now marks the forward half and unwraps, matching the sibling test in `concurrent_fuzz.rs`. --- flow-entry/src/flow_table/mod.rs | 2 +- flow-entry/src/flow_table/table.rs | 247 ++++++++++++++++++++++++++--- 2 files changed, 230 insertions(+), 19 deletions(-) diff --git a/flow-entry/src/flow_table/mod.rs b/flow-entry/src/flow_table/mod.rs index a8d246a881..10307d6ace 100644 --- a/flow-entry/src/flow_table/mod.rs +++ b/flow-entry/src/flow_table/mod.rs @@ -9,7 +9,7 @@ pub mod table; mod concurrent_fuzz; pub use nf_lookup::FlowLookup; -pub use table::{FlowTable, FlowTableReadGuard}; +pub use table::{FlowTable, FlowTableReadGuard, Insertion}; pub use net::flows::atomic_instant::AtomicInstant; pub use net::flows::flow_info::*; diff --git a/flow-entry/src/flow_table/table.rs b/flow-entry/src/flow_table/table.rs index 27695322f7..0f15439e01 100644 --- a/flow-entry/src/flow_table/table.rs +++ b/flow-entry/src/flow_table/table.rs @@ -19,6 +19,16 @@ pub enum FlowTableError { CapacityExceeded, } +/// What [`FlowTable::insert_if_absent`] did. +#[derive(Debug)] +pub enum Insertion { + /// The flow is in the table, put there by this call. + Installed, + /// Nothing was inserted: a live flow already holds the key. That flow is carried here, since a + /// caller that lost the key generally wants to go on with whatever won it. + Occupied(Arc), +} + type Table = DashMap, RandomState>; #[derive(Debug)] @@ -212,25 +222,52 @@ impl FlowTable { }); } + /// Whether a flow may go into a table that is at its limit. + /// + /// Reject new flows when at capacity. Exception: always admit the second half of a + /// related pair (e.g. the reverse NAT flow) to avoid leaving a one-sided entry. + fn admit(&self, table: &Table, val: &Arc) -> Result<(), FlowTableError> { + if table.len() < self.capacity.load(Ordering::Relaxed) { + return Ok(()); + } + let has_related_in_table = val + .related + .as_ref() + .and_then(Weak::upgrade) + .is_some_and(|rel| rel.is_active()); + + if has_related_in_table { + Ok(()) + } else { + Err(FlowTableError::CapacityExceeded) + } + } + + /// Retire a flow that has just been displaced from the table, and the other half of its pair. + /// + /// The displaced flow is unreachable by key from here on. The half of its pair that is still + /// in the table is unreachable in a worse way: it stays live under a key of its own, mapping a + /// translation whose allocation goes back to the pool as soon as the displaced half is + /// collected. Return traffic matching that key once the public pair has been handed out again + /// would then be translated for whoever held it before. So the pair goes together. + fn displace(old: Option<&Arc>) { + let Some(old) = old else { + return; + }; + old.update_status(FlowStatus::Detached); + old.token.cancel(); + if let Some(related) = old.related.as_ref().and_then(Weak::upgrade) { + debug!("insert: invalidating the partner of a displaced flow"); + related.invalidate(); + } + } + fn insert_common(&self, val: &Arc) -> Result>, FlowTableError> { let table = self.table.read(); - let capacity = self.capacity.load(Ordering::Relaxed); let flow_key = val.flowkey(); debug!("insert: inserting flow {flow_key}"); - // Reject new flows when at capacity. Exception: always admit the second half of a - // related pair (e.g. the reverse NAT flow) to avoid leaving a one-sided entry. - if table.len() >= capacity { - let has_related_in_table = val - .related - .as_ref() - .and_then(Weak::upgrade) - .is_some_and(|rel| rel.is_active()); - - if !has_related_in_table { - return Err(FlowTableError::CapacityExceeded); - } - } + self.admit(&table, val)?; let result = table.insert(*flow_key, val.clone()); // Set Active only after the insert so that the invariant holds: Active iff in the @@ -243,10 +280,7 @@ impl FlowTable { #[cfg(not(any(feature = "shuttle", feature = "loom")))] Self::start_timer(self.table.clone(), val.clone()); - if let Some(old) = result.as_ref() { - old.update_status(FlowStatus::Detached); - old.token.cancel(); - } + Self::displace(result.as_ref()); let Some(ret) = result else { return Ok(None); @@ -259,6 +293,80 @@ impl FlowTable { Ok(Some(ret)) } + /// Add a flow entry to the table, unless a live flow already holds its key. + /// + /// Two packets of one new flow can reach a NAT stage at the same time. Packets of a 5-tuple + /// usually land on one core, but nothing guarantees that, and each packet builds a pair of its + /// own. With a plain insert whoever gets there second displaces the other's forward flow while + /// its reverse stays in the table: reverse keys carry the allocation that made them, no two + /// allocations agree, so the two never collide and the loser's reverse is never displaced with + /// its partner. That orphan outlives the allocation it maps, and return traffic for a public + /// pair handed out again later reaches whoever held it before. + /// + /// Arbitrating on one key is enough to prevent it, because racing packets of a single flow + /// share their forward key by construction: only the caller who wins it goes on to insert a + /// reverse. + /// + /// A flow that is present but no longer live is displaced as usual. It is a corpse its timer + /// has not swept yet, and standing aside for one would drop a packet that could have replaced + /// it. + /// + /// # Returns + /// + /// [`Insertion::Installed`] if the flow went in, or [`Insertion::Occupied`] with the live flow + /// that holds the key if it did not. + /// + /// # Panics + /// + /// Panics if this thread already holds the read lock on the table. + /// + /// # Errors + /// + /// Returns [`FlowTableError::CapacityExceeded`] when the table has reached its hard limit. + pub fn insert_if_absent(&self, val: &Arc) -> Result { + let table = self.table.read(); + let flow_key = val.flowkey(); + debug!("insert: inserting flow {flow_key} unless it is already held"); + + self.admit(&table, val)?; + + // The entry guard holds this key's shard, so testing the incumbent and standing aside + // cannot race another insert of the same key. It has to end before the table guard does, + // hence resolving to a value here rather than returning from inside the match. + let displaced = match table.entry(*flow_key) { + dashmap::Entry::Occupied(mut occupied) => { + if occupied.get().is_active() { + Err(occupied.get().clone()) + } else { + Ok(Some(occupied.insert(val.clone()))) + } + } + dashmap::Entry::Vacant(vacant) => { + vacant.insert(val.clone()); + Ok(None) + } + }; + let displaced = match displaced { + Ok(displaced) => displaced, + Err(held) => { + drop(table); + debug!("insert: flow {flow_key} is already held by a live flow"); + return Ok(Insertion::Occupied(held)); + } + }; + + // Active only after the insert, as in `insert_common`. + val.update_status(FlowStatus::Active); + drop(table); + + #[cfg(not(any(feature = "shuttle", feature = "loom")))] + Self::start_timer(self.table.clone(), val.clone()); + + Self::displace(displaced.as_ref()); + + Ok(Insertion::Installed) + } + /// Lookup a flow in the table. /// /// # Panics @@ -412,6 +520,7 @@ mod tests { #[concurrency_mode(std)] mod std_tests { + use net::flows::FlowInfoFlags; use std::time::Instant; use tracing_test::traced_test; @@ -634,6 +743,108 @@ mod tests { assert_eq!(flow_table.active_len().unwrap(), 0); } + fn key_for(src_port: u16) -> FlowKey { + FlowKey::new( + Some(VpcDiscriminant::VNI(Vni::new_checked(1).unwrap())), + "1.2.3.4".parse::().unwrap(), + "4.5.6.7".parse::().unwrap(), + IpProtoKey::Tcp(TcpProtoKey { + src_port: TcpPort::new_checked(src_port).unwrap(), + dst_port: TcpPort::new_checked(2048).unwrap(), + }), + ) + } + + /// A live flow keeps its key against a second insertion. + /// + /// This is what stops two packets of one new flow from each installing a pair. Without it + /// the second one displaces the first's forward flow, and the first's reverse -- which + /// carries an allocation of its own and so collides with nothing -- is left in the table + /// mapping a translation whose allocation is about to go back to the pool. + #[tokio::test] + async fn an_active_flow_holds_its_key_against_a_second_insertion() { + let flow_table = FlowTable::default(); + let key = key_for(1025); + let far_future = Instant::now() + Duration::from_hours(1); + + let first = Arc::new(FlowInfo::new(key, far_future)); + assert!(matches!( + flow_table.insert_if_absent(&first).unwrap(), + Insertion::Installed + )); + + let second = Arc::new(FlowInfo::new(key, far_future)); + let outcome = flow_table.insert_if_absent(&second).unwrap(); + let Insertion::Occupied(held) = outcome else { + panic!("a live flow was displaced by a second insertion: {outcome:?}"); + }; + assert!(Arc::ptr_eq(&held, &first), "the wrong flow was reported"); + + // The incumbent is still the one serving the key, and the newcomer never went in. + let found = flow_table.lookup(&key).expect("the key is still served"); + assert!(Arc::ptr_eq(&found, &first)); + assert_ne!(second.status(), FlowStatus::Active); + } + + /// A flow that is no longer live does not hold its key. + /// + /// Standing aside for one would drop a packet that could have replaced it: the entry is a + /// corpse whose timer has not swept it yet. + #[tokio::test] + async fn a_flow_that_is_not_live_is_displaced() { + let flow_table = FlowTable::default(); + let key = key_for(1026); + let far_future = Instant::now() + Duration::from_hours(1); + + let first = Arc::new(FlowInfo::new(key, far_future)); + flow_table.insert_if_absent(&first).unwrap(); + first.invalidate(); + + let second = Arc::new(FlowInfo::new(key, far_future)); + assert!( + matches!( + flow_table.insert_if_absent(&second).unwrap(), + Insertion::Installed + ), + "a flow that was no longer live held its key" + ); + let found = flow_table.lookup(&key).expect("the key is served"); + assert!(Arc::ptr_eq(&found, &second)); + } + + /// Displacing a flow takes the other half of its pair with it. + /// + /// The two halves have different keys, so nothing displaces the partner in its own right. + /// Left behind it stays live, mapping a translation whose allocation returns to the pool + /// with the half that was displaced. + #[tokio::test] + async fn displacing_a_flow_invalidates_its_partner() { + let flow_table = FlowTable::default(); + let (forward_key, reverse_key) = (key_for(1027), key_for(1028)); + let far_future = Instant::now() + Duration::from_hours(1); + + let (forward, reverse) = FlowInfo::related_pair( + far_future, + forward_key, + FlowInfoFlags::INITIATOR, + reverse_key, + FlowInfoFlags::default(), + ) + .expect("related_pair should succeed for distinct keys"); + flow_table.insert_from_arc(&forward).unwrap(); + flow_table.insert_from_arc(&reverse).unwrap(); + assert!(reverse.is_active()); + + // The unconditional path still displaces, and has to clean up after itself. + let replacement = Arc::new(FlowInfo::new(forward_key, far_future)); + flow_table.insert_from_arc(&replacement).unwrap(); + + assert!( + !reverse.is_active(), + "the partner of a displaced flow was left live in the table" + ); + } + #[tokio::test] async fn test_flow_table_capacity_exceeded() { let flow_table = FlowTable::default();