perf(ci): make the nix dependency build reusable across revisions

.cargo/config.toml

@@ -13,7 +13,7 @@ CARGO_LLVM_COV_BUILD_DIR = { value = "target/llvm-cov/target", relative = true,
 
 [build]
 # Register `emulated` so cfg_attr sites don't trip unexpected_cfgs natively.
-rustflags = ["--cfg=tokio_unstable", "--check-cfg=cfg(emulated)"]
+rustflags = ["--cfg=tokio_unstable", "--check-cfg=cfg(emulated)", "--check-cfg=cfg(instrumented)"]
 
 [target.wasm32-wasip1]
 # Trailing `--` separates wasmtime's CLI from the module + module args

.github/actions/nix-shell/action.yml

@@ -19,7 +19,58 @@ inputs:
 runs:
   using: "composite"
   steps:
+    # Realizing the dev shell costs every job ~45s on a cold store: ~22s
+    # re-fetching the npins tarballs and ~20s substituting ~340 store paths.
+    # The lab runner image can carry both already warm (see the `nix.sh` change
+    # drafted for githedgehog/gha-runner), which reduces that to near zero.
+    #
+    # `install-nix-action` aborts as soon as it finds nix on PATH, and that
+    # early exit also skips the nix.conf and NIX_PATH setup it would otherwise
+    # do.  Supply the parts that matter here so a job is configured the same
+    # way whether or not the image was prewarmed.  Nothing below assumes a
+    # prewarmed image: on one without nix this step is a no-op and the install
+    # proceeds exactly as before.
+    - name: "Configure a prewarmed nix"
+      id: "prewarmed"
+      shell: "bash"
+      env:
+        # Indirect through the environment rather than interpolating into the
+        # script body.
+        github_token: "${{ github.token }}"
+      # The only values written to the environment file below are two
+      # literals and `RUNNER_TEMP`, all runner controlled; none of them can
+      # carry anything from the event payload.
+      run: | # zizmor: ignore[github-env]
+        set -euo pipefail
+        if ! command -v nix >/dev/null 2>&1; then
+          printf 'prewarmed=false\n' >>"${GITHUB_OUTPUT}"
+          exit 0
+        fi
+        printf 'prewarmed=true\n' >>"${GITHUB_OUTPUT}"
+        # `access-tokens` is per-run, so it is the one setting the image cannot
+        # bake in.  Without it a pin that misses the image's tarball cache is
+        # fetched anonymously and shares github.com's unauthenticated rate
+        # limit with every other runner behind the lab's address.  A
+        # single-user nix honors this file without a `trusted-users` entry.
+        install -d -m "0700" "${HOME}/.config/nix"
+        umask "0077"
+        cat >"${HOME}/.config/nix/nix.conf" <<EOF
+        experimental-features = nix-command flakes
+        always-allow-substitutes = true
+        access-tokens = github.com=${github_token}
+        EOF
+        # Nothing in this repository resolves <nixpkgs> today -- npins pins
+        # every source -- but the installed path sets this, so match it.
+        printf 'NIX_PATH=nixpkgs=channel:nixpkgs-unstable\n' >>"${GITHUB_ENV}"
+        # Also match where the installed path puts nix's scratch space: the
+        # job's temp directory on the work volume, not the container's own
+        # writable layer.  The image cannot set this; it is per-run.
+        if [ -z "${TMPDIR:-}" ]; then
+          printf 'TMPDIR=%s\n' "${RUNNER_TEMP}" >>"${GITHUB_ENV}"
+        fi
+
     - name: "Install nix"
+      if: "${{ steps.prewarmed.outputs.prewarmed != 'true' }}"
       uses: "cachix/install-nix-action@8aa03977d8d733052d78f4e008a241fd1dbf36b3" # v31.10.6
       with:
         github_access_token: "${{ github.token }}"
@@ -32,6 +83,8 @@ runs:
         signingKey: '${{ inputs.cachix_signing_key }}'
         # prettier-ignore
         authToken: '${{ inputs.cachix_auth_token }}'
+        # Exclude source-volatile builds and image paths that can reintroduce them through a closure.
+        pushFilter: '(-dataplane-volatile-|-(dataplane|core-viewer|dev-debugger|syscall-tracer)-(customisation-layer|conf\.json)$|-stream-(dataplane|core-viewer|dev-debugger|syscall-tracer|frr)$|-(frr-conf\.json|layers\.json|excludePaths)$)'
 
     - name: "use nix shell"
       uses: "rrbutani/use-nix-shell-action@59a52b2b9bbfe3cc0e7deb8f9059abe37a439edf" # v1.1.0

.github/workflows/README.md

@@ -61,17 +61,22 @@ Production artifacts are produced via nix builds in a separate CI workflow.
 - `ci:+miri` - Run Miri checks
 - `ci:+wasm` - Run the WASM build check
 - `ci:+concurrency` - Run Shuttle and Loom tests
+- `ci:+debug-images` - Also build and push the core viewer, DAP debugger, and
+  syscall tracer images. They are built on main, in the merge queue, and on
+  dispatch regardless; this is for when the build itself needs debugging
 - `ci:+cross` - Build all cross-platform containers
 - `ci:+cross/full` - Also run the workspace test suite under qemu-user, on the
   two aarch64 musl legs. Gated like every other job, so the merge queue and
   `ci:+merge-ready` include it
 - `ci:+vlab` - Run VLAB tests on this PR
 - `ci:+hlab` - Run HLAB tests on this PR
 - `ci:+release` - Enable release tests for VLAB/HLAB on this PR
+- `ci:-vlab` - Skip VLAB and HLAB tests on this PR, even with `ci:+merge-ready`
 - `ci:-upgrade` - Disable upgrade tests on this PR
 
 Labels are additive, and optional: a pull request needs none of them.
-`ci:-upgrade` is the sole exception, subtracting a job that would otherwise run.
+`ci:-vlab` and `ci:-upgrade` are the exceptions, subtracting jobs that would
+otherwise run.
 
 Adding a label starts a **new** workflow run, and that run repeats the default
 jobs as well as the ones the label enabled.
@@ -94,13 +99,85 @@ If those queue failures stop being rare, the phasing is worth revisiting.
 - Coverage: `debug` by default; `fuzz` on deep runs
 - Miri: required on deep runs; opt-in on pull requests with `ci:+miri`
 - Containers: debug/release for dataplane and FRR; release for validator
+- Debug images (core viewer, DAP debugger, syscall tracer): deep runs only,
+  or on a pull request with `ci:+debug-images`
 - VLAB configurations: spine-leaf fabric mode, L2VNI/L3VNI VPC modes,
   with gateway enabled
 
 ### Artifacts
 
 - Container images pushed to GitHub Container Registry (GHCR)
 - Release containers published on tag pushes via `just push`
+- `ghcr.io/githedgehog/dataplane/core-viewer` opens a core file from the lab.
+  It carries gdb plus the unstripped binaries and sources for the matching
+  `ghcr.io/githedgehog/dataplane` build.
+  Pull the tag matching the build the core came from; symbols only line up with
+  the exact version and profile that produced it.
+  The entrypoint takes the core as its only argument:
+
+  ```console
+  docker run --rm -it -v /path/to/cores:/cores \
+    ghcr.io/githedgehog/dataplane/core-viewer:TAG /cores/core.1234
+  ```
+
+- `ghcr.io/githedgehog/dataplane/dev-debugger` debugs a live dataplane from an
+  editor. It carries bugstalker, which understands Rust's std collections and
+  enum layouts, and listens for a Debug Adapter Protocol client on port 4711.
+  Publish the port and point the editor's DAP client at it:
+
+  ```console
+  docker run --rm -p 4711:4711 ghcr.io/githedgehog/dataplane/dev-debugger:TAG
+  ```
+
+  Connecting does not by itself start anything. In remote-DAP mode bugstalker
+  waits for the client's `launch` request to name the program, so the editor
+  has to send `program`, and any dataplane arguments as `args`. A request
+  without `program` is rejected with `launch: missing arguments.program`.
+  For VS Code, in `.vscode/launch.json`:
+
+  ```json
+  {
+    "type": "bs",
+    "request": "launch",
+    "name": "dataplane (container)",
+    "debugServer": 4711,
+    "program": "/bin/dataplane",
+    "args": []
+  }
+  ```
+
+  For `nvim-dap`:
+
+  ```lua
+  dap.adapters.bs = { type = "server", host = "127.0.0.1", port = 4711 }
+  dap.configurations.rust = {
+    {
+      type = "bs",
+      request = "launch",
+      name = "dataplane (container)",
+      program = "/bin/dataplane",
+      args = {},
+    },
+  }
+  ```
+
+- `ghcr.io/githedgehog/dataplane/syscall-tracer` records what the dataplane
+  asks the kernel for, as JSON, using lurk.
+  It carries the same stripped binaries the release image ships, since nothing
+  here symbolizes, which is why it is a fraction of the size of the other two:
+
+  ```console
+  docker run --rm ghcr.io/githedgehog/dataplane/syscall-tracer:TAG > trace.jsonl
+  ```
+
+  The stream is one JSON object per line, except that tracing child threads
+  makes lurk announce each one with a bare `Attaching to child <pid>` line.
+  Filter those out if the consumer needs strict JSONL:
+
+  ```console
+  jq -R 'fromjson? // empty' < trace.jsonl
+  ```
+
 - Coverage reports from each `coverage/<profile>` job, kept for 7 days:
   - `coverage-html-<profile>.tar.gz` - `llvm-cov` HTML report, including the
     per-branch counts that Codecov does not render. Unpack and open

.github/workflows/dev.yml

@@ -71,6 +71,8 @@ jobs:
     runs-on: "ubuntu-latest"
     outputs:
       container_profiles: "${{ steps.container-profiles.outputs.value }}"
+      parallel: "${{ steps.parallel.outputs.value }}"
+      container_targets: "${{ steps.container-targets.outputs.value }}"
       profiles: "${{ steps.profiles.outputs.value }}"
       concurrency: "${{ steps.concurrency.outputs.value }}"
       cross: "${{ steps.cross.outputs.value }}"
@@ -85,6 +87,27 @@ jobs:
         with:
           persist-credentials: "false"
 
+      # Deep runs get a wider slice of the lab pool.  A queued merge blocks
+      # everything behind it and its result is what actually gates the merge,
+      # so it is worth finishing sooner even though the extra runners come out
+      # of the pool the pull requests are waiting in.  Pull requests stay at
+      # one so they cannot crowd the queue out.
+      #
+      # `strategy` cannot read `env`, only `github`/`inputs`/`needs`/`vars`, so
+      # this travels as a `plan` output.  Jobs pick it up through
+      # `strategy.max-parallel`, which is also what feeds `ci::parallel` into
+      # `JUST_VARS` -- the per-job core budget follows without a second switch.
+      - id: "parallel"
+        env:
+          EVENT: "${{ github.event_name }}"
+        run: |
+          set -euo pipefail
+          case "${EVENT}" in
+            merge_group | push) value="4" ;;
+            *) value="1" ;;
+          esac
+          printf 'value=%s\n' "${value}" >>"${GITHUB_OUTPUT}"
+
       - id: "miri"
         uses: &gate "./.github/actions/ci-gate"
         with:
@@ -122,6 +145,16 @@ jobs:
           on-value: '["debug", "release", "fuzz"]'
           off-value: '["debug"]'
 
+      # Debug images are opt-in on pull requests because of their size.
+      - id: "container-targets"
+        uses: *gate
+        with:
+          labels: "debug-images"
+          # Keep this on one line: `ci-gate` writes the value to GITHUB_OUTPUT
+          # with a plain printf, which a multi-line value would corrupt.
+          on-value: '["frr.dataplane", "dataplane", "dataplane-core-viewer", "dataplane-dev-debugger", "dataplane-syscall-tracer", "validator"]'
+          off-value: '["frr.dataplane", "dataplane", "validator"]'
+
       # Lab jobs require release images but not other release/fuzz checks.
       - id: "container-profiles"
         uses: *gate
@@ -183,9 +216,18 @@ jobs:
       JUST_VARS: "ci::parallel=${{ strategy.max-parallel }}"
     strategy:
       fail-fast: false
-      max-parallel: 1
+      max-parallel: ${{ fromJSON(needs.plan.outputs.parallel) }}
       matrix:
         profile: "${{ fromJSON(needs.plan.outputs.profiles) }}"
+        exclude:
+          # The fuzz profile is release plus the safety checks, so checking it
+          # here mostly re-verifies what `check/release` already covered, at
+          # the price of compiling the workspace optimised a second time --
+          # 944s in the differential run, the most expensive job in it.  The
+          # fuzz build is exercised where it matters: `coverage/fuzz`, both
+          # sanitizers, and the fuzzing campaign these worker-minutes are being
+          # freed up for.
+          - profile: "fuzz"
     steps:
       - *checkout
 
@@ -219,6 +261,15 @@ jobs:
           recipe: "ci::check-doctest"
           recipe_args: "${{ matrix.profile }}"
 
+      # Nothing built the API docs, so they rotted: thirty broken intra-doc
+      # links across five crates, plus a missing `--check-cfg` in RUSTDOCFLAGS
+      # that no amount of link fixing would have gotten past.
+      - name: "docs"
+        uses: *just
+        with:
+          recipe: "ci::check-docs"
+          recipe_args: "${{ matrix.profile }}"
+
       - &verify-clean-tree
         name: "verify-clean-tree"
         run: |
@@ -319,6 +370,16 @@ jobs:
         with:
           recipe: "markdownlint"
 
+      # Guards the property the dependency split rests on.  Two per-commit
+      # inputs have already leaked into it and each cost several CI runs to
+      # notice, because the symptom is a cache miss rather than a failure.
+      - name: "check-deps-reuse"
+        id: "check-deps-reuse"
+        continue-on-error: true
+        uses: *just
+        with:
+          recipe: "check-deps-reuse"
+
       - name: "license-headers"
         id: "license-headers"
         continue-on-error: true
@@ -337,6 +398,7 @@ jobs:
             pinact=${{ steps.pinact.outcome }}
             actionlint=${{ steps.actionlint.outcome }}
             markdownlint=${{ steps.markdownlint.outcome }}
+            check-deps-reuse=${{ steps.check-deps-reuse.outcome }}
             license-headers=${{ steps.license-headers.outcome }}
         run: |
           set -euo pipefail
@@ -365,19 +427,22 @@ jobs:
     env: *ci-env
     strategy:
       fail-fast: false
-      max-parallel: 1
+      max-parallel: ${{ fromJSON(needs.plan.outputs.parallel) }}
       matrix:
-        nix-target:
-          - frr.dataplane
-          - dataplane
-          - validator
+        nix-target: "${{ fromJSON(needs.plan.outputs.container_targets) }}"
         # TODO: enable cfi and safe-stack on release when possible
         profile: "${{ fromJSON(needs.plan.outputs.container_profiles) }}"
         exclude:
           - nix-target: frr.dataplane
             profile: fuzz
           - nix-target: dataplane
             profile: fuzz
+          - nix-target: dataplane-core-viewer
+            profile: fuzz
+          - nix-target: dataplane-dev-debugger
+            profile: fuzz
+          - nix-target: dataplane-syscall-tracer
+            profile: fuzz
           - nix-target: validator
             profile: debug
           - nix-target: validator
@@ -391,6 +456,17 @@ jobs:
         with:
           recipe: "ci::push-container"
           recipe_args: "${{ matrix.nix-target }} ${{ matrix.profile }} ${{ needs.version.outputs.version }}"
+
+      # A debug image that builds is not a debug image that works.  Two of
+      # these shipped with entrypoints that could not do what the README
+      # documents, and building them said nothing about it.
+      - name: "smoke"
+        if: "${{ startsWith(matrix.nix-target, 'dataplane-') }}"
+        uses: *just
+        with:
+          recipe: "ci::smoke-container"
+          recipe_args: "${{ matrix.nix-target }} ${{ matrix.profile }}"
+
       - *verify-clean-tree
       - *tmate
 
@@ -405,7 +481,7 @@ jobs:
     env: *ci-env
     strategy:
       fail-fast: false
-      max-parallel: 1
+      max-parallel: ${{ fromJSON(needs.plan.outputs.parallel) }}
       matrix:
         sanitizer:
           - thread
@@ -449,7 +525,7 @@ jobs:
     env: *ci-env
     strategy:
       fail-fast: false
-      max-parallel: 1
+      max-parallel: ${{ fromJSON(needs.plan.outputs.parallel) }}
       # Fuzz provides optimized coverage while retaining safety checks.
       matrix:
         profile: "${{ fromJSON(needs.plan.outputs.profiles) }}"
@@ -639,6 +715,10 @@ jobs:
       JUST_VARS: "ci::parallel=${{ strategy.max-parallel }}"
     strategy:
       fail-fast: false
+      # Deliberately not widened.  `ci.just` turns `ci::parallel` into
+      # `share = 1/N`, and only `ci::cross`/`ci::cross-test` consume it, so
+      # raising this divides each job's core budget rather than adding
+      # throughput unless the runner pods are genuinely core-isolated.
       max-parallel: 1
       matrix:
         platform:
@@ -706,14 +786,16 @@ jobs:
     with:
       # ci:+hlab is required to enable hybrid lab tests on PR
       # ci:+vlab is required to enable virtual lab tests on PR
+      # ci:-vlab opts back out, including for ci:+merge-ready
       # ci:-upgrade disables upgrade tests on PR
       # hlab is disabled for main and merge_queue till we have gateway tests for it
       # ci:+merge-ready mirrors the merge queue, which skips HLAB.
       skip: >-
         ${{
           github.event_name == 'pull_request'
           && (
-            matrix.hybrid && !contains(github.event.pull_request.labels.*.name, 'ci:+hlab')
+            contains(github.event.pull_request.labels.*.name, 'ci:-vlab')
+            || matrix.hybrid && !contains(github.event.pull_request.labels.*.name, 'ci:+hlab')
             || !matrix.hybrid
                && !contains(github.event.pull_request.labels.*.name, 'ci:+vlab')
                && !contains(github.event.pull_request.labels.*.name, 'ci:+merge-ready')