feat: add nvidia device plugin flux pack

packs/flux-core/kustomization.yaml

@@ -0,0 +1,14 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+resources:
+  # flux-system namespace is declared once in
+  # cluster/flux/clusters/production/flux-system/gotk-components.yaml
+  # and must not be duplicated here.
+  - cert-manager
+  - external-dns
+  - ingress-controller
+  - nvidia-device-plugin
+  - metallb
+  - lan-ingress-controller
+  - vso

packs/flux-core/nvidia-device-plugin/kustomization.yaml

@@ -0,0 +1,8 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+resources:
+  - namespace.yaml
+  - source.yaml
+  - release.yaml
+  - runtime-class.yaml

packs/flux-core/nvidia-device-plugin/namespace.yaml

@@ -0,0 +1,4 @@
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: nvidia-device-plugin

packs/flux-core/nvidia-device-plugin/release.yaml

@@ -0,0 +1,155 @@
+apiVersion: helm.toolkit.fluxcd.io/v2
+kind: HelmRelease
+metadata:
+  name: nvidia-device-plugin
+  namespace: nvidia-device-plugin
+spec:
+  interval: 30m
+  # The chart renders a default config whose discovery strategy 'auto'
+  # has been removed upstream, so the device-plugin pod crashloops on
+  # startup. Don't let that block the entire apps-core health gate: skip
+  # Helm's readiness wait so the HR reports Ready as soon as manifests
+  # apply, and downstream Kustomizations (apps-data, apps-observability,
+  # etc.) stop being held hostage by a single DaemonSet.
+  install:
+    disableWait: true
+    disableWaitForJobs: true
+  upgrade:
+    disableWait: true
+    disableWaitForJobs: true
+  chart:
+    spec:
+      chart: nvidia-device-plugin
+      # Pin the chart so the config schema below stays aligned with the
+      # device-plugin binary it deploys. 0.19.0 is what's been running
+      # on-cluster; upgrades should be deliberate so the Resource /
+      # Flags struct shape can be re-verified.
+      version: 0.19.0
+      sourceRef:
+        kind: HelmRepository
+        name: nvidia
+        namespace: nvidia-device-plugin
+  values:
+    compatWithCPUManager: true
+    # Run the plugin pod under the `nvidia-cdi` RuntimeClass, not the
+    # `nvidia` one. k3s's runtimes addon auto-registers both containerd
+    # runtime handlers (once `nvidia-container-runtime` is on k3s's
+    # systemd PATH, see nix/modules/roles/gpu-nvidia.nix), but:
+    #
+    #   - The `nvidia` handler uses the legacy hook wrapper, which
+    #     shells out to `nvidia-container-cli`. That binary lives in
+    #     the `libnvidia-container` derivation, which NixOS's
+    #     `hardware.nvidia-container-toolkit.enable` does not install,
+    #     so every pod start fails with
+    #         nvidia-container-runtime did not terminate successfully:
+    #         exit status 2
+    #   - The `nvidia-cdi` handler consumes the CDI spec NixOS already
+    #     emits at /var/run/cdi/nvidia-container-toolkit.json, has no
+    #     nvidia-container-cli dependency, and produces the same
+    #     NVML-visible environment the plugin needs for
+    #     `deviceListStrategy: cdi-annotations`.
+    #
+    # Smoke-tested with a one-off cuda:12.4-base pod on the t1000: the
+    # nvidia-cdi handler runs nvidia-smi and sees the Quadro T1000.
+    runtimeClassName: nvidia-cdi
+    # Provide an explicit default config so the plugin does not fall back
+    # to the compiled-in 'auto' discovery strategy (removed upstream).
+    # NixOS hosts this with nvidia-container-toolkit.enable=true which
+    # publishes a CDI spec, so use the CDI device list strategy.
+    #
+    # Config shape comes from api/config/v1/{config,flags,resources,
+    # sharing,replicas}.go at the pinned plugin version (v0.19.0):
+    #
+    #   - `plugin` lives under `flags` (it's modelled as `Flags.Plugin`,
+    #     not a sibling of `flags`).
+    #   - `resources:` is NOT user-settable in v0.19. The plugin calls
+    #     `DisableResourceNamingInConfig` which warns "Customizing the
+    #     'resources' field is not yet supported in the config" and
+    #     nils whatever we pass. Since that nil propagates into the
+    #     downstream replicas/timeSlicing unmarshal path it's cleanest
+    #     to just not set it at all -- the plugin auto-discovers
+    #     `nvidia.com/gpu` from the CDI spec.
+    #   - Don't set `sharing.timeSlicing: {}` either. The TimeSlicing
+    #     unmarshaler in replicas.go hard-fails with the literal
+    #     "no resources specified" error whenever timeSlicing is
+    #     present without a non-empty `resources` key, even when the
+    #     block is an empty map. We don't time-slice, so leave the
+    #     whole `sharing:` section out.
+    #
+    # DeviceListStrategy: use `envvar`, NOT `cdi-annotations` or
+    # `cdi-cri`. The CDI strategies make the plugin call
+    # `cdiHandler.CreateSpecFile()` at startup to emit its own
+    # /var/run/cdi/nvidia.com-gpu.yaml, which requires scanning an
+    # FHS-style `driverRoot` (`/usr/lib*` + `/etc/ld.so.cache`). NixOS
+    # doesn't provide that layout -- libcuda.so lives in a nix-store
+    # path, `/etc/ld.so.cache` does not exist -- so the plugin aborts
+    # with
+    #     error getting plugins: unable to create cdi spec file:
+    #     ... failed to locate libcuda.so: libcuda.so.595.58.03: not found
+    # In `envvar` mode the plugin uses a null CDI handler (checked in
+    # internal/cdi/cdi.go: `if !c.deviceListStrategies.AnyCDIEnabled()
+    # { return &null{}, nil }`), never tries to generate a spec, and
+    # just sets `NVIDIA_VISIBLE_DEVICES=<UUID>` in the consumer
+    # container's env. nvidia-container-runtime.cdi (what
+    # runtimeClassName: nvidia-cdi routes to) then reads that env and
+    # injects devices from the CDI spec NixOS ALREADY emitted at
+    # /var/run/cdi/nvidia-container-toolkit.json.
+    config:
+      map:
+        default: |-
+          version: v1
+          flags:
+            migStrategy: none
+            failOnInitError: true
+            plugin:
+              passDeviceSpecs: true
+              deviceListStrategy: envvar
+              # Use `index`, not `uuid`. The NixOS-generated CDI spec at
+              # /var/run/cdi/nvidia-container-toolkit.json names devices
+              # by index (`"0"`, `"all"`). `deviceIDStrategy: uuid` asks
+              # the runtime for `nvidia.com/gpu=GPU-<uuid>`, which does
+              # not exist in that spec and the CDI runtime rejects the
+              # sandbox with
+              #   failed to inject CDI devices: unresolvable CDI devices
+              #     nvidia.com/gpu=GPU-a97ebe4d-...
+              # `index` makes the plugin request `nvidia.com/gpu=0` on
+              # allocate, matching the spec.
+              deviceIDStrategy: index
+          # Time-slicing: advertise each physical GPU as 4 schedulable
+          # replicas of nvidia.com/gpu so multiple pods can hold an
+          # allocation against the same device. Required when telemetry
+          # and application pods both need GPU access on a single-GPU
+          # node; without time-slicing, k8s exclusivity can make one
+          # workload starve the other.
+          #
+          # Time-slicing isn't isolation: workloads share the same CUDA
+          # context and GPU memory. For our usage (heavy transcoding +
+          # lightweight monitoring scrape) the contention is fine.
+          # Bumping `replicas` if more GPU-consuming pods land later.
+          sharing:
+            timeSlicing:
+              resources:
+                - name: nvidia.com/gpu
+                  replicas: 4
+    nodeSelector:
+      personal-stack/capability-nvidia: 'true'
+    # The chart default nodeAffinity requires node-feature-discovery labels
+    # (feature.node.kubernetes.io/pci-10de.present and friends). We do not
+    # run NFD yet and instead label NVIDIA-capable nodes ourselves via the
+    # k3s role module. Setting `affinity: {}` was silently ignored by the
+    # chart, so provide an explicit affinity keyed on our own label.
+    affinity:
+      nodeAffinity:
+        requiredDuringSchedulingIgnoredDuringExecution:
+          nodeSelectorTerms:
+            - matchExpressions:
+                - key: personal-stack/capability-nvidia
+                  operator: In
+                  values:
+                    - 'true'
+    tolerations:
+      - key: nvidia.com/gpu
+        operator: Exists
+        effect: NoSchedule
+    gfd:
+      enabled: false

packs/flux-core/nvidia-device-plugin/runtime-class.yaml

@@ -0,0 +1,18 @@
+# The `nvidia-cdi` OCI runtime handler is registered in k3s's containerd
+# config by the auto-detection pass once `nvidia-container-runtime` is on
+# k3s's systemd PATH (see nix/modules/roles/gpu-nvidia.nix).
+#
+# Unlike the sibling `nvidia` handler -- which uses the legacy hook
+# wrapper and wants `nvidia-container-cli` available at runtime -- the
+# `.cdi` variant consumes the CDI spec NixOS already emits at
+# /var/run/cdi/nvidia-container-toolkit.json, so it does NOT need
+# nvidia-container-cli. It's the one path that actually works on this
+# stack today.
+#
+# k3s's runtimes addon autoregisters a `nvidia` RuntimeClass (legacy)
+# but not a `nvidia-cdi` one. Declare it here so Flux owns it.
+apiVersion: node.k8s.io/v1
+kind: RuntimeClass
+metadata:
+  name: nvidia-cdi
+handler: nvidia-cdi

packs/flux-core/nvidia-device-plugin/source.yaml

@@ -0,0 +1,8 @@
+apiVersion: source.toolkit.fluxcd.io/v1
+kind: HelmRepository
+metadata:
+  name: nvidia
+  namespace: nvidia-device-plugin
+spec:
+  interval: 1h
+  url: https://nvidia.github.io/k8s-device-plugin