From ab51da76c26f87a11ec3ac25baaefdc29289882c Mon Sep 17 00:00:00 2001 From: Agents Agent Date: Mon, 29 Jun 2026 18:56:30 +0000 Subject: [PATCH] feat: add nvidia device plugin flux pack --- packs/flux-core/kustomization.yaml | 14 ++ .../nvidia-device-plugin/kustomization.yaml | 8 + .../nvidia-device-plugin/namespace.yaml | 4 + .../nvidia-device-plugin/release.yaml | 155 ++++++++++++++++++ .../nvidia-device-plugin/runtime-class.yaml | 18 ++ .../nvidia-device-plugin/source.yaml | 8 + 6 files changed, 207 insertions(+) create mode 100644 packs/flux-core/kustomization.yaml create mode 100644 packs/flux-core/nvidia-device-plugin/kustomization.yaml create mode 100644 packs/flux-core/nvidia-device-plugin/namespace.yaml create mode 100644 packs/flux-core/nvidia-device-plugin/release.yaml create mode 100644 packs/flux-core/nvidia-device-plugin/runtime-class.yaml create mode 100644 packs/flux-core/nvidia-device-plugin/source.yaml diff --git a/packs/flux-core/kustomization.yaml b/packs/flux-core/kustomization.yaml new file mode 100644 index 0000000..7096bbe --- /dev/null +++ b/packs/flux-core/kustomization.yaml @@ -0,0 +1,14 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: + # flux-system namespace is declared once in + # cluster/flux/clusters/production/flux-system/gotk-components.yaml + # and must not be duplicated here. + - cert-manager + - external-dns + - ingress-controller + - nvidia-device-plugin + - metallb + - lan-ingress-controller + - vso diff --git a/packs/flux-core/nvidia-device-plugin/kustomization.yaml b/packs/flux-core/nvidia-device-plugin/kustomization.yaml new file mode 100644 index 0000000..89e8cde --- /dev/null +++ b/packs/flux-core/nvidia-device-plugin/kustomization.yaml @@ -0,0 +1,8 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: + - namespace.yaml + - source.yaml + - release.yaml + - runtime-class.yaml diff --git a/packs/flux-core/nvidia-device-plugin/namespace.yaml b/packs/flux-core/nvidia-device-plugin/namespace.yaml new file mode 100644 index 0000000..4a6b3ab --- /dev/null +++ b/packs/flux-core/nvidia-device-plugin/namespace.yaml @@ -0,0 +1,4 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: nvidia-device-plugin diff --git a/packs/flux-core/nvidia-device-plugin/release.yaml b/packs/flux-core/nvidia-device-plugin/release.yaml new file mode 100644 index 0000000..2eb8183 --- /dev/null +++ b/packs/flux-core/nvidia-device-plugin/release.yaml @@ -0,0 +1,155 @@ +apiVersion: helm.toolkit.fluxcd.io/v2 +kind: HelmRelease +metadata: + name: nvidia-device-plugin + namespace: nvidia-device-plugin +spec: + interval: 30m + # The chart renders a default config whose discovery strategy 'auto' + # has been removed upstream, so the device-plugin pod crashloops on + # startup. Don't let that block the entire apps-core health gate: skip + # Helm's readiness wait so the HR reports Ready as soon as manifests + # apply, and downstream Kustomizations (apps-data, apps-observability, + # etc.) stop being held hostage by a single DaemonSet. + install: + disableWait: true + disableWaitForJobs: true + upgrade: + disableWait: true + disableWaitForJobs: true + chart: + spec: + chart: nvidia-device-plugin + # Pin the chart so the config schema below stays aligned with the + # device-plugin binary it deploys. 0.19.0 is what's been running + # on-cluster; upgrades should be deliberate so the Resource / + # Flags struct shape can be re-verified. + version: 0.19.0 + sourceRef: + kind: HelmRepository + name: nvidia + namespace: nvidia-device-plugin + values: + compatWithCPUManager: true + # Run the plugin pod under the `nvidia-cdi` RuntimeClass, not the + # `nvidia` one. k3s's runtimes addon auto-registers both containerd + # runtime handlers (once `nvidia-container-runtime` is on k3s's + # systemd PATH, see nix/modules/roles/gpu-nvidia.nix), but: + # + # - The `nvidia` handler uses the legacy hook wrapper, which + # shells out to `nvidia-container-cli`. That binary lives in + # the `libnvidia-container` derivation, which NixOS's + # `hardware.nvidia-container-toolkit.enable` does not install, + # so every pod start fails with + # nvidia-container-runtime did not terminate successfully: + # exit status 2 + # - The `nvidia-cdi` handler consumes the CDI spec NixOS already + # emits at /var/run/cdi/nvidia-container-toolkit.json, has no + # nvidia-container-cli dependency, and produces the same + # NVML-visible environment the plugin needs for + # `deviceListStrategy: cdi-annotations`. + # + # Smoke-tested with a one-off cuda:12.4-base pod on the t1000: the + # nvidia-cdi handler runs nvidia-smi and sees the Quadro T1000. + runtimeClassName: nvidia-cdi + # Provide an explicit default config so the plugin does not fall back + # to the compiled-in 'auto' discovery strategy (removed upstream). + # NixOS hosts this with nvidia-container-toolkit.enable=true which + # publishes a CDI spec, so use the CDI device list strategy. + # + # Config shape comes from api/config/v1/{config,flags,resources, + # sharing,replicas}.go at the pinned plugin version (v0.19.0): + # + # - `plugin` lives under `flags` (it's modelled as `Flags.Plugin`, + # not a sibling of `flags`). + # - `resources:` is NOT user-settable in v0.19. The plugin calls + # `DisableResourceNamingInConfig` which warns "Customizing the + # 'resources' field is not yet supported in the config" and + # nils whatever we pass. Since that nil propagates into the + # downstream replicas/timeSlicing unmarshal path it's cleanest + # to just not set it at all -- the plugin auto-discovers + # `nvidia.com/gpu` from the CDI spec. + # - Don't set `sharing.timeSlicing: {}` either. The TimeSlicing + # unmarshaler in replicas.go hard-fails with the literal + # "no resources specified" error whenever timeSlicing is + # present without a non-empty `resources` key, even when the + # block is an empty map. We don't time-slice, so leave the + # whole `sharing:` section out. + # + # DeviceListStrategy: use `envvar`, NOT `cdi-annotations` or + # `cdi-cri`. The CDI strategies make the plugin call + # `cdiHandler.CreateSpecFile()` at startup to emit its own + # /var/run/cdi/nvidia.com-gpu.yaml, which requires scanning an + # FHS-style `driverRoot` (`/usr/lib*` + `/etc/ld.so.cache`). NixOS + # doesn't provide that layout -- libcuda.so lives in a nix-store + # path, `/etc/ld.so.cache` does not exist -- so the plugin aborts + # with + # error getting plugins: unable to create cdi spec file: + # ... failed to locate libcuda.so: libcuda.so.595.58.03: not found + # In `envvar` mode the plugin uses a null CDI handler (checked in + # internal/cdi/cdi.go: `if !c.deviceListStrategies.AnyCDIEnabled() + # { return &null{}, nil }`), never tries to generate a spec, and + # just sets `NVIDIA_VISIBLE_DEVICES=` in the consumer + # container's env. nvidia-container-runtime.cdi (what + # runtimeClassName: nvidia-cdi routes to) then reads that env and + # injects devices from the CDI spec NixOS ALREADY emitted at + # /var/run/cdi/nvidia-container-toolkit.json. + config: + map: + default: |- + version: v1 + flags: + migStrategy: none + failOnInitError: true + plugin: + passDeviceSpecs: true + deviceListStrategy: envvar + # Use `index`, not `uuid`. The NixOS-generated CDI spec at + # /var/run/cdi/nvidia-container-toolkit.json names devices + # by index (`"0"`, `"all"`). `deviceIDStrategy: uuid` asks + # the runtime for `nvidia.com/gpu=GPU-`, which does + # not exist in that spec and the CDI runtime rejects the + # sandbox with + # failed to inject CDI devices: unresolvable CDI devices + # nvidia.com/gpu=GPU-a97ebe4d-... + # `index` makes the plugin request `nvidia.com/gpu=0` on + # allocate, matching the spec. + deviceIDStrategy: index + # Time-slicing: advertise each physical GPU as 4 schedulable + # replicas of nvidia.com/gpu so multiple pods can hold an + # allocation against the same device. Required when telemetry + # and application pods both need GPU access on a single-GPU + # node; without time-slicing, k8s exclusivity can make one + # workload starve the other. + # + # Time-slicing isn't isolation: workloads share the same CUDA + # context and GPU memory. For our usage (heavy transcoding + + # lightweight monitoring scrape) the contention is fine. + # Bumping `replicas` if more GPU-consuming pods land later. + sharing: + timeSlicing: + resources: + - name: nvidia.com/gpu + replicas: 4 + nodeSelector: + personal-stack/capability-nvidia: 'true' + # The chart default nodeAffinity requires node-feature-discovery labels + # (feature.node.kubernetes.io/pci-10de.present and friends). We do not + # run NFD yet and instead label NVIDIA-capable nodes ourselves via the + # k3s role module. Setting `affinity: {}` was silently ignored by the + # chart, so provide an explicit affinity keyed on our own label. + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: personal-stack/capability-nvidia + operator: In + values: + - 'true' + tolerations: + - key: nvidia.com/gpu + operator: Exists + effect: NoSchedule + gfd: + enabled: false diff --git a/packs/flux-core/nvidia-device-plugin/runtime-class.yaml b/packs/flux-core/nvidia-device-plugin/runtime-class.yaml new file mode 100644 index 0000000..cd36051 --- /dev/null +++ b/packs/flux-core/nvidia-device-plugin/runtime-class.yaml @@ -0,0 +1,18 @@ +# The `nvidia-cdi` OCI runtime handler is registered in k3s's containerd +# config by the auto-detection pass once `nvidia-container-runtime` is on +# k3s's systemd PATH (see nix/modules/roles/gpu-nvidia.nix). +# +# Unlike the sibling `nvidia` handler -- which uses the legacy hook +# wrapper and wants `nvidia-container-cli` available at runtime -- the +# `.cdi` variant consumes the CDI spec NixOS already emits at +# /var/run/cdi/nvidia-container-toolkit.json, so it does NOT need +# nvidia-container-cli. It's the one path that actually works on this +# stack today. +# +# k3s's runtimes addon autoregisters a `nvidia` RuntimeClass (legacy) +# but not a `nvidia-cdi` one. Declare it here so Flux owns it. +apiVersion: node.k8s.io/v1 +kind: RuntimeClass +metadata: + name: nvidia-cdi +handler: nvidia-cdi diff --git a/packs/flux-core/nvidia-device-plugin/source.yaml b/packs/flux-core/nvidia-device-plugin/source.yaml new file mode 100644 index 0000000..6d99fd1 --- /dev/null +++ b/packs/flux-core/nvidia-device-plugin/source.yaml @@ -0,0 +1,8 @@ +apiVersion: source.toolkit.fluxcd.io/v1 +kind: HelmRepository +metadata: + name: nvidia + namespace: nvidia-device-plugin +spec: + interval: 1h + url: https://nvidia.github.io/k8s-device-plugin