refactor: migrate to dra amdgpu - #3449
Draft
Tanguille wants to merge 7 commits into
Draft
Conversation
@@ metadata.labels.app.kubernetes.io/controller @@
# apps/v1/DaemonSet/kube-system/generic-device-plugin
! ± value change
- generic-device-plugin
+ generic-device-plugin-igpu
@@ spec.selector.matchLabels.app.kubernetes.io/controller @@
# apps/v1/DaemonSet/kube-system/generic-device-plugin
! ± value change
- generic-device-plugin
+ generic-device-plugin-igpu
@@ spec.template.metadata.labels.app.kubernetes.io/controller @@
# apps/v1/DaemonSet/kube-system/generic-device-plugin
! ± value change
- generic-device-plugin
+ generic-device-plugin-igpu
@@ spec.template.spec.containers.app.args @@
# apps/v1/DaemonSet/kube-system/generic-device-plugin
! - one list entry removed:
- - |
- name: dri
- groups:
- - count: 4
- paths:
- - path: /dev/dri
- - path: /dev/kfd
! + one list entry added:
+ - |
+ name: dri
+ groups:
+ - count: 2
+ paths:
+ - path: /dev/dri
@@ spec.template.spec.nodeSelector @@
# apps/v1/DaemonSet/kube-system/generic-device-plugin
! - one map entry removed:
- amd.com/gpu: "true"
! + one map entry added:
+ amd.com/igpu: "true"
@@ spec.template.spec @@
# apps/v1/Deployment/media/fileflows
! + one map entry added:
+ resourceClaims:
+ - name: gpu
+ resourceClaimName: shared-dgpu
@@ spec.template.spec.containers.app.resources @@
# apps/v1/Deployment/media/fileflows
! + one map entry added:
+ claims:
+ - name: gpu
@@ spec.template.spec.containers.app.resources.limits @@
# apps/v1/Deployment/media/fileflows
! - one map entry removed:
- squat.ai/dri: 1
@@ spec.template.spec.containers.app.resources.requests @@
# apps/v1/Deployment/media/fileflows
! - one map entry removed:
- squat.ai/dri: 1
@@ spec.template.spec @@
# apps/v1/Deployment/media/jellyfin
! + one map entry added:
+ resourceClaims:
+ - name: gpu
+ resourceClaimName: shared-dgpu
@@ spec.template.spec.containers.app.resources @@
# apps/v1/Deployment/media/jellyfin
! + one map entry added:
+ claims:
+ - name: gpu
@@ spec.template.spec.containers.app.resources.limits @@
# apps/v1/Deployment/media/jellyfin
! - one map entry removed:
- squat.ai/dri: 1
@@ spec.template.spec.containers.app.resources.requests @@
# apps/v1/Deployment/media/jellyfin
! - one map entry removed:
- squat.ai/dri: 1
@@ (root level) @@
# apps/v1/DaemonSet/kube-system/generic-device-plugin-generic-device-plugin-igpu
! - one document removed:
- apiVersion: apps/v1
- kind: DaemonSet
- metadata:
- name: generic-device-plugin-generic-device-plugin-igpu
- namespace: kube-system
- labels:
- app.kubernetes.io/controller: generic-device-plugin-igpu
- app.kubernetes.io/instance: generic-device-plugin
- app.kubernetes.io/managed-by: Helm
- app.kubernetes.io/name: generic-device-plugin
- helm.toolkit.fluxcd.io/name: generic-device-plugin
- helm.toolkit.fluxcd.io/namespace: kube-system
- spec:
- revisionHistoryLimit: 3
- selector:
- matchLabels:
- app.kubernetes.io/controller: generic-device-plugin-igpu
- app.kubernetes.io/instance: generic-device-plugin
- app.kubernetes.io/name: generic-device-plugin
- template:
- metadata:
- annotations: null
- labels:
- app.kubernetes.io/controller: generic-device-plugin-igpu
- app.kubernetes.io/instance: generic-device-plugin
- app.kubernetes.io/name: generic-device-plugin
- spec:
- automountServiceAccountToken: false
- containers:
- - name: app
- image: "ghcr.io/squat/generic-device-plugin:latest@sha256:dc192e164c69b03f156765793a1be62ca437709ae477b27ca7d8f3dcf5021576"
- args:
- - "--domain=squat.ai"
- - "--device"
- - |
- name: dri
- groups:
- - count: 2
- paths:
- - path: /dev/dri
-
- securityContext:
- privileged: true
- resources:
- limits:
- memory: 64Mi
- requests:
- cpu: 10m
- memory: 16Mi
- volumeMounts:
- - name: dev
- mountPath: /dev
- - name: device-plugin
- mountPath: /var/lib/kubelet/device-plugins
- dnsPolicy: ClusterFirst
- enableServiceLinks: false
- hostIPC: false
- hostNetwork: false
- hostPID: false
- nodeSelector:
- amd.com/igpu: "true"
- priorityClassName: system-node-critical
- serviceAccountName: generic-device-plugin
- volumes:
- - name: dev
- hostPath:
- path: /dev
- - name: device-plugin
- hostPath:
- path: /var/lib/kubelet/device-plugins
@@ (root level) @@
# v1/ServiceAccount/observability/drm-exporter
! - one document removed:
- apiVersion: v1
- kind: ServiceAccount
- metadata:
- name: drm-exporter
- namespace: observability
- labels:
- app.kubernetes.io/component: exporter
- app.kubernetes.io/instance: drm-exporter
- app.kubernetes.io/managed-by: Helm
- app.kubernetes.io/name: drm-exporter
- helm.toolkit.fluxcd.io/name: drm-exporter
- helm.toolkit.fluxcd.io/namespace: observability
- automountServiceAccountToken: false
@@ (root level) @@
# v1/ConfigMap/observability/drm-exporter-dashboard
! - one document removed:
- apiVersion: v1
- kind: ConfigMap
- metadata:
- name: drm-exporter-dashboard
- namespace: observability
- labels:
- app.kubernetes.io/component: exporter
- app.kubernetes.io/instance: drm-exporter
- app.kubernetes.io/managed-by: Helm
- app.kubernetes.io/name: drm-exporter
- helm.toolkit.fluxcd.io/name: drm-exporter
- helm.toolkit.fluxcd.io/namespace: observability
- data:
- drm-exporter.json: |
-
- {
- "uid": "drm-exporter",
- "title": "drm-exporter \u2014 GPU metrics",
- "tags": [
- "drm-exporter",
- "gpu",
- "intel",
- "amd"
- ],
- "editable": true,
- "schemaVersion": 39,
- "timezone": "",
- "refresh": "30s",
- "time": {
- "from": "now-6h",
- "to": "now"
- },
- "timepicker": {},
- "annotations": {
- "list": [
- {
- "builtIn": 1,
- "datasource": {
- "type": "grafana",
- "uid": "-- Grafana --"
- },
- "enable": true,
- "hide": true,
- "name": "Annotations & Alerts",
- "type": "dashboard"
- }
- ]
- },
- "templating": {
- "list": [
- {
- "name": "datasource",
- "type": "datasource",
- "query": "prometheus",
- "label": "Data source",
- "current": {},
- "hide": 0,
- "refresh": 1,
- "regex": ""
- },
- {
- "name": "node",
- "type": "query",
- "label": "Node",
- "datasource": {
- "type": "prometheus",
- "uid": "${datasource}"
- },
- "query": {
- "qryType": 1,
- "query": "label_values(drm_info, node)",
- "refId": "PrometheusVariableQueryEditor-VariableQuery"
- },
- "current": {},
- "refresh": 2,
- "includeAll": true,
- "multi": true,
- "allValue": ".*",
- "sort": 1
- },
- {
- "name": "device",
- "type": "query",
- "label": "Device",
- "datasource": {
- "type": "prometheus",
- "uid": "${datasource}"
- },
- "query": {
- "qryType": 1,
- "query": "label_values(drm_info{node=~\"$node\"}, device)",
- "refId": "PrometheusVariableQueryEditor-VariableQuery"
- },
- "current": {},
- "refresh": 2,
- "includeAll": true,
- "multi": true,
- "allValue": ".*",
- "sort": 1
- }
- ]
- },
- "panels": [
- {
- "id": 1,
- "type": "stat",
- "title": "GPUs",
- "datasource": {
- "type": "prometheus",
- "uid": "${datasource}"
- },
- "gridPos": {
- "h": 4,
- "w": 4,
- "x": 0,
- "y": 0
- },
- "fieldConfig": {
- "defaults": {
- "unit": "none",
- "color": {
- "mode": "thresholds"
- },
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green",
- "value": null
- }
- ]
- }
- },
- "overrides": []
- },
- "options": {
- "reduceOptions": {
- "calcs": [
- "lastNotNull"
- ],
- "fields": "",
- "values": false
- },
- "colorMode": "value",
- "graphMode": "area",
- "textMode": "auto",
- "justifyMode": "auto"
- },
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${datasource}"
- },
- "editorMode": "code",
- "expr": "count(group by (node, device) (drm_info{node=~\"$node\", device=~\"$device\"}))",
- "legendFormat": "",
- "range": true,
- "instant": false,
- "format": "time_series",
- "refId": "A"
- }
- ]
- },
- {
- "id": 2,
- "type": "stat",
- "title": "Peak engine busy",
- "datasource": {
- "type": "prometheus",
- "uid": "${datasource}"
- },
- "gridPos": {
- "h": 4,
- "w": 4,
- "x": 4,
- "y": 0
- },
- "fieldConfig": {
- "defaults": {
- "unit": "percentunit",
- "color": {
- "mode": "thresholds"
- },
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green",
- "value": null
- }
- ]
- },
- "min": 0,
- "max": 1
- },
- "overrides": []
- },
- "options": {
- "reduceOptions": {
- "calcs": [
- "max"
- ],
- "fields": "",
- "values": false
- },
- "colorMode": "value",
- "graphMode": "area",
- "textMode": "auto",
- "justifyMode": "auto"
- },
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${datasource}"
- },
- "editorMode": "code",
- "expr": "max(drm_engine_utilization_ratio{node=~\"$node\", device=~\"$device\"})",
- "legendFormat": "",
- "range": true,
- "instant": false,
- "format": "time_series",
- "refId": "A"
- }
- ]
- },
- {
- "id": 3,
- "type": "table",
- "title": "GPU inventory",
- "datasource": {
- "type": "prometheus",
- "uid": "${datasource}"
- },
- "gridPos": {
- "h": 4,
- "w": 16,
- "x": 8,
- "y": 0
- },
- "fieldConfig": {
- "defaults": {},
- "overrides": []
- },
- "options": {
- "showHeader": true
- },
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${datasource}"
- },
- "editorMode": "code",
- "expr": "drm_info{node=~\"$node\", device=~\"$device\"}",
- "legendFormat": "",
- "range": false,
- "instant": true,
- "format": "table",
- "refId": "A"
- }
- ],
- "transformations": [
- {
- "id": "filterFieldsByName",
- "options": {
- "include": {
- "names": [
- "node",
- "device",
- "vendor",
- "model",
- "driver",
- "type",
- "revision",
- "pci_id",
- "dev_node"
- ]
- }
- }
- }
- ]
- },
- {
- "id": 4,
- "type": "timeseries",
- "title": "Engine utilization",
- "datasource": {
- "type": "prometheus",
- "uid": "${datasource}"
- },
- "gridPos": {
- "h": 8,
- "w": 24,
- "x": 0,
- "y": 4
- },
- "fieldConfig": {
- "defaults": {
- "unit": "percentunit",
- "custom": {
- "drawStyle": "line",
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "fillOpacity": 10,
- "showPoints": "never",
- "spanNulls": false,
- "axisPlacement": "auto"
- },
- "color": {
- "mode": "palette-classic"
- },
- "min": 0,
- "max": 1
- },
- "overrides": []
- },
- "options": {
- "legend": {
- "displayMode": "table",
- "placement": "bottom",
- "calcs": [
- "lastNotNull",
- "max"
- ]
- },
- "tooltip": {
- "mode": "multi",
- "sort": "desc"
- }
- },
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${datasource}"
- },
- "editorMode": "code",
- "expr": "drm_engine_utilization_ratio{node=~\"$node\", device=~\"$device\"}",
- "legendFormat": "{{node}} \u00b7 {{device}} \u00b7 {{engine}}",
- "range": true,
- "instant": false,
- "format": "time_series",
- "refId": "A"
- }
- ]
- },
- {
- "id": 5,
- "type": "timeseries",
- "title": "Memory used",
- "datasource": {
- "type": "prometheus",
- "uid": "${datasource}"
- },
- "gridPos": {
- "h": 8,
- "w": 12,
- "x": 0,
- "y": 12
- },
- "fieldConfig": {
- "defaults": {
- "unit": "bytes",
- "custom": {
- "drawStyle": "line",
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "fillOpacity": 10,
- "showPoints": "never",
- "spanNulls": false,
- "axisPlacement": "auto"
- },
- "color": {
- "mode": "palette-classic"
- }
- },
- "overrides": []
- },
- "options": {
- "legend": {
- "displayMode": "table",
- "placement": "bottom",
- "calcs": [
- "lastNotNull",
- "max"
- ]
- },
- "tooltip": {
- "mode": "multi",
- "sort": "desc"
- }
- },
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${datasource}"
- },
- "editorMode": "code",
- "expr": "drm_memory_used_bytes{node=~\"$node\", device=~\"$device\"}",
- "legendFormat": "{{node}} \u00b7 {{device}} \u00b7 {{pool}}",
- "range": true,
- "instant": false,
- "format": "time_series",
- "refId": "A"
- }
- ]
- },
- {
- "id": 6,
- "type": "timeseries",
- "title": "Memory utilization",
- "datasource": {
- "type": "prometheus",
- "uid": "${datasource}"
- },
- "gridPos": {
- "h": 8,
- "w": 12,
- "x": 12,
- "y": 12
- },
- "fieldConfig": {
- "defaults": {
- "unit": "percentunit",
- "custom": {
- "drawStyle": "line",
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "fillOpacity": 10,
- "showPoints": "never",
- "spanNulls": false,
- "axisPlacement": "auto"
- },
- "color": {
- "mode": "palette-classic"
- },
- "min": 0,
- "max": 1
- },
- "overrides": []
- },
- "options": {
- "legend": {
- "displayMode": "table",
- "placement": "bottom",
- "calcs": [
- "lastNotNull",
- "max"
- ]
- },
- "tooltip": {
- "mode": "multi",
- "sort": "desc"
- }
- },
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${datasource}"
- },
- "editorMode": "code",
- "expr": "drm_memory_used_bytes{node=~\"$node\", device=~\"$device\"} / drm_memory_total_bytes{node=~\"$node\", device=~\"$device\"}",
- "legendFormat": "{{node}} \u00b7 {{device}} \u00b7 {{pool}}",
- "range": true,
- "instant": false,
- "format": "time_series",
- "refId": "A"
- }
- ]
- },
- {
- "id": 7,
- "type": "timeseries",
- "title": "Clock frequency",
- "datasource": {
- "type": "prometheus",
- "uid": "${datasource}"
- },
- "gridPos": {
- "h": 8,
- "w": 12,
- "x": 0,
- "y": 20
- },
- "fieldConfig": {
- "defaults": {
- "unit": "hertz",
- "custom": {
- "drawStyle": "line",
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "fillOpacity": 10,
- "showPoints": "never",
- "spanNulls": false,
- "axisPlacement": "auto"
- },
- "color": {
- "mode": "palette-classic"
- }
- },
- "overrides": []
- },
- "options": {
- "legend": {
- "displayMode": "table",
- "placement": "bottom",
- "calcs": [
- "lastNotNull",
- "max"
- ]
- },
- "tooltip": {
- "mode": "multi",
- "sort": "desc"
- }
- },
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${datasource}"
- },
- "editorMode": "code",
- "expr": "drm_frequency_hertz{node=~\"$node\", device=~\"$device\"}",
- "legendFormat": "{{node}} \u00b7 {{device}} \u00b7 {{domain}} \u00b7 {{kind}}",
- "range": true,
- "instant": false,
- "format": "time_series",
- "refId": "A"
- }
- ]
- },
- {
- "id": 8,
- "type": "timeseries",
- "title": "Power draw",
- "datasource": {
- "type": "prometheus",
- "uid": "${datasource}"
- },
- "gridPos": {
- "h": 8,
- "w": 12,
- "x": 12,
- "y": 20
- },
- "fieldConfig": {
- "defaults": {
- "unit": "watt",
- "custom": {
- "drawStyle": "line",
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "fillOpacity": 10,
- "showPoints": "never",
- "spanNulls": false,
- "axisPlacement": "auto"
- },
- "color": {
- "mode": "palette-classic"
- }
- },
- "overrides": []
- },
- "options": {
- "legend": {
- "displayMode": "table",
- "placement": "bottom",
- "calcs": [
- "lastNotNull",
- "max"
- ]
- },
- "tooltip": {
- "mode": "multi",
- "sort": "desc"
- }
- },
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${datasource}"
- },
- "editorMode": "code",
- "expr": "drm_power_watts{node=~\"$node\", device=~\"$device\"}",
- "legendFormat": "{{node}} \u00b7 {{device}} \u00b7 {{domain}}",
- "range": true,
- "instant": false,
- "format": "time_series",
- "refId": "A"
- }
- ]
- },
- {
- "id": 9,
- "type": "timeseries",
- "title": "Temperature",
- "datasource": {
- "type": "prometheus",
- "uid": "${datasource}"
- },
- "gridPos": {
- "h": 8,
- "w": 12,
- "x": 0,
- "y": 28
- },
- "fieldConfig": {
- "defaults": {
- "unit": "celsius",
- "custom": {
- "drawStyle": "line",
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "fillOpacity": 10,
- "showPoints": "never",
- "spanNulls": false,
- "axisPlacement": "auto",
- "thresholdsStyle": {
- "mode": "dashed"
- }
- },
- "color": {
- "mode": "palette-classic"
- },
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green",
- "value": null
- },
- {
- "color": "orange",
- "value": 80
- },
- {
- "color": "red",
- "value": 95
- }
- ]
- }
- },
- "overrides": []
- },
- "options": {
- "legend": {
- "displayMode": "table",
- "placement": "bottom",
- "calcs": [
- "lastNotNull",
- "max"
- ]
- },
- "tooltip": {
- "mode": "multi",
- "sort": "desc"
- }
- },
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${datasource}"
- },
- "editorMode": "code",
- "expr": "drm_temperature_celsius{node=~\"$node\", device=~\"$device\"}",
- "legendFormat": "{{node}} \u00b7 {{device}} \u00b7 {{sensor}}",
- "range": true,
- "instant": false,
- "format": "time_series",
- "refId": "A"
- }
- ]
- },
- {
- "id": 10,
- "type": "timeseries",
- "title": "Fan speed",
- "datasource": {
- "type": "prometheus",
- "uid": "${datasource}"
- },
- "gridPos": {
- "h": 8,
- "w": 12,
- "x": 12,
- "y": 28
- },
- "fieldConfig": {
- "defaults": {
- "unit": "rotrpm",
- "custom": {
- "drawStyle": "line",
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "fillOpacity": 10,
- "showPoints": "never",
- "spanNulls": false,
- "axisPlacement": "auto"
- },
- "color": {
- "mode": "palette-classic"
- }
- },
- "overrides": []
- },
- "options": {
- "legend": {
- "displayMode": "table",
- "placement": "bottom",
- "calcs": [
- "lastNotNull",
- "max"
- ]
- },
- "tooltip": {
- "mode": "multi",
- "sort": "desc"
- }
- },
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${datasource}"
- },
- "editorMode": "code",
- "expr": "drm_fan_speed_rpm{node=~\"$node\", device=~\"$device\"}",
- "legendFormat": "{{node}} \u00b7 {{device}} \u00b7 {{fan}}",
- "range": true,
- "instant": false,
- "format": "time_series",
- "refId": "A"
- }
- ]
- }
- ]
- }
@@ (root level) @@
# v1/Service/observability/drm-exporter
! - one document removed:
- apiVersion: v1
- kind: Service
- metadata:
- name: drm-exporter
- namespace: observability
- labels:
- app.kubernetes.io/component: exporter
- app.kubernetes.io/instance: drm-exporter
- app.kubernetes.io/managed-by: Helm
- app.kubernetes.io/name: drm-exporter
- helm.toolkit.fluxcd.io/name: drm-exporter
- helm.toolkit.fluxcd.io/namespace: observability
- spec:
- type: ClusterIP
- selector:
- app.kubernetes.io/instance: drm-exporter
- app.kubernetes.io/name: drm-exporter
- ports:
- - name: metrics
- port: 8081
- protocol: TCP
- targetPort: metrics
@@ (root level) @@
# apps/v1/DaemonSet/observability/drm-exporter
! - one document removed:
- apiVersion: apps/v1
- kind: DaemonSet
- metadata:
- name: drm-exporter
- namespace: observability
- labels:
- app.kubernetes.io/component: exporter
- app.kubernetes.io/instance: drm-exporter
- app.kubernetes.io/managed-by: Helm
- app.kubernetes.io/name: drm-exporter
- helm.toolkit.fluxcd.io/name: drm-exporter
- helm.toolkit.fluxcd.io/namespace: observability
- spec:
- selector:
- matchLabels:
- app.kubernetes.io/instance: drm-exporter
- app.kubernetes.io/name: drm-exporter
- template:
- metadata:
- labels:
- app.kubernetes.io/instance: drm-exporter
- app.kubernetes.io/name: drm-exporter
- spec:
- automountServiceAccountToken: false
- containers:
- - name: drm-exporter
- image: "ghcr.io/home-operations/drm-exporter@sha256:7229811068328ff3d9f9ac70b17319bb036e5e13dff6d743d5c5e3705d961488"
- imagePullPolicy: IfNotPresent
- env:
- - name: DRM_EXPORTER_ADDRESS
- value: "0.0.0.0"
- - name: DRM_EXPORTER_PORT
- value: "8081"
- - name: DRM_EXPORTER_INTERVAL_SECONDS
- value: "5"
- - name: RUST_LOG
- value: info
- livenessProbe:
- httpGet:
- path: /healthz
- port: metrics
- initialDelaySeconds: 5
- periodSeconds: 20
- ports:
- - name: metrics
- containerPort: 8081
- protocol: TCP
- readinessProbe:
- httpGet:
- path: /readyz
- port: metrics
- initialDelaySeconds: 2
- periodSeconds: 10
- resources:
- limits:
- memory: 128Mi
- squat.ai/dri: 1
- requests:
- cpu: 10m
- memory: 32Mi
- squat.ai/dri: 1
- volumeMounts:
- - name: sys
- mountPath: /sys
- readOnly: true
- securityContext:
- allowPrivilegeEscalation: false
- capabilities:
- add:
- - PERFMON
- - SYS_RAWIO
- drop:
- - ALL
- privileged: false
- readOnlyRootFilesystem: true
- nodeSelector:
- amd.com/gpu: "true"
- securityContext:
- runAsGroup: 0
- runAsNonRoot: false
- runAsUser: 0
- seccompProfile:
- type: Unconfined
- supplementalGroups:
- - 44
- - 226
- serviceAccountName: drm-exporter
- terminationGracePeriodSeconds: 30
- tolerations:
- - key: amd.com/gpu
- effect: NoSchedule
- operator: Exists
- volumes:
- - name: sys
- hostPath:
- type: Directory
- path: /sys
- updateStrategy:
- type: RollingUpdate
@@ (root level) @@
# grafana.integreatly.org/v1beta1/GrafanaDashboard/observability/drm-exporter-dashboard
! - one document removed:
- apiVersion: grafana.integreatly.org/v1beta1
- kind: GrafanaDashboard
- metadata:
- name: drm-exporter-dashboard
- namespace: observability
- labels:
- app.kubernetes.io/component: exporter
- app.kubernetes.io/instance: drm-exporter
- app.kubernetes.io/managed-by: Helm
- app.kubernetes.io/name: drm-exporter
- helm.toolkit.fluxcd.io/name: drm-exporter
- helm.toolkit.fluxcd.io/namespace: observability
- spec:
- allowCrossNamespaceImport: true
- configMapRef:
- name: drm-exporter-dashboard
- key: drm-exporter.json
- folder: hardware
- instanceSelector:
- matchLabels:
- grafana.internal/instance: grafana
- resyncPeriod: 10m
@@ (root level) @@
# monitoring.coreos.com/v1/ServiceMonitor/observability/drm-exporter
! - one document removed:
- apiVersion: monitoring.coreos.com/v1
- kind: ServiceMonitor
- metadata:
- name: drm-exporter
- namespace: observability
- labels:
- app.kubernetes.io/component: exporter
- app.kubernetes.io/instance: drm-exporter
- app.kubernetes.io/managed-by: Helm
- app.kubernetes.io/name: drm-exporter
- helm.toolkit.fluxcd.io/name: drm-exporter
- helm.toolkit.fluxcd.io/namespace: observability
- spec:
- endpoints:
- - interval: 30s
- path: /metrics
- port: metrics
- relabelings:
- - sourceLabels:
- - __meta_kubernetes_pod_node_name
- targetLabel: node
- - action: replace
- regex: "(.*)"
- replacement: $1
- sourceLabels:
- - __meta_kubernetes_pod_node_name
- targetLabel: kubernetes_node
- - action: replace
- regex: "(.*)"
- replacement: $1
- sourceLabels:
- - __meta_kubernetes_pod_node_name
- targetLabel: nodename
- scrapeTimeout: 10s
- selector:
- matchLabels:
- app.kubernetes.io/instance: drm-exporter
- app.kubernetes.io/name: drm-exporter
@@ (root level) @@
# v1/ServiceAccount/kube-system/drm-exporter
! + one document added:
+ apiVersion: v1
+ kind: ServiceAccount
+ metadata:
+ name: drm-exporter
+ namespace: kube-system
+ labels:
+ app.kubernetes.io/component: exporter
+ app.kubernetes.io/instance: drm-exporter
+ app.kubernetes.io/managed-by: Helm
+ app.kubernetes.io/name: drm-exporter
+ helm.toolkit.fluxcd.io/name: drm-exporter
+ helm.toolkit.fluxcd.io/namespace: kube-system
+ automountServiceAccountToken: false
@@ (root level) @@
# v1/ConfigMap/kube-system/drm-exporter-dashboard
! + one document added:
+ apiVersion: v1
+ kind: ConfigMap
+ metadata:
+ name: drm-exporter-dashboard
+ namespace: kube-system
+ labels:
+ app.kubernetes.io/component: exporter
+ app.kubernetes.io/instance: drm-exporter
+ app.kubernetes.io/managed-by: Helm
+ app.kubernetes.io/name: drm-exporter
+ helm.toolkit.fluxcd.io/name: drm-exporter
+ helm.toolkit.fluxcd.io/namespace: kube-system
+ data:
+ drm-exporter.json: |
+
+ {
+ "uid": "drm-exporter",
+ "title": "drm-exporter \u2014 GPU metrics",
+ "tags": [
+ "drm-exporter",
+ "gpu",
+ "intel",
+ "amd"
+ ],
+ "editable": true,
+ "schemaVersion": 39,
+ "timezone": "",
+ "refresh": "30s",
+ "time": {
+ "from": "now-6h",
+ "to": "now"
+ },
+ "timepicker": {},
+ "annotations": {
+ "list": [
+ {
+ "builtIn": 1,
+ "datasource": {
+ "type": "grafana",
+ "uid": "-- Grafana --"
+ },
+ "enable": true,
+ "hide": true,
+ "name": "Annotations & Alerts",
+ "type": "dashboard"
+ }
+ ]
+ },
+ "templating": {
+ "list": [
+ {
+ "name": "datasource",
+ "type": "datasource",
+ "query": "prometheus",
+ "label": "Data source",
+ "current": {},
+ "hide": 0,
+ "refresh": 1,
+ "regex": ""
+ },
+ {
+ "name": "node",
+ "type": "query",
+ "label": "Node",
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${datasource}"
+ },
+ "query": {
+ "qryType": 1,
+ "query": "label_values(drm_info, node)",
+ "refId": "PrometheusVariableQueryEditor-VariableQuery"
+ },
+ "current": {},
+ "refresh": 2,
+ "includeAll": true,
+ "multi": true,
+ "allValue": ".*",
+ "sort": 1
+ },
+ {
+ "name": "device",
+ "type": "query",
+ "label": "Device",
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${datasource}"
+ },
+ "query": {
+ "qryType": 1,
+ "query": "label_values(drm_info{node=~\"$node\"}, device)",
+ "refId": "PrometheusVariableQueryEditor-VariableQuery"
+ },
+ "current": {},
+ "refresh": 2,
+ "includeAll": true,
+ "multi": true,
+ "allValue": ".*",
+ "sort": 1
+ }
+ ]
+ },
+ "panels": [
+ {
+ "id": 1,
+ "type": "stat",
+ "title": "GPUs",
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${datasource}"
+ },
+ "gridPos": {
+ "h": 4,
+ "w": 4,
+ "x": 0,
+ "y": 0
+ },
+ "fieldConfig": {
+ "defaults": {
+ "unit": "none",
+ "color": {
+ "mode": "thresholds"
+ },
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ }
+ ]
+ }
+ },
+ "overrides": []
+ },
+ "options": {
+ "reduceOptions": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "fields": "",
+ "values": false
+ },
+ "colorMode": "value",
+ "graphMode": "area",
+ "textMode": "auto",
+ "justifyMode": "auto"
+ },
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${datasource}"
+ },
+ "editorMode": "code",
+ "expr": "count(group by (node, device) (drm_info{node=~\"$node\", device=~\"$device\"}))",
+ "legendFormat": "",
+ "range": true,
+ "instant": false,
+ "format": "time_series",
+ "refId": "A"
+ }
+ ]
+ },
+ {
+ "id": 2,
+ "type": "stat",
+ "title": "Peak engine busy",
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${datasource}"
+ },
+ "gridPos": {
+ "h": 4,
+ "w": 4,
+ "x": 4,
+ "y": 0
+ },
+ "fieldConfig": {
+ "defaults": {
+ "unit": "percentunit",
+ "color": {
+ "mode": "thresholds"
+ },
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ }
+ ]
+ },
+ "min": 0,
+ "max": 1
+ },
+ "overrides": []
+ },
+ "options": {
+ "reduceOptions": {
+ "calcs": [
+ "max"
+ ],
+ "fields": "",
+ "values": false
+ },
+ "colorMode": "value",
+ "graphMode": "area",
+ "textMode": "auto",
+ "justifyMode": "auto"
+ },
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${datasource}"
+ },
+ "editorMode": "code",
+ "expr": "max(drm_engine_utilization_ratio{node=~\"$node\", device=~\"$device\"})",
+ "legendFormat": "",
+ "range": true,
+ "instant": false,
+ "format": "time_series",
+ "refId": "A"
+ }
+ ]
+ },
+ {
+ "id": 3,
+ "type": "table",
+ "title": "GPU inventory",
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${datasource}"
+ },
+ "gridPos": {
+ "h": 4,
+ "w": 16,
+ "x": 8,
+ "y": 0
+ },
+ "fieldConfig": {
+ "defaults": {},
+ "overrides": []
+ },
+ "options": {
+ "showHeader": true
+ },
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${datasource}"
+ },
+ "editorMode": "code",
+ "expr": "drm_info{node=~\"$node\", device=~\"$device\"}",
+ "legendFormat": "",
+ "range": false,
+ "instant": true,
+ "format": "table",
+ "refId": "A"
+ }
+ ],
+ "transformations": [
+ {
+ "id": "filterFieldsByName",
+ "options": {
+ "include": {
+ "names": [
+ "node",
+ "device",
+ "vendor",
+ "model",
+ "driver",
+ "type",
+ "revision",
+ "pci_id",
+ "dev_node"
+ ]
+ }
+ }
+ }
+ ]
+ },
+ {
+ "id": 4,
+ "type": "timeseries",
+ "title": "Engine utilization",
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${datasource}"
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 24,
+ "x": 0,
+ "y": 4
+ },
+ "fieldConfig": {
+ "defaults": {
+ "unit": "percentunit",
+ "custom": {
+ "drawStyle": "line",
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "fillOpacity": 10,
+ "showPoints": "never",
+ "spanNulls": false,
+ "axisPlacement": "auto"
+ },
+ "color": {
+ "mode": "palette-classic"
+ },
+ "min": 0,
+ "max": 1
+ },
+ "overrides": []
+ },
+ "options": {
+ "legend": {
+ "displayMode": "table",
+ "placement": "bottom",
+ "calcs": [
+ "lastNotNull",
+ "max"
+ ]
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "desc"
+ }
+ },
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${datasource}"
+ },
+ "editorMode": "code",
+ "expr": "drm_engine_utilization_ratio{node=~\"$node\", device=~\"$device\"}",
+ "legendFormat": "{{node}} \u00b7 {{device}} \u00b7 {{engine}}",
+ "range": true,
+ "instant": false,
+ "format": "time_series",
+ "refId": "A"
+ }
+ ]
+ },
+ {
+ "id": 5,
+ "type": "timeseries",
+ "title": "Memory used",
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${datasource}"
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 0,
+ "y": 12
+ },
+ "fieldConfig": {
+ "defaults": {
+ "unit": "bytes",
+ "custom": {
+ "drawStyle": "line",
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "fillOpacity": 10,
+ "showPoints": "never",
+ "spanNulls": false,
+ "axisPlacement": "auto"
+ },
+ "color": {
+ "mode": "palette-classic"
+ }
+ },
+ "overrides": []
+ },
+ "options": {
+ "legend": {
+ "displayMode": "table",
+ "placement": "bottom",
+ "calcs": [
+ "lastNotNull",
+ "max"
+ ]
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "desc"
+ }
+ },
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${datasource}"
+ },
+ "editorMode": "code",
+ "expr": "drm_memory_used_bytes{node=~\"$node\", device=~\"$device\"}",
+ "legendFormat": "{{node}} \u00b7 {{device}} \u00b7 {{pool}}",
+ "range": true,
+ "instant": false,
+ "format": "time_series",
+ "refId": "A"
+ }
+ ]
+ },
+ {
+ "id": 6,
+ "type": "timeseries",
+ "title": "Memory utilization",
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${datasource}"
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 12,
+ "y": 12
+ },
+ "fieldConfig": {
+ "defaults": {
+ "unit": "percentunit",
+ "custom": {
+ "drawStyle": "line",
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "fillOpacity": 10,
+ "showPoints": "never",
+ "spanNulls": false,
+ "axisPlacement": "auto"
+ },
+ "color": {
+ "mode": "palette-classic"
+ },
+ "min": 0,
+ "max": 1
+ },
+ "overrides": []
+ },
+ "options": {
+ "legend": {
+ "displayMode": "table",
+ "placement": "bottom",
+ "calcs": [
+ "lastNotNull",
+ "max"
+ ]
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "desc"
+ }
+ },
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${datasource}"
+ },
+ "editorMode": "code",
+ "expr": "drm_memory_used_bytes{node=~\"$node\", device=~\"$device\"} / drm_memory_total_bytes{node=~\"$node\", device=~\"$device\"}",
+ "legendFormat": "{{node}} \u00b7 {{device}} \u00b7 {{pool}}",
+ "range": true,
+ "instant": false,
+ "format": "time_series",
+ "refId": "A"
+ }
+ ]
+ },
+ {
+ "id": 7,
+ "type": "timeseries",
+ "title": "Clock frequency",
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${datasource}"
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 0,
+ "y": 20
+ },
+ "fieldConfig": {
+ "defaults": {
+ "unit": "hertz",
+ "custom": {
+ "drawStyle": "line",
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "fillOpacity": 10,
+ "showPoints": "never",
+ "spanNulls": false,
+ "axisPlacement": "auto"
+ },
+ "color": {
+ "mode": "palette-classic"
+ }
+ },
+ "overrides": []
+ },
+ "options": {
+ "legend": {
+ "displayMode": "table",
+ "placement": "bottom",
+ "calcs": [
+ "lastNotNull",
+ "max"
+ ]
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "desc"
+ }
+ },
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${datasource}"
+ },
+ "editorMode": "code",
+ "expr": "drm_frequency_hertz{node=~\"$node\", device=~\"$device\"}",
+ "legendFormat": "{{node}} \u00b7 {{device}} \u00b7 {{domain}} \u00b7 {{kind}}",
+ "range": true,
+ "instant": false,
+ "format": "time_series",
+ "refId": "A"
+ }
+ ]
+ },
+ {
+ "id": 8,
+ "type": "timeseries",
+ "title": "Power draw",
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${datasource}"
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 12,
+ "y": 20
+ },
+ "fieldConfig": {
+ "defaults": {
+ "unit": "watt",
+ "custom": {
+ "drawStyle": "line",
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "fillOpacity": 10,
+ "showPoints": "never",
+ "spanNulls": false,
+ "axisPlacement": "auto"
+ },
+ "color": {
+ "mode": "palette-classic"
+ }
+ },
+ "overrides": []
+ },
+ "options": {
+ "legend": {
+ "displayMode": "table",
+ "placement": "bottom",
+ "calcs": [
+ "lastNotNull",
+ "max"
+ ]
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "desc"
+ }
+ },
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${datasource}"
+ },
+ "editorMode": "code",
+ "expr": "drm_power_watts{node=~\"$node\", device=~\"$device\"}",
+ "legendFormat": "{{node}} \u00b7 {{device}} \u00b7 {{domain}}",
+ "range": true,
+ "instant": false,
+ "format": "time_series",
+ "refId": "A"
+ }
+ ]
+ },
+ {
+ "id": 9,
+ "type": "timeseries",
+ "title": "Temperature",
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${datasource}"
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 0,
+ "y": 28
+ },
+ "fieldConfig": {
+ "defaults": {
+ "unit": "celsius",
+ "custom": {
+ "drawStyle": "line",
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "fillOpacity": 10,
+ "showPoints": "never",
+ "spanNulls": false,
+ "axisPlacement": "auto",
+ "thresholdsStyle": {
+ "mode": "dashed"
+ }
+ },
+ "color": {
+ "mode": "palette-classic"
+ },
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ },
+ {
+ "color": "orange",
+ "value": 80
+ },
+ {
+ "color": "red",
+ "value": 95
+ }
+ ]
+ }
+ },
+ "overrides": []
+ },
+ "options": {
+ "legend": {
+ "displayMode": "table",
+ "placement": "bottom",
+ "calcs": [
+ "lastNotNull",
+ "max"
+ ]
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "desc"
+ }
+ },
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${datasource}"
+ },
+ "editorMode": "code",
+ "expr": "drm_temperature_celsius{node=~\"$node\", device=~\"$device\"}",
+ "legendFormat": "{{node}} \u00b7 {{device}} \u00b7 {{sensor}}",
+ "range": true,
+ "instant": false,
+ "format": "time_series",
+ "refId": "A"
+ }
+ ]
+ },
+ {
+ "id": 10,
+ "type": "timeseries",
+ "title": "Fan speed",
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${datasource}"
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 12,
+ "y": 28
+ },
+ "fieldConfig": {
+ "defaults": {
+ "unit": "rotrpm",
+ "custom": {
+ "drawStyle": "line",
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "fillOpacity": 10,
+ "showPoints": "never",
+ "spanNulls": false,
+ "axisPlacement": "auto"
+ },
+ "color": {
+ "mode": "palette-classic"
+ }
+ },
+ "overrides": []
+ },
+ "options": {
+ "legend": {
+ "displayMode": "table",
+ "placement": "bottom",
+ "calcs": [
+ "lastNotNull",
+ "max"
+ ]
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "desc"
+ }
+ },
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${datasource}"
+ },
+ "editorMode": "code",
+ "expr": "drm_fan_speed_rpm{node=~\"$node\", device=~\"$device\"}",
+ "legendFormat": "{{node}} \u00b7 {{device}} \u00b7 {{fan}}",
+ "range": true,
+ "instant": false,
+ "format": "time_series",
+ "refId": "A"
+ }
+ ]
+ }
+ ]
+ }
@@ (root level) @@
# v1/Service/kube-system/drm-exporter
! + one document added:
+ apiVersion: v1
+ kind: Service
+ metadata:
+ name: drm-exporter
+ namespace: kube-system
+ labels:
+ app.kubernetes.io/component: exporter
+ app.kubernetes.io/instance: drm-exporter
+ app.kubernetes.io/managed-by: Helm
+ app.kubernetes.io/name: drm-exporter
+ helm.toolkit.fluxcd.io/name: drm-exporter
+ helm.toolkit.fluxcd.io/namespace: kube-system
+ spec:
+ type: ClusterIP
+ selector:
+ app.kubernetes.io/instance: drm-exporter
+ app.kubernetes.io/name: drm-exporter
+ ports:
+ - name: metrics
+ port: 8081
+ protocol: TCP
+ targetPort: metrics
@@ (root level) @@
# apps/v1/DaemonSet/kube-system/drm-exporter
! + one document added:
+ apiVersion: apps/v1
+ kind: DaemonSet
+ metadata:
+ name: drm-exporter
+ namespace: kube-system
+ labels:
+ app.kubernetes.io/component: exporter
+ app.kubernetes.io/instance: drm-exporter
+ app.kubernetes.io/managed-by: Helm
+ app.kubernetes.io/name: drm-exporter
+ helm.toolkit.fluxcd.io/name: drm-exporter
+ helm.toolkit.fluxcd.io/namespace: kube-system
+ spec:
+ selector:
+ matchLabels:
+ app.kubernetes.io/instance: drm-exporter
+ app.kubernetes.io/name: drm-exporter
+ template:
+ metadata:
+ labels:
+ app.kubernetes.io/instance: drm-exporter
+ app.kubernetes.io/name: drm-exporter
+ spec:
+ affinity:
+ nodeAffinity:
+ requiredDuringSchedulingIgnoredDuringExecution:
+ nodeSelectorTerms:
+ - matchExpressions:
+ - key: amd.com/gpu
+ operator: In
+ values:
+ - "true"
+ - matchExpressions:
+ - key: amd.com/igpu
+ operator: In
+ values:
+ - "true"
+ automountServiceAccountToken: false
+ containers:
+ - name: drm-exporter
+ image: "ghcr.io/home-operations/drm-exporter@sha256:7229811068328ff3d9f9ac70b17319bb036e5e13dff6d743d5c5e3705d961488"
+ imagePullPolicy: IfNotPresent
+ env:
+ - name: DRM_EXPORTER_ADDRESS
+ value: "0.0.0.0"
+ - name: DRM_EXPORTER_PORT
+ value: "8081"
+ - name: DRM_EXPORTER_INTERVAL_SECONDS
+ value: "5"
+ - name: RUST_LOG
+ value: info
+ livenessProbe:
+ httpGet:
+ path: /healthz
+ port: metrics
+ initialDelaySeconds: 5
+ periodSeconds: 20
+ ports:
+ - name: metrics
+ containerPort: 8081
+ protocol: TCP
+ readinessProbe:
+ httpGet:
+ path: /readyz
+ port: metrics
+ initialDelaySeconds: 2
+ periodSeconds: 10
+ resources:
+ claims:
+ - name: gpu
+ limits:
+ memory: 128Mi
+ requests:
+ cpu: 10m
+ volumeMounts:
+ - name: sys
+ mountPath: /sys
+ readOnly: true
+ securityContext:
+ allowPrivilegeEscalation: false
+ capabilities:
+ add:
+ - PERFMON
+ drop:
+ - ALL
+ privileged: false
+ readOnlyRootFilesystem: true
+ resourceClaims:
+ - name: gpu
+ resourceClaimTemplateName: drm-exporter-gpu
+ securityContext:
+ runAsGroup: 0
+ runAsNonRoot: false
+ runAsUser: 0
+ seccompProfile:
+ type: RuntimeDefault
+ serviceAccountName: drm-exporter
+ terminationGracePeriodSeconds: 30
+ volumes:
+ - name: sys
+ hostPath:
+ type: Directory
+ path: /sys
+ updateStrategy:
+ type: RollingUpdate
@@ (root level) @@
# grafana.integreatly.org/v1beta1/GrafanaDashboard/kube-system/drm-exporter-dashboard
! + one document added:
+ apiVersion: grafana.integreatly.org/v1beta1
+ kind: GrafanaDashboard
+ metadata:
+ name: drm-exporter-dashboard
+ namespace: kube-system
+ labels:
+ app.kubernetes.io/component: exporter
+ app.kubernetes.io/instance: drm-exporter
+ app.kubernetes.io/managed-by: Helm
+ app.kubernetes.io/name: drm-exporter
+ helm.toolkit.fluxcd.io/name: drm-exporter
+ helm.toolkit.fluxcd.io/namespace: kube-system
+ spec:
+ allowCrossNamespaceImport: true
+ configMapRef:
+ name: drm-exporter-dashboard
+ key: drm-exporter.json
+ folder: hardware
+ instanceSelector:
+ matchLabels:
+ grafana.internal/instance: grafana
+ resyncPeriod: 10m
@@ (root level) @@
# resource.k8s.io/v1/ResourceClaimTemplate/kube-system/drm-exporter-gpu
! + one document added:
+ apiVersion: resource.k8s.io/v1
+ kind: ResourceClaimTemplate
+ metadata:
+ name: drm-exporter-gpu
+ namespace: kube-system
+ labels:
+ app.kubernetes.io/component: exporter
+ app.kubernetes.io/instance: drm-exporter
+ app.kubernetes.io/managed-by: Helm
+ app.kubernetes.io/name: drm-exporter
+ helm.toolkit.fluxcd.io/name: drm-exporter
+ helm.toolkit.fluxcd.io/namespace: kube-system
+ spec:
+ spec:
+ devices:
+ requests:
+ - name: drm-exporter
+ exactly:
+ adminAccess: true
+ allocationMode: All
+ deviceClassName: gpu.amd.com
@@ (root level) @@
# monitoring.coreos.com/v1/ServiceMonitor/kube-system/drm-exporter
! + one document added:
+ apiVersion: monitoring.coreos.com/v1
+ kind: ServiceMonitor
+ metadata:
+ name: drm-exporter
+ namespace: kube-system
+ labels:
+ app.kubernetes.io/component: exporter
+ app.kubernetes.io/instance: drm-exporter
+ app.kubernetes.io/managed-by: Helm
+ app.kubernetes.io/name: drm-exporter
+ helm.toolkit.fluxcd.io/name: drm-exporter
+ helm.toolkit.fluxcd.io/namespace: kube-system
+ spec:
+ endpoints:
+ - interval: 30s
+ path: /metrics
+ port: metrics
+ relabelings:
+ - sourceLabels:
+ - __meta_kubernetes_pod_node_name
+ targetLabel: node
+ - action: replace
+ regex: "(.*)"
+ replacement: $1
+ sourceLabels:
+ - __meta_kubernetes_pod_node_name
+ targetLabel: kubernetes_node
+ - action: replace
+ regex: "(.*)"
+ $$ This message was truncated. Download full message |
@@ metadata.labels @@
# v1/Namespace/kube-system
! + one map entry added:
+ resource.kubernetes.io/admin-access: "true"
@@ spec @@
# kustomize.toolkit.fluxcd.io/v1/Kustomization/media/fileflows
! + one map entry added:
+ dependsOn:
+ - name: k8s-gpu-dra-driver
+ namespace: kube-system
@@ spec @@
# kustomize.toolkit.fluxcd.io/v1/Kustomization/media/jellyfin
! + one map entry added:
+ dependsOn:
+ - name: k8s-gpu-dra-driver
+ namespace: kube-system
@@ spec.values.controllers @@
# helm.toolkit.fluxcd.io/v2/HelmRelease/kube-system/generic-device-plugin
! - one map entry removed:
- generic-device-plugin:
- type: daemonset
- pod:
- nodeSelector:
- amd.com/gpu: "true"
- containers:
- app:
- resources:
- limits:
- memory: 64Mi
- requests:
- cpu: 10m
- memory: 16Mi
- args:
- - "--domain=squat.ai"
- - "--device"
- - |
- name: dri
- groups:
- - count: 4
- paths:
- - path: /dev/dri
- - path: /dev/kfd
-
- image:
- repository: ghcr.io/squat/generic-device-plugin
- tag: "latest@sha256:dc192e164c69b03f156765793a1be62ca437709ae477b27ca7d8f3dcf5021576"
- securityContext:
- privileged: true
@@ spec.values.controllers.fileflows.containers.app.resources @@
# helm.toolkit.fluxcd.io/v2/HelmRelease/media/fileflows
! + one map entry added:
+ claims:
+ - name: gpu
@@ spec.values.controllers.fileflows.containers.app.resources.limits @@
# helm.toolkit.fluxcd.io/v2/HelmRelease/media/fileflows
! - one map entry removed:
- squat.ai/dri: 1
@@ spec.values.controllers.fileflows.containers.app.resources.requests @@
# helm.toolkit.fluxcd.io/v2/HelmRelease/media/fileflows
! - one map entry removed:
- squat.ai/dri: 1
@@ spec.values.defaultPodOptions @@
# helm.toolkit.fluxcd.io/v2/HelmRelease/media/fileflows
! + one map entry added:
+ resourceClaims:
+ - name: gpu
+ resourceClaimName: shared-dgpu
@@ spec.values.controllers.jellyfin.containers.app.resources @@
# helm.toolkit.fluxcd.io/v2/HelmRelease/media/jellyfin
! + one map entry added:
+ claims:
+ - name: gpu
@@ spec.values.controllers.jellyfin.containers.app.resources.limits @@
# helm.toolkit.fluxcd.io/v2/HelmRelease/media/jellyfin
! - one map entry removed:
- squat.ai/dri: 1
@@ spec.values.controllers.jellyfin.containers.app.resources.requests @@
# helm.toolkit.fluxcd.io/v2/HelmRelease/media/jellyfin
! - one map entry removed:
- squat.ai/dri: 1
@@ spec.values.defaultPodOptions @@
# helm.toolkit.fluxcd.io/v2/HelmRelease/media/jellyfin
! + one map entry added:
+ resourceClaims:
+ - name: gpu
+ resourceClaimName: shared-dgpu
@@ (root level) @@
# kustomize.toolkit.fluxcd.io/v1/Kustomization/observability/drm-exporter
! - one document removed:
- apiVersion: kustomize.toolkit.fluxcd.io/v1
- kind: Kustomization
- metadata:
- name: drm-exporter
- namespace: observability
- labels:
- kustomize.toolkit.fluxcd.io/name: cluster-apps
- kustomize.toolkit.fluxcd.io/namespace: flux-system
- spec:
- decryption:
- provider: sops
- secretRef:
- name: sops-age
- deletionPolicy: WaitForTermination
- dependsOn:
- - name: generic-device-plugin
- namespace: kube-system
- - name: prometheus-operator-crds
- namespace: observability
- interval: 1h
- patches:
- - patch: |
- apiVersion: helm.toolkit.fluxcd.io/v2
- kind: HelmRelease
- metadata:
- name: _
- spec:
- install:
- crds: CreateReplace
- rollback:
- cleanupOnFail: true
- upgrade:
- cleanupOnFail: true
- crds: CreateReplace
- strategy:
- name: RemediateOnFailure
- remediation:
- remediateLastFailure: true
- retries: 2
- target:
- kind: HelmRelease
- group: helm.toolkit.fluxcd.io
- path: ./kubernetes/apps/observability/exporters/drm-exporter/app
- postBuild:
- substituteFrom:
- - name: cluster-settings
- kind: ConfigMap
- optional: false
- - name: cluster-secrets
- kind: Secret
- optional: false
- prune: true
- retryInterval: 2m
- sourceRef:
- name: flux-system
- kind: GitRepository
- namespace: flux-system
- targetNamespace: observability
- timeout: 5m
- wait: false
@@ (root level) @@
# helm.toolkit.fluxcd.io/v2/HelmRelease/observability/drm-exporter
! - one document removed:
- apiVersion: helm.toolkit.fluxcd.io/v2
- kind: HelmRelease
- metadata:
- name: drm-exporter
- namespace: observability
- labels:
- kustomize.toolkit.fluxcd.io/name: drm-exporter
- kustomize.toolkit.fluxcd.io/namespace: observability
- spec:
- chartRef:
- name: drm-exporter
- kind: OCIRepository
- install:
- crds: CreateReplace
- interval: 1h
- rollback:
- cleanupOnFail: true
- upgrade:
- cleanupOnFail: true
- crds: CreateReplace
- remediation:
- remediateLastFailure: true
- retries: 2
- strategy:
- name: RemediateOnFailure
- values:
- resources:
- limits:
- memory: 128Mi
- squat.ai/dri: 1
- requests:
- cpu: 10m
- memory: 32Mi
- squat.ai/dri: 1
- fullnameOverride: drm-exporter
- nodeSelector:
- amd.com/gpu: "true"
- podSecurityContext:
- seccompProfile:
- type: Unconfined
- supplementalGroups:
- - 44
- - 226
- tolerations:
- - key: amd.com/gpu
- effect: NoSchedule
- operator: Exists
- monitoring:
- dashboards:
- enabled: true
- grafanaOperator:
- enabled: true
- folder: hardware
- matchLabels:
- grafana.internal/instance: grafana
- serviceMonitor:
- enabled: true
- relabelings:
- - action: replace
- regex: "(.*)"
- replacement: $1
- sourceLabels:
- - __meta_kubernetes_pod_node_name
- targetLabel: kubernetes_node
- - action: replace
- regex: "(.*)"
- replacement: $1
- sourceLabels:
- - __meta_kubernetes_pod_node_name
- targetLabel: nodename
@@ (root level) @@
# source.toolkit.fluxcd.io/v1/OCIRepository/observability/drm-exporter
! - one document removed:
- apiVersion: source.toolkit.fluxcd.io/v1
- kind: OCIRepository
- metadata:
- name: drm-exporter
- namespace: observability
- labels:
- kustomize.toolkit.fluxcd.io/name: drm-exporter
- kustomize.toolkit.fluxcd.io/namespace: observability
- spec:
- url: "oci://ghcr.io/home-operations/charts/drm-exporter"
- interval: 1h
- layerSelector:
- mediaType: application/vnd.cncf.helm.chart.content.v1.tar+gzip
- operation: copy
- ref:
- tag: "0.3.3"
@@ (root level) @@
# monitoring.coreos.com/v1/PrometheusRule/observability/drm-exporter-rules
! - one document removed:
- apiVersion: monitoring.coreos.com/v1
- kind: PrometheusRule
- metadata:
- name: drm-exporter-rules
- namespace: observability
- labels:
- kustomize.toolkit.fluxcd.io/name: drm-exporter
- kustomize.toolkit.fluxcd.io/namespace: observability
- spec:
- groups:
- - name: drm-exporter.rules
- rules:
- - alert: GpuFanStalled
- annotations:
- summary: "{{ $labels.kubernetes_node }} GPU fan has stalled under load — card has no thermal margin without it, power off the GPU workload"
- expr: |
- max by (kubernetes_node) (drm_fan_speed_rpm) < 1000
- and
- max by (kubernetes_node) (drm_power_watts) > 100
- for: 5m
- labels:
- severity: critical
@@ (root level) @@
# kustomize.toolkit.fluxcd.io/v1/Kustomization/kube-system/drm-exporter
! + one document added:
+ apiVersion: kustomize.toolkit.fluxcd.io/v1
+ kind: Kustomization
+ metadata:
+ name: drm-exporter
+ namespace: kube-system
+ labels:
+ kustomize.toolkit.fluxcd.io/name: cluster-apps
+ kustomize.toolkit.fluxcd.io/namespace: flux-system
+ spec:
+ decryption:
+ provider: sops
+ secretRef:
+ name: sops-age
+ deletionPolicy: WaitForTermination
+ dependsOn:
+ - name: k8s-gpu-dra-driver
+ namespace: kube-system
+ - name: prometheus-operator-crds
+ namespace: observability
+ interval: 1h
+ patches:
+ - patch: |
+ apiVersion: helm.toolkit.fluxcd.io/v2
+ kind: HelmRelease
+ metadata:
+ name: _
+ spec:
+ install:
+ crds: CreateReplace
+ rollback:
+ cleanupOnFail: true
+ upgrade:
+ cleanupOnFail: true
+ crds: CreateReplace
+ strategy:
+ name: RemediateOnFailure
+ remediation:
+ remediateLastFailure: true
+ retries: 2
+ target:
+ kind: HelmRelease
+ group: helm.toolkit.fluxcd.io
+ path: ./kubernetes/apps/kube-system/drm-exporter/app
+ postBuild:
+ substituteFrom:
+ - name: cluster-settings
+ kind: ConfigMap
+ optional: false
+ - name: cluster-secrets
+ kind: Secret
+ optional: false
+ prune: true
+ retryInterval: 2m
+ sourceRef:
+ name: flux-system
+ kind: GitRepository
+ namespace: flux-system
+ targetNamespace: kube-system
+ timeout: 5m
+ wait: false
@@ (root level) @@
# kustomize.toolkit.fluxcd.io/v1/Kustomization/kube-system/k8s-gpu-dra-driver
! + one document added:
+ apiVersion: kustomize.toolkit.fluxcd.io/v1
+ kind: Kustomization
+ metadata:
+ name: k8s-gpu-dra-driver
+ namespace: kube-system
+ labels:
+ kustomize.toolkit.fluxcd.io/name: cluster-apps
+ kustomize.toolkit.fluxcd.io/namespace: flux-system
+ spec:
+ decryption:
+ provider: sops
+ secretRef:
+ name: sops-age
+ deletionPolicy: WaitForTermination
+ interval: 1h
+ patches:
+ - patch: |
+ apiVersion: helm.toolkit.fluxcd.io/v2
+ kind: HelmRelease
+ metadata:
+ name: _
+ spec:
+ install:
+ crds: CreateReplace
+ rollback:
+ cleanupOnFail: true
+ upgrade:
+ cleanupOnFail: true
+ crds: CreateReplace
+ strategy:
+ name: RemediateOnFailure
+ remediation:
+ remediateLastFailure: true
+ retries: 2
+ target:
+ kind: HelmRelease
+ group: helm.toolkit.fluxcd.io
+ path: ./kubernetes/apps/kube-system/k8s-gpu-dra-driver/app
+ postBuild:
+ substituteFrom:
+ - name: cluster-settings
+ kind: ConfigMap
+ optional: false
+ - name: cluster-secrets
+ kind: Secret
+ optional: false
+ prune: true
+ retryInterval: 2m
+ sourceRef:
+ name: flux-system
+ kind: GitRepository
+ namespace: flux-system
+ targetNamespace: kube-system
+ timeout: 5m
+ wait: true
@@ (root level) @@
# resource.k8s.io/v1/ResourceClaim/media/shared-dgpu
! + one document added:
+ apiVersion: resource.k8s.io/v1
+ kind: ResourceClaim
+ metadata:
+ name: shared-dgpu
+ namespace: media
+ labels:
+ kustomize.toolkit.fluxcd.io/name: cluster-apps
+ kustomize.toolkit.fluxcd.io/namespace: flux-system
+ spec:
+ devices:
+ requests:
+ - name: gpu
+ exactly:
+ allocationMode: ExactCount
+ count: 1
+ deviceClassName: gpu.amd.com
+ selectors:
+ - cel:
+ expression: "device.attributes[\"gpu.amd.com\"].type == \"amdgpu\""
@@ (root level) @@
# helm.toolkit.fluxcd.io/v2/HelmRelease/kube-system/drm-exporter
! + one document added:
+ apiVersion: helm.toolkit.fluxcd.io/v2
+ kind: HelmRelease
+ metadata:
+ name: drm-exporter
+ namespace: kube-system
+ labels:
+ kustomize.toolkit.fluxcd.io/name: drm-exporter
+ kustomize.toolkit.fluxcd.io/namespace: kube-system
+ spec:
+ chartRef:
+ name: drm-exporter
+ kind: OCIRepository
+ install:
+ crds: CreateReplace
+ interval: 1h
+ rollback:
+ cleanupOnFail: true
+ upgrade:
+ cleanupOnFail: true
+ crds: CreateReplace
+ remediation:
+ remediateLastFailure: true
+ retries: 2
+ strategy:
+ name: RemediateOnFailure
+ values:
+ resources:
+ limits:
+ memory: 128Mi
+ requests:
+ cpu: 10m
+ dra:
+ deviceClassName: gpu.amd.com
+ enabled: true
+ securityContext:
+ capabilities:
+ add:
+ - PERFMON
+ monitoring:
+ dashboards:
+ enabled: true
+ grafanaOperator:
+ enabled: true
+ folder: hardware
+ matchLabels:
+ grafana.internal/instance: grafana
+ serviceMonitor:
+ enabled: true
+ relabelings:
+ - action: replace
+ regex: "(.*)"
+ replacement: $1
+ sourceLabels:
+ - __meta_kubernetes_pod_node_name
+ targetLabel: kubernetes_node
+ - action: replace
+ regex: "(.*)"
+ replacement: $1
+ sourceLabels:
+ - __meta_kubernetes_pod_node_name
+ targetLabel: nodename
+ affinity:
+ nodeAffinity:
+ requiredDuringSchedulingIgnoredDuringExecution:
+ nodeSelectorTerms:
+ - matchExpressions:
+ - key: amd.com/gpu
+ operator: In
+ values:
+ - "true"
+ - matchExpressions:
+ - key: amd.com/igpu
+ operator: In
+ values:
+ - "true"
@@ (root level) @@
# source.toolkit.fluxcd.io/v1/OCIRepository/kube-system/drm-exporter
! + one document added:
+ apiVersion: source.toolkit.fluxcd.io/v1
+ kind: OCIRepository
+ metadata:
+ name: drm-exporter
+ namespace: kube-system
+ labels:
+ kustomize.toolkit.fluxcd.io/name: drm-exporter
+ kustomize.toolkit.fluxcd.io/namespace: kube-system
+ spec:
+ url: "oci://ghcr.io/home-operations/charts/drm-exporter"
+ interval: 1h
+ layerSelector:
+ mediaType: application/vnd.cncf.helm.chart.content.v1.tar+gzip
+ operation: copy
+ ref:
+ tag: "0.3.3"
@@ (root level) @@
# monitoring.coreos.com/v1/PrometheusRule/kube-system/drm-exporter-rules
! + one document added:
+ apiVersion: monitoring.coreos.com/v1
+ kind: PrometheusRule
+ metadata:
+ name: drm-exporter-rules
+ namespace: kube-system
+ labels:
+ kustomize.toolkit.fluxcd.io/name: drm-exporter
+ kustomize.toolkit.fluxcd.io/namespace: kube-system
+ spec:
+ groups:
+ - name: drm-exporter.rules
+ rules:
+ - alert: GpuFanStalled
+ annotations:
+ summary: "{{ $labels.kubernetes_node }} GPU fan has stalled under load — card has no thermal margin without it, power off the GPU workload"
+ expr: |
+ max by (kubernetes_node) (drm_fan_speed_rpm) < 1000
+ and
+ max by (kubernetes_node) (drm_power_watts) > 100
+ for: 5m
+ labels:
+ severity: critical
@@ (root level) @@
# helm.toolkit.fluxcd.io/v2/HelmRelease/kube-system/k8s-gpu-dra-driver
! + one document added:
+ apiVersion: helm.toolkit.fluxcd.io/v2
+ kind: HelmRelease
+ metadata:
+ name: k8s-gpu-dra-driver
+ namespace: kube-system
+ labels:
+ kustomize.toolkit.fluxcd.io/name: k8s-gpu-dra-driver
+ kustomize.toolkit.fluxcd.io/namespace: kube-system
+ spec:
+ chart:
+ spec:
+ version: v1.0.1
+ chart: k8s-gpu-dra-driver
+ sourceRef:
+ name: k8s-gpu-dra-driver
+ kind: HelmRepository
+ install:
+ crds: CreateReplace
+ interval: 1h
+ rollback:
+ cleanupOnFail: true
+ upgrade:
+ cleanupOnFail: true
+ crds: CreateReplace
+ remediation:
+ remediateLastFailure: true
+ retries: 2
+ strategy:
+ name: RemediateOnFailure
+ values:
+ kubeletPlugin:
+ affinity:
+ nodeAffinity:
+ requiredDuringSchedulingIgnoredDuringExecution:
+ nodeSelectorTerms:
+ - matchExpressions:
+ - key: amd.com/gpu
+ operator: In
+ values:
+ - "true"
@@ (root level) @@
# source.toolkit.fluxcd.io/v1/HelmRepository/kube-system/k8s-gpu-dra-driver
! + one document added:
+ apiVersion: source.toolkit.fluxcd.io/v1
+ kind: HelmRepository
+ metadata:
+ name: k8s-gpu-dra-driver
+ namespace: kube-system
+ labels:
+ kustomize.toolkit.fluxcd.io/name: k8s-gpu-dra-driver
+ kustomize.toolkit.fluxcd.io/namespace: kube-system
+ spec:
+ url: "https://rocm.github.io/k8s-gpu-dra-driver"
+ interval: 1h
|
Tanguille
marked this pull request as draft
June 23, 2026 17:47
Owner
Author
|
Parking as draft — blocked on GPU sharing, not the config itself. This migrates the dGPU (R9700, control-1) off Revisit when: DRA Consumable Capacity (KEP-5075) reaches beta and the AMD GPU DRA driver advertises |
Tanguille
marked this pull request as ready for review
June 27, 2026 13:07
Tanguille
force-pushed
the
refactor/migrate-to-dra-amdgpu
branch
2 times, most recently
from
June 27, 2026 13:26
3c33d46 to
f430e1a
Compare
AI Automated ReviewAnalysis engine: qwen-3.6-fast@http://litellm.ai.svc.cluster.local/v1 (openai) RecommendationApprove with verification pending. This PR correctly migrates the cluster from the The implementation aligns with the repository's GitOps structure, introducing appropriate Change-by-Change Findings1. DRA Driver Deployment (
|
Tanguille
force-pushed
the
refactor/migrate-to-dra-amdgpu
branch
from
June 28, 2026 21:29
f430e1a to
583d60f
Compare
Tanguille
marked this pull request as draft
June 29, 2026 20:29
![github-actions[bot]](https://avatars.githubusercontent.com/in/15368?s=60&v=4){kind=small}
Move the shared-dgpu ResourceClaim to the media namespace level (kubernetes/apps/media/shared-dgpu-claim.yaml) so that jellyfin no longer depends on fileflows for claim lifecycle. The claim is now reconciled by the root cluster-apps Kustomization alongside the namespace bootstrap, removing cross-app coupling between two independent workloads that happen to share the dGPU. Also clarify the kube-system/kustomization.yaml patch comment to explain why the target name is "not-used" (the common component placeholder, renamed to "kube-system" by the namespace transformer).
…nventions Main moved on since this PR was opened: gpu-builder was deleted entirely (chore(ci): remove gpu builder), and the iGPU embedding fleet grew a 3-way squat.ai/dri sharing scenario (qwen3-embedding + vmcp-embedding + qwen35-2b) that the AMD DRA driver can't serve (no timeslicing/consumable capacity). Scope this PR's DRA migration to the exclusive dGPU only: - Drop the dra-igpu component and revert qwen3-embedding.yaml to squat.ai/dri; keep generic-device-plugin's iGPU daemonset alive (trimmed the now-DRA-managed dGPU daemonset out of it). - k8s-gpu-dra-driver's kubeletPlugin now targets only amd.com/gpu nodes. - Fix llmkube-models' dependsOn back to generic-device-plugin (nothing there uses DRA) and drop the dead k8s-gpu-dra-driver dependsOn from the actions-runner-controller-runners Kustomization (gpu-builder is gone). - Adopt main's now-standard OCIRepository interval: 1h (was 15m/10m).
Conflict resolutions needing judgment: - ai/sglang/app/helmrelease.yaml: deletion accepted. main retired the sglang HelmRelease entirely (7f6e08f/7149d0483, cutover to the llmkube qwen36-27b InferenceService). This branch's only change to the file was the squat.ai/dri -> DRA claim swap, which dies with the app. - observability/exporters/drm-exporter -> kube-system/drm-exporter: relocation stands. main's only substantive change to the old path (dropping runAsUser/runAsGroup 0) is already satisfied by the DRA-mode HelmRelease, which has no podSecurityContext at all. Ported main's serviceMonitor relabelings forward: main's new GpuFanStalled PrometheusRule groups by kubernetes_node, a label only those relabelings produce. - media/jellyfin/app/helmrelease.yaml: union. Took main's cpu 100m / memory 1Gi request and the new container securityContext; kept this branch's DRA claim in place of squat.ai/dri. - ai/litellm/instance/kustomization.yaml and kube-system/node-problem-detector/ks.yaml: false rename pairings against the deleted sglang kustomization.yaml/ks.yaml. Took main verbatim. - kube-system/drm-exporter/app/prometheusrule.yaml: main's new alert, carried onto the relocated path by git's directory-rename detection. Kept. Known gap for follow-up: llmkube's qwen36-27b-sglang/-vllm Models still request squat.ai/dri on control-1, whose generic-device-plugin daemonset this branch removed in favour of DRA. They are the successor to the sglang dGPU tenant and need a DRA path before this branch is deployable.
Tanguille
force-pushed
the
refactor/migrate-to-dra-amdgpu
branch
from
August 8, 2026 13:48
3bf4745 to
adcca03
Compare
|
Important Review skippedDraft detected. Please check the settings in the CodeRabbit UI or the ⚙️ Run configurationConfiguration used: Organization UI Review profile: CHILL Plan: Pro Plus Run ID: You can disable this status message by setting the Use the checkbox below for a quick retry:
Thanks for using CodeRabbit! It's free for OSS, and your support helps us grow. If you like it, consider giving us a shout-out. Comment |
Contributor
 |
|
Overall Grade |
Security Reliability Complexity Hygiene |
Code Review Summary
| Analyzer | Status | Updated (UTC) | Details |
|---|---|---|---|
| JavaScript |  |
Aug 8, 2026 3:33p.m. | Review ↗ |
| Shell | |
Aug 8, 2026 3:33p.m. | Review ↗ |
Important
AI Review is run only on demand for your team. We're only showing results of static analysis review right now. To trigger AI Review, comment @deepsourcebot review on this thread.
…leted ghcr.io/buroa/helm/k8s-gpu-dra-driver is gone — the repo behind it (buroa/helm) was deleted upstream, returning 403 even anonymously. ROCm ships its own chart in-tree now (helm-charts-k8s/) instead of publishing an OCI package, so this tracks it via GitRepository + chart.spec instead of chartRef/OCIRepository. Pinned to v1.0.1 (the current driver release; Chart.yaml's own appVersion is stale at v0.1.0, hence the explicit image.tag override). Verified with 'helm template' against the real chart + our values — renders clean, node affinity applies as expected.
# Conflicts: # kubernetes/apps/kube-system/kustomization.yaml
ROCm publishes the chart via GitHub Pages (rocm.github.io/k8s-gpu-dra-driver, .github/workflows/release.yaml's helm-gh-pages job) rather than an OCI package. HelmRepository + chart.spec is the native fit for that — closer to how every other app in this repo resolves a chart than pinning a raw git tag/path, and the published chart's version already matches its appVersion (v1.0.1), so the explicit image.tag override from the previous commit is gone too. Verified with 'helm template' against the actual published .tgz artifact.
![coderabbitai[bot]](https://avatars.githubusercontent.com/in/347564?s=60&v=4){kind=small}
Tanguille
force-pushed
the
refactor/migrate-to-dra-amdgpu
branch
from
August 21, 2026 21:03
d7557e8 to
9f29bcb
Compare
|
Warning Your free Security trial is over. An organization admin can activate billing to continue. |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
1 participant
Add this suggestion to a batch that can be applied as a single commit.This suggestion is invalid because no changes were made to the code.Suggestions cannot be applied while the pull request is closed.Suggestions cannot be applied while viewing a subset of changes.Only one suggestion per line can be applied in a batch.Add this suggestion to a batch that can be applied as a single commit.Applying suggestions on deleted lines is not supported.You must change the existing code in this line in order to create a valid suggestion.Outdated suggestions cannot be applied.This suggestion has been applied or marked resolved.Suggestions cannot be applied from pending reviews.Suggestions cannot be applied on multi-line comments.Suggestions cannot be applied while the pull request is queued to merge.Suggestion cannot be applied right now. Please check back later.
Summary
Migrates AMD GPU access from
squat/generic-device-plugin(extended resourcesquat.ai/dri) to Kubernetes Dynamic Resource Allocation (DRA) viak8s-gpu-dra-driver(ROCm upstream,buroa/helmchart v0.1.0).generic-device-plugin; addsk8s-gpu-dra-driver(wait: true) inkube-systemsquat.ai/dri: 1requests with DRAresourceClaims/claimsacross llama-server, llama-embeddings, fileflows, jellyfin, gpu-builder, drm-exportercomponents/dra-dgpu(+ igpu template)ResourceClaimTemplatesdrm-exportertokube-systemwith DRA-native mode +resource.kubernetes.io/admin-accessnamespace labelmediashares the dGPU between jellyfin + fileflows via a namespace-levelshared-dgpuResourceClaimBack to draft because the core scenario does not work as written.
The ROCm driver (
docker.io/rocm/k8s-gpu-dra-driver, v1.0.0 image) implements neither:adminAccess— no code path (lets a second claim bind an already-allocated device), andResourceClaimis namespaced, so a pod can only reference a claim in its own namespace. control-1's single dGPU is wanted by three namespaces simultaneously:aidgputemplate → exclusivemediashared-dgpuclaim → shared within mediakube-systemWith no adminAccess and no consumable capacity, DRA allocates the device to exactly one claim — the other namespaces' pods go Pending. jellyfin + fileflows share fine with each other (same namespace, one claim); they cannot share with inference or monitoring across namespaces.
(
gpu-builderis exempt — it is scale-to-zero and mutually exclusive with inference by design.)When this works properly
adminAccessor consumable capacity (KEP-5075). The K8s scheduler side is already in place (cluster on ~1.36, KEP-5075 beta); the driver is the only blocker.Canary test before any full flip
drm-exporter (
kube-system, admin-access) + llama-server (ai, exclusive) are already a cross-namespace pair in this PR. Deploy the driver and observe whether both land Running on control-1:adminAccessworks (it is control-plane-enforced; the driver may not need explicit code). Path forward: give jellyfin/fileflows adminAccess claims and labelmedia+ainamespaces./dev/dri+/dev/kfdfor secondary consumers, or keepgeneric-device-pluginfor the dGPU and use DRA only where cross-namespace sharing is not required.Upstream tracking
No dated timeline exists yet for either blocker — tracking for follow-up:
k8s-gpu-dra-driver— no tracking issue exists upstream for adminAccess/consumable-capacity support (checked all open issues as of 2026-08-09).Recent changes on this branch
refactor(media): decouple shared-dgpu claim from fileflows— moved theshared-dgpuResourceClaimto themedianamespace level so jellyfin no longer depends on fileflows for claim lifecycle; clarified thekube-systemnamespace-patch comment.Decision needed
Run the canary first. Do not flip media/inference off
generic-device-pluginuntil co-scheduling on control-1 is confirmed.