diff --git a/.claude/CLAUDE.md b/.claude/CLAUDE.md index e20b6df3..c33224c1 100644 --- a/.claude/CLAUDE.md +++ b/.claude/CLAUDE.md @@ -18,6 +18,8 @@ Topograph discovers the physical network topology of a cluster (NVLink domains, Providers differ by environment. The canonical `topology.Graph` is stable. Engines only translate — they do not discover. +Within a provider, network-fabric and accelerator-domain discovery may be composed independently through `pkg/accelerator`; the provider remains responsible for combining both dimensions into the canonical graph. + This separation is load-bearing. If you find yourself reading the fabric in an engine, or emitting scheduler-specific output from a provider, stop and reconsider. ### Repository map @@ -25,6 +27,7 @@ This separation is load-bearing. If you find yourself reading the fabric in an e ``` cmd/ # Entry points: topograph, node-observer, node-data-broker, kwok-nodes pkg/ + accelerator/ # Pluggable accelerator-domain discovery composed by providers providers/ # One directory per provider: aws, gcp, oci, nebius, netq, dra, infiniband, lambdai, test engines/ # One directory per engine: k8s, nfd, slinky, slurm topology/ # Canonical Graph, Vertex tree, and topology constants (DO NOT CHANGE CASUALLY) diff --git a/AGENTS.md b/AGENTS.md index 424be31e..d02098cd 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -18,6 +18,8 @@ Topograph discovers the physical network topology of a cluster (NVLink domains, Providers differ by environment. The canonical `topology.Graph` is stable. Engines only translate — they do not discover. +Within a provider, network-fabric and accelerator-domain discovery may be composed independently through `pkg/accelerator`; the provider remains responsible for combining both dimensions into the canonical graph. + This separation is load-bearing. If you find yourself reading the fabric in an engine, or emitting scheduler-specific output from a provider, stop and reconsider. ### Repository map @@ -25,6 +27,7 @@ This separation is load-bearing. If you find yourself reading the fabric in an e ``` cmd/ # Entry points: topograph, node-observer, node-data-broker, kwok-nodes pkg/ + accelerator/ # Pluggable accelerator-domain discovery composed by providers providers/ # One directory per provider: aws, gcp, oci, nebius, netq, dra, infiniband, lambdai, test engines/ # One directory per engine: k8s, nfd, slinky, slurm topology/ # Canonical Graph, Vertex tree, and topology constants (DO NOT CHANGE CASUALLY) diff --git a/CHANGELOG.md b/CHANGELOG.md index 782dedd8..fd2f8f2f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/). ### Added +- Pluggable accelerator-domain discovery for InfiniBand providers, independently selectable from fabric discovery with `nvidia-smi`, an explicitly configured Kubernetes Node label, or no accelerator source. Discovery is disabled when `accelerator` is omitted or empty; a non-empty section must set `source` explicitly. Helm defaults the `nvidia-smi` workload location to the `gpu-operator` namespace and `nvidia-device-plugin-daemonset` DaemonSet when those values are omitted. - Helm `kubeClient.qps` and `kubeClient.burst` values for tuning the DRA provider and the Kubernetes, NFD, and Slinky engine clients through deployment-level `KUBE_QPS` and `KUBE_BURST` settings. - The Kubernetes engine now publishes `accelerator.topograph.run/sub-domain` when a provider supplies `InstanceTopology.XclrSubDomainID`. - The NFD engine now publishes separate `xclr-domain` and `xclr-sub-domain` attributes and groups. @@ -26,6 +27,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/). ### Changed +- InfiniBand providers now query NVL partition IDs with the `nvidia-smi` CSV query interface, merge identical per-GPU rows, reject unavailable (`N/A`) fields, and normalize the result to `ClusterUUID.CliqueId`. - **BREAKING:** The default Kubernetes topology labels now use the vendor-neutral Topograph domains `fabric.topograph.run/tier-N`, `accelerator.topograph.run/domain`, and `accelerator.topograph.run/sub-domain`. Consumers must update topology keys, selectors, allowlists, and scheduling policies to use the new labels. - The node-observer now processes its existing topology-generation triggers through a client-go rate-limiting work queue, coalescing event bursts into a single cluster-wide reconciliation while preserving existing trigger and retry behavior. - Simulation models now define accelerator topology through inherited `switches[].annotations` and `blocks[].annotations` using `accelerator.topology.test/domain` and optional `accelerator.topology.test/sub-domain`. diff --git a/charts/topograph/templates/NOTES.txt b/charts/topograph/templates/NOTES.txt index 228d60e1..fa9f84ab 100644 --- a/charts/topograph/templates/NOTES.txt +++ b/charts/topograph/templates/NOTES.txt @@ -42,6 +42,11 @@ NOTE: node-data-broker applies node annotations once when each broker pod starts. {{- end }} +{{- if eq .Values.provider.name "infiniband-k8s" }} + + NOTE: InfiniBand fabric discovery uses ibnetdiscover; accelerator-domain + discovery source is {{ include "nodeDataBroker.acceleratorSource" . }}. +{{- end }} {{- if eq .Values.engine.name "nfd" }} NOTE: The NFD engine writes NodeFeature and NodeFeatureGroup resources in diff --git a/charts/topograph/templates/_validation.tpl b/charts/topograph/templates/_validation.tpl index 3bfd55d5..4eda04c2 100644 --- a/charts/topograph/templates/_validation.tpl +++ b/charts/topograph/templates/_validation.tpl @@ -20,6 +20,40 @@ {{- fail "env.KUBE_BURST is managed by the chart; configure kubeClient.burst instead" }} {{- end }} +{{- if or (eq .Values.provider.name "infiniband-k8s") (eq .Values.provider.name "infiniband-bm") }} +{{- $params := default dict .Values.provider.params }} +{{- $acceleratorValue := get $params "accelerator" }} +{{- $accelerator := default dict $acceleratorValue }} +{{- $source := "none" }} +{{- if hasKey $params "accelerator" }} +{{- if and (kindIs "map" $acceleratorValue) (eq (len $acceleratorValue) 0) }} +{{- $source = "none" }} +{{- else }} +{{- $source = lower (toString (get $accelerator "source")) }} +{{- if eq $source "" }} + {{- fail "provider.params.accelerator.source must be set when provider.params.accelerator is present" }} +{{- end }} +{{- end }} +{{- end }} + +{{- if not (has $source (list "nvidia-smi" "kubernetes-label" "none")) }} + {{- fail (printf "unsupported provider.params.accelerator.source %q" $source) }} +{{- end }} + +{{- if and (eq .Values.provider.name "infiniband-k8s") (eq $source "kubernetes-label") }} +{{- $kubernetesLabel := default dict (get $accelerator "kubernetesLabel") }} +{{- $key := trim (toString (get $kubernetesLabel "key")) }} +{{- if eq $key "" }} + {{- fail "provider.params.accelerator.kubernetesLabel.key must be set for source kubernetes-label" }} +{{- end }} +{{- end }} + +{{- if and (eq .Values.provider.name "infiniband-bm") (eq $source "kubernetes-label") }} + {{- fail "provider.params.accelerator.source kubernetes-label is not supported by infiniband-bm" }} +{{- end }} + +{{- end }} + {{- if eq .Values.provider.name "gcp" }} {{- $params := default dict .Values.provider.params }} diff --git a/charts/topograph/templates/nodeDataBroker/_helpers.tpl b/charts/topograph/templates/nodeDataBroker/_helpers.tpl index 7c30964f..d3f2c1e1 100644 --- a/charts/topograph/templates/nodeDataBroker/_helpers.tpl +++ b/charts/topograph/templates/nodeDataBroker/_helpers.tpl @@ -53,6 +53,50 @@ Create the name of the RBAC resources. {{- include "nodeDataBroker.fullname" . }} {{- end }} +{{/* Resolve the configured accelerator source. */}} +{{- define "nodeDataBroker.acceleratorSource" -}} +{{- $providerParams := default dict .Values.provider.params -}} +{{- $acceleratorValue := get $providerParams "accelerator" -}} +{{- $accelerator := default dict $acceleratorValue -}} +{{- $source := "none" -}} +{{- if hasKey $providerParams "accelerator" -}} +{{- if and (kindIs "map" $acceleratorValue) (eq (len $acceleratorValue) 0) -}} +{{- $source = "none" -}} +{{- else -}} +{{- $source = get $accelerator "source" -}} +{{- if empty $source -}} +{{- fail "provider.params.accelerator.source must be set when provider.params.accelerator is present" -}} +{{- end -}} +{{- end -}} +{{- end -}} +{{- lower (toString $source) -}} +{{- end }} + +{{/* +Render the provider configuration used by node-data-broker. The broker needs +the GPU Operator workload location when nvidia-smi discovery is enabled, so +materialize its defaults in the generated configuration while preserving +explicit overrides. +*/}} +{{- define "nodeDataBroker.providerConfig" -}} +{{- $provider := deepCopy .Values.provider -}} +{{- if eq (include "nodeDataBroker.acceleratorSource" .) "nvidia-smi" -}} +{{- $params := default dict (get $provider "params") -}} +{{- $accelerator := default dict (get $params "accelerator") -}} +{{- $nvidiaSmi := default dict (get $accelerator "nvidiaSmi") -}} +{{- if empty (trim (toString (get $nvidiaSmi "gpuOperatorNamespace"))) -}} +{{- $_ := set $nvidiaSmi "gpuOperatorNamespace" "gpu-operator" -}} +{{- end -}} +{{- if empty (trim (toString (get $nvidiaSmi "devicePluginDaemonSet"))) -}} +{{- $_ := set $nvidiaSmi "devicePluginDaemonSet" "nvidia-device-plugin-daemonset" -}} +{{- end -}} +{{- $_ := set $accelerator "nvidiaSmi" $nvidiaSmi -}} +{{- $_ := set $params "accelerator" $accelerator -}} +{{- $_ := set $provider "params" $params -}} +{{- end -}} +{{- toYaml $provider -}} +{{- end }} + {{/* Create the name of a generated ConfigMap mount. */}} diff --git a/charts/topograph/templates/nodeDataBroker/configmap.yaml b/charts/topograph/templates/nodeDataBroker/configmap.yaml new file mode 100644 index 00000000..d79afa50 --- /dev/null +++ b/charts/topograph/templates/nodeDataBroker/configmap.yaml @@ -0,0 +1,14 @@ +{{- if .Values.nodeDataBroker.enabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "nodeDataBroker.fullname" . }} + namespace: {{ .Release.Namespace }} + labels: + {{- include "nodeDataBroker.labels" . | nindent 4 }} +data: + node-data-broker-config.yaml: |- + provider: + {{- include "nodeDataBroker.providerConfig" . | nindent 6 }} + healthzPort: {{ .Values.nodeDataBroker.port }} +{{- end }} diff --git a/charts/topograph/templates/nodeDataBroker/daemonset.yaml b/charts/topograph/templates/nodeDataBroker/daemonset.yaml index 3a1d52df..ef4ca5a1 100644 --- a/charts/topograph/templates/nodeDataBroker/daemonset.yaml +++ b/charts/topograph/templates/nodeDataBroker/daemonset.yaml @@ -1,6 +1,4 @@ {{- if .Values.nodeDataBroker.enabled }} -{{- $providerParams := default dict .Values.provider.params }} -{{- $useGpuCliqueLabel := and (eq .Values.provider.name "infiniband-k8s") (eq (lower (toString (get $providerParams "useGpuCliqueLabel"))) "true") }} {{- $configMapMounts := default list .Values.nodeDataBroker.configMapMounts }} apiVersion: apps/v1 kind: DaemonSet @@ -15,6 +13,8 @@ spec: {{- include "nodeDataBroker.selectorLabels" . | nindent 6 }} template: metadata: + annotations: + checksum/config: {{ include (print $.Template.BasePath "/nodeDataBroker/configmap.yaml") . | sha256sum }} labels: {{- include "nodeDataBroker.labels" . | nindent 8 }} spec: @@ -38,15 +38,9 @@ spec: command: - /usr/local/bin/node-data-broker args: - - --provider={{ .Values.provider.name }} + - -c + - /etc/topograph/node-data-broker-config.yaml - -v={{ .Values.verbosity }} - - --port={{ .Values.nodeDataBroker.port }} - {{- if $useGpuCliqueLabel }} - - --set=useGpuCliqueLabel=true - {{- end }} - {{- range .Values.nodeDataBroker.extraArgs }} - - --set={{ . }} - {{- end }} env: - name: NODE_NAME valueFrom: @@ -85,8 +79,10 @@ spec: port: http resources: {{- toYaml .Values.nodeDataBroker.resources | nindent 12 }} - {{- if or $configMapMounts .Values.nodeDataBroker.volumeMounts }} volumeMounts: + - name: config-volume + mountPath: /etc/topograph + readOnly: true {{- range $configMapMounts }} - name: {{ include "nodeDataBroker.configMapMountVolumeName" (dict "name" .name) }} mountPath: {{ required "nodeDataBroker.configMapMounts[].mountPath is required" .mountPath | quote }} @@ -98,9 +94,11 @@ spec: {{- with .Values.nodeDataBroker.volumeMounts }} {{- toYaml . | nindent 12 }} {{- end }} - {{- end }} - {{- if or $configMapMounts .Values.nodeDataBroker.volumes }} volumes: + - name: config-volume + configMap: + defaultMode: 420 + name: {{ include "nodeDataBroker.fullname" . }} {{- range $configMapMounts }} - name: {{ include "nodeDataBroker.configMapMountVolumeName" (dict "name" .name) }} configMap: @@ -109,7 +107,6 @@ spec: {{- with .Values.nodeDataBroker.volumes }} {{- toYaml . | nindent 8 }} {{- end }} - {{- end }} {{- with .Values.nodeDataBroker.nodeSelector }} nodeSelector: {{- toYaml . | nindent 8 }} diff --git a/charts/topograph/templates/nodeDataBroker/rbac.yaml b/charts/topograph/templates/nodeDataBroker/rbac.yaml index d3d710a7..075d6c41 100644 --- a/charts/topograph/templates/nodeDataBroker/rbac.yaml +++ b/charts/topograph/templates/nodeDataBroker/rbac.yaml @@ -1,6 +1,5 @@ {{- if and .Values.nodeDataBroker.enabled .Values.nodeDataBroker.rbac.create }} -{{- $providerParams := default dict .Values.provider.params }} -{{- $useGpuCliqueLabel := and (eq .Values.provider.name "infiniband-k8s") (eq (lower (toString (get $providerParams "useGpuCliqueLabel"))) "true") }} +{{- $acceleratorSource := include "nodeDataBroker.acceleratorSource" . }} apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: @@ -9,7 +8,7 @@ rules: - apiGroups: [""] resources: [nodes] verbs: [get,update] -{{- if and (eq .Values.provider.name "infiniband-k8s") (not $useGpuCliqueLabel) }} +{{- if and (eq .Values.provider.name "infiniband-k8s") (eq $acceleratorSource "nvidia-smi") }} - apiGroups: [apps] resources: [daemonsets] verbs: [get] diff --git a/charts/topograph/tests/__snapshot__/render_snapshot_test.yaml.snap b/charts/topograph/tests/__snapshot__/render_snapshot_test.yaml.snap index fff19d6b..5b72eca7 100644 --- a/charts/topograph/tests/__snapshot__/render_snapshot_test.yaml.snap +++ b/charts/topograph/tests/__snapshot__/render_snapshot_test.yaml.snap @@ -112,6 +112,23 @@ renders default values.yaml: name: chart-ci-topograph name: config-volume 4: | + apiVersion: v1 + data: + node-data-broker-config.yaml: |- + provider: + name: test + healthzPort: 8080 + kind: ConfigMap + metadata: + labels: + app.kubernetes.io/instance: chart-ci + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/name: node-data-broker + app.kubernetes.io/version: v0.0.0 + helm.sh/chart: node-data-broker-0.0.0 + name: chart-ci-topograph-node-data-broker + namespace: topograph + 5: | apiVersion: apps/v1 kind: DaemonSet metadata: @@ -130,6 +147,8 @@ renders default values.yaml: app.kubernetes.io/name: node-data-broker template: metadata: + annotations: + checksum/config: bd532548889904dc125da3ebef942b98b2a68aeedd1f4932a07abe35989e5aa1 labels: app.kubernetes.io/instance: chart-ci app.kubernetes.io/managed-by: Helm @@ -139,9 +158,9 @@ renders default values.yaml: spec: containers: - args: - - --provider=test + - -c + - /etc/topograph/node-data-broker-config.yaml - -v=3 - - --port=8080 command: - /usr/local/bin/node-data-broker env: @@ -184,6 +203,10 @@ renders default values.yaml: path: /healthz port: http periodSeconds: 10 + volumeMounts: + - mountPath: /etc/topograph + name: config-volume + readOnly: true securityContext: fsGroup: 65532 runAsGroup: 65532 @@ -192,7 +215,12 @@ renders default values.yaml: seccompProfile: type: RuntimeDefault serviceAccountName: chart-ci-topograph-node-data-broker - 5: | + volumes: + - configMap: + defaultMode: 420 + name: chart-ci-topograph-node-data-broker + name: config-volume + 6: | apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: @@ -205,7 +233,7 @@ renders default values.yaml: verbs: - get - update - 6: | + 7: | apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: @@ -219,7 +247,7 @@ renders default values.yaml: kind: ServiceAccount name: chart-ci-topograph-node-data-broker namespace: topograph - 7: | + 8: | apiVersion: v1 automountServiceAccountToken: true kind: ServiceAccount @@ -232,7 +260,7 @@ renders default values.yaml: helm.sh/chart: node-data-broker-0.0.0 name: chart-ci-topograph-node-data-broker namespace: topograph - 8: | + 9: | apiVersion: v1 data: node-observer-config.yaml: |- @@ -260,7 +288,7 @@ renders default values.yaml: helm.sh/chart: node-observer-0.0.0 name: chart-ci-topograph-node-observer namespace: topograph - 9: | + 10: | apiVersion: apps/v1 kind: Deployment metadata: @@ -332,7 +360,7 @@ renders default values.yaml: defaultMode: 420 name: chart-ci-topograph-node-observer name: config-volume - 10: | + 11: | apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: @@ -352,7 +380,7 @@ renders default values.yaml: verbs: - list - watch - 11: | + 12: | apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: @@ -366,7 +394,7 @@ renders default values.yaml: kind: ServiceAccount name: chart-ci-topograph-node-observer namespace: topograph - 12: | + 13: | apiVersion: v1 automountServiceAccountToken: true kind: ServiceAccount @@ -379,7 +407,7 @@ renders default values.yaml: helm.sh/chart: node-observer-0.0.0 name: chart-ci-topograph-node-observer namespace: topograph - 13: | + 14: | apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: @@ -410,7 +438,7 @@ renders default values.yaml: - daemonsets verbs: - get - 14: | + 15: | apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: @@ -424,7 +452,7 @@ renders default values.yaml: kind: ServiceAccount name: chart-ci-topograph namespace: topograph - 15: | + 16: | apiVersion: v1 kind: Service metadata: @@ -446,7 +474,7 @@ renders default values.yaml: app.kubernetes.io/instance: chart-ci app.kubernetes.io/name: topograph type: ClusterIP - 16: | + 17: | apiVersion: v1 automountServiceAccountToken: true kind: ServiceAccount @@ -459,7 +487,7 @@ renders default values.yaml: helm.sh/chart: topograph-0.0.0 name: chart-ci-topograph namespace: topograph - 17: | + 18: | apiVersion: v1 kind: Pod metadata: @@ -502,7 +530,7 @@ renders default values.yaml: runAsUser: 65532 seccompProfile: type: RuntimeDefault - 18: | + 19: | apiVersion: v1 kind: Pod metadata: @@ -682,6 +710,23 @@ renders values.k8s.gateway-api-example.yaml: - name: chart-ci-topograph port: 49021 5: | + apiVersion: v1 + data: + node-data-broker-config.yaml: |- + provider: + name: test + healthzPort: 8080 + kind: ConfigMap + metadata: + labels: + app.kubernetes.io/instance: chart-ci + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/name: node-data-broker + app.kubernetes.io/version: v0.0.0 + helm.sh/chart: node-data-broker-0.0.0 + name: chart-ci-topograph-node-data-broker + namespace: topograph + 6: | apiVersion: apps/v1 kind: DaemonSet metadata: @@ -700,6 +745,8 @@ renders values.k8s.gateway-api-example.yaml: app.kubernetes.io/name: node-data-broker template: metadata: + annotations: + checksum/config: bd532548889904dc125da3ebef942b98b2a68aeedd1f4932a07abe35989e5aa1 labels: app.kubernetes.io/instance: chart-ci app.kubernetes.io/managed-by: Helm @@ -709,9 +756,9 @@ renders values.k8s.gateway-api-example.yaml: spec: containers: - args: - - --provider=test + - -c + - /etc/topograph/node-data-broker-config.yaml - -v=3 - - --port=8080 command: - /usr/local/bin/node-data-broker env: @@ -754,6 +801,10 @@ renders values.k8s.gateway-api-example.yaml: path: /healthz port: http periodSeconds: 10 + volumeMounts: + - mountPath: /etc/topograph + name: config-volume + readOnly: true securityContext: fsGroup: 65532 runAsGroup: 65532 @@ -762,7 +813,12 @@ renders values.k8s.gateway-api-example.yaml: seccompProfile: type: RuntimeDefault serviceAccountName: chart-ci-topograph-node-data-broker - 6: | + volumes: + - configMap: + defaultMode: 420 + name: chart-ci-topograph-node-data-broker + name: config-volume + 7: | apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: @@ -775,7 +831,7 @@ renders values.k8s.gateway-api-example.yaml: verbs: - get - update - 7: | + 8: | apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: @@ -789,7 +845,7 @@ renders values.k8s.gateway-api-example.yaml: kind: ServiceAccount name: chart-ci-topograph-node-data-broker namespace: topograph - 8: | + 9: | apiVersion: v1 automountServiceAccountToken: true kind: ServiceAccount @@ -802,7 +858,7 @@ renders values.k8s.gateway-api-example.yaml: helm.sh/chart: node-data-broker-0.0.0 name: chart-ci-topograph-node-data-broker namespace: topograph - 9: | + 10: | apiVersion: v1 data: node-observer-config.yaml: |- @@ -830,7 +886,7 @@ renders values.k8s.gateway-api-example.yaml: helm.sh/chart: node-observer-0.0.0 name: chart-ci-topograph-node-observer namespace: topograph - 10: | + 11: | apiVersion: apps/v1 kind: Deployment metadata: @@ -902,7 +958,7 @@ renders values.k8s.gateway-api-example.yaml: defaultMode: 420 name: chart-ci-topograph-node-observer name: config-volume - 11: | + 12: | apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: @@ -922,7 +978,7 @@ renders values.k8s.gateway-api-example.yaml: verbs: - list - watch - 12: | + 13: | apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: @@ -936,7 +992,7 @@ renders values.k8s.gateway-api-example.yaml: kind: ServiceAccount name: chart-ci-topograph-node-observer namespace: topograph - 13: | + 14: | apiVersion: v1 automountServiceAccountToken: true kind: ServiceAccount @@ -949,7 +1005,7 @@ renders values.k8s.gateway-api-example.yaml: helm.sh/chart: node-observer-0.0.0 name: chart-ci-topograph-node-observer namespace: topograph - 14: | + 15: | apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: @@ -980,7 +1036,7 @@ renders values.k8s.gateway-api-example.yaml: - daemonsets verbs: - get - 15: | + 16: | apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: @@ -994,7 +1050,7 @@ renders values.k8s.gateway-api-example.yaml: kind: ServiceAccount name: chart-ci-topograph namespace: topograph - 16: | + 17: | apiVersion: v1 kind: Service metadata: @@ -1016,7 +1072,7 @@ renders values.k8s.gateway-api-example.yaml: app.kubernetes.io/instance: chart-ci app.kubernetes.io/name: topograph type: ClusterIP - 17: | + 18: | apiVersion: v1 automountServiceAccountToken: true kind: ServiceAccount @@ -1029,7 +1085,7 @@ renders values.k8s.gateway-api-example.yaml: helm.sh/chart: topograph-0.0.0 name: chart-ci-topograph namespace: topograph - 18: | + 19: | apiVersion: v1 kind: Pod metadata: @@ -1072,7 +1128,7 @@ renders values.k8s.gateway-api-example.yaml: runAsUser: 65532 seccompProfile: type: RuntimeDefault - 19: | + 20: | apiVersion: v1 kind: Pod metadata: @@ -1248,6 +1304,27 @@ renders values.k8s.gcp-federated-workload-identity-example.yaml: expirationSeconds: 3600 path: token 4: | + apiVersion: v1 + data: + node-data-broker-config.yaml: |- + provider: + name: gcp + params: + workloadIdentityFederation: + audience: //iam.googleapis.com/projects/123/locations/global/workloadIdentityPools/my-pool/providers/my-workload-provider + credentialsConfigmap: gcp-credentials-config + healthzPort: 8080 + kind: ConfigMap + metadata: + labels: + app.kubernetes.io/instance: chart-ci + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/name: node-data-broker + app.kubernetes.io/version: v0.0.0 + helm.sh/chart: node-data-broker-0.0.0 + name: chart-ci-topograph-node-data-broker + namespace: topograph + 5: | apiVersion: apps/v1 kind: DaemonSet metadata: @@ -1266,6 +1343,8 @@ renders values.k8s.gcp-federated-workload-identity-example.yaml: app.kubernetes.io/name: node-data-broker template: metadata: + annotations: + checksum/config: 7709945b32d9a79d791ff2d44e602304104eb3614fb569bffea667f4c4d93ac6 labels: app.kubernetes.io/instance: chart-ci app.kubernetes.io/managed-by: Helm @@ -1275,9 +1354,9 @@ renders values.k8s.gcp-federated-workload-identity-example.yaml: spec: containers: - args: - - --provider=gcp + - -c + - /etc/topograph/node-data-broker-config.yaml - -v=3 - - --port=8080 command: - /usr/local/bin/node-data-broker env: @@ -1320,6 +1399,10 @@ renders values.k8s.gcp-federated-workload-identity-example.yaml: path: /healthz port: http periodSeconds: 10 + volumeMounts: + - mountPath: /etc/topograph + name: config-volume + readOnly: true nodeSelector: brightcomputing.com/node-category: dgx securityContext: @@ -1330,7 +1413,12 @@ renders values.k8s.gcp-federated-workload-identity-example.yaml: seccompProfile: type: RuntimeDefault serviceAccountName: chart-ci-topograph-node-data-broker - 5: | + volumes: + - configMap: + defaultMode: 420 + name: chart-ci-topograph-node-data-broker + name: config-volume + 6: | apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: @@ -1343,7 +1431,7 @@ renders values.k8s.gcp-federated-workload-identity-example.yaml: verbs: - get - update - 6: | + 7: | apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: @@ -1357,7 +1445,7 @@ renders values.k8s.gcp-federated-workload-identity-example.yaml: kind: ServiceAccount name: chart-ci-topograph-node-data-broker namespace: topograph - 7: | + 8: | apiVersion: v1 automountServiceAccountToken: true kind: ServiceAccount @@ -1370,7 +1458,7 @@ renders values.k8s.gcp-federated-workload-identity-example.yaml: helm.sh/chart: node-data-broker-0.0.0 name: chart-ci-topograph-node-data-broker namespace: topograph - 8: | + 9: | apiVersion: v1 data: node-observer-config.yaml: |- @@ -1403,7 +1491,7 @@ renders values.k8s.gcp-federated-workload-identity-example.yaml: helm.sh/chart: node-observer-0.0.0 name: chart-ci-topograph-node-observer namespace: topograph - 9: | + 10: | apiVersion: apps/v1 kind: Deployment metadata: @@ -1475,7 +1563,7 @@ renders values.k8s.gcp-federated-workload-identity-example.yaml: defaultMode: 420 name: chart-ci-topograph-node-observer name: config-volume - 10: | + 11: | apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: @@ -1502,7 +1590,7 @@ renders values.k8s.gcp-federated-workload-identity-example.yaml: verbs: - list - watch - 11: | + 12: | apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: @@ -1516,7 +1604,7 @@ renders values.k8s.gcp-federated-workload-identity-example.yaml: kind: ServiceAccount name: chart-ci-topograph-node-observer namespace: topograph - 12: | + 13: | apiVersion: v1 automountServiceAccountToken: true kind: ServiceAccount @@ -1529,7 +1617,7 @@ renders values.k8s.gcp-federated-workload-identity-example.yaml: helm.sh/chart: node-observer-0.0.0 name: chart-ci-topograph-node-observer namespace: topograph - 13: | + 14: | apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: @@ -1560,7 +1648,7 @@ renders values.k8s.gcp-federated-workload-identity-example.yaml: - daemonsets verbs: - get - 14: | + 15: | apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: @@ -1574,7 +1662,7 @@ renders values.k8s.gcp-federated-workload-identity-example.yaml: kind: ServiceAccount name: chart-ci-topograph namespace: topograph - 15: | + 16: | apiVersion: v1 kind: Service metadata: @@ -1596,7 +1684,7 @@ renders values.k8s.gcp-federated-workload-identity-example.yaml: app.kubernetes.io/instance: chart-ci app.kubernetes.io/name: topograph type: ClusterIP - 16: | + 17: | apiVersion: v1 automountServiceAccountToken: true kind: ServiceAccount @@ -1609,7 +1697,7 @@ renders values.k8s.gcp-federated-workload-identity-example.yaml: helm.sh/chart: topograph-0.0.0 name: chart-ci-topograph namespace: topograph - 17: | + 18: | apiVersion: v1 kind: Pod metadata: @@ -1652,7 +1740,7 @@ renders values.k8s.gcp-federated-workload-identity-example.yaml: runAsUser: 65532 seccompProfile: type: RuntimeDefault - 18: | + 19: | apiVersion: v1 kind: Pod metadata: @@ -1818,6 +1906,25 @@ renders values.k8s.gcp-service-account-example.yaml: secret: secretName: gcp-service-account-keys-secret 4: | + apiVersion: v1 + data: + node-data-broker-config.yaml: |- + provider: + name: gcp + params: + serviceAccountKeysSecret: gcp-service-account-keys-secret + healthzPort: 8080 + kind: ConfigMap + metadata: + labels: + app.kubernetes.io/instance: chart-ci + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/name: node-data-broker + app.kubernetes.io/version: v0.0.0 + helm.sh/chart: node-data-broker-0.0.0 + name: chart-ci-topograph-node-data-broker + namespace: topograph + 5: | apiVersion: apps/v1 kind: DaemonSet metadata: @@ -1836,6 +1943,8 @@ renders values.k8s.gcp-service-account-example.yaml: app.kubernetes.io/name: node-data-broker template: metadata: + annotations: + checksum/config: d28de17e030df39953566950c02556951ed6dfebd979edd598366a83cfbdb425 labels: app.kubernetes.io/instance: chart-ci app.kubernetes.io/managed-by: Helm @@ -1845,9 +1954,9 @@ renders values.k8s.gcp-service-account-example.yaml: spec: containers: - args: - - --provider=gcp + - -c + - /etc/topograph/node-data-broker-config.yaml - -v=3 - - --port=8080 command: - /usr/local/bin/node-data-broker env: @@ -1890,6 +1999,10 @@ renders values.k8s.gcp-service-account-example.yaml: path: /healthz port: http periodSeconds: 10 + volumeMounts: + - mountPath: /etc/topograph + name: config-volume + readOnly: true nodeSelector: brightcomputing.com/node-category: dgx securityContext: @@ -1900,7 +2013,12 @@ renders values.k8s.gcp-service-account-example.yaml: seccompProfile: type: RuntimeDefault serviceAccountName: chart-ci-topograph-node-data-broker - 5: | + volumes: + - configMap: + defaultMode: 420 + name: chart-ci-topograph-node-data-broker + name: config-volume + 6: | apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: @@ -1913,7 +2031,7 @@ renders values.k8s.gcp-service-account-example.yaml: verbs: - get - update - 6: | + 7: | apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: @@ -1927,7 +2045,7 @@ renders values.k8s.gcp-service-account-example.yaml: kind: ServiceAccount name: chart-ci-topograph-node-data-broker namespace: topograph - 7: | + 8: | apiVersion: v1 automountServiceAccountToken: true kind: ServiceAccount @@ -1940,7 +2058,7 @@ renders values.k8s.gcp-service-account-example.yaml: helm.sh/chart: node-data-broker-0.0.0 name: chart-ci-topograph-node-data-broker namespace: topograph - 8: | + 9: | apiVersion: v1 data: node-observer-config.yaml: |- @@ -1971,7 +2089,7 @@ renders values.k8s.gcp-service-account-example.yaml: helm.sh/chart: node-observer-0.0.0 name: chart-ci-topograph-node-observer namespace: topograph - 9: | + 10: | apiVersion: apps/v1 kind: Deployment metadata: @@ -2043,7 +2161,7 @@ renders values.k8s.gcp-service-account-example.yaml: defaultMode: 420 name: chart-ci-topograph-node-observer name: config-volume - 10: | + 11: | apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: @@ -2070,7 +2188,7 @@ renders values.k8s.gcp-service-account-example.yaml: verbs: - list - watch - 11: | + 12: | apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: @@ -2084,7 +2202,7 @@ renders values.k8s.gcp-service-account-example.yaml: kind: ServiceAccount name: chart-ci-topograph-node-observer namespace: topograph - 12: | + 13: | apiVersion: v1 automountServiceAccountToken: true kind: ServiceAccount @@ -2097,7 +2215,7 @@ renders values.k8s.gcp-service-account-example.yaml: helm.sh/chart: node-observer-0.0.0 name: chart-ci-topograph-node-observer namespace: topograph - 13: | + 14: | apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: @@ -2128,7 +2246,7 @@ renders values.k8s.gcp-service-account-example.yaml: - daemonsets verbs: - get - 14: | + 15: | apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: @@ -2142,7 +2260,7 @@ renders values.k8s.gcp-service-account-example.yaml: kind: ServiceAccount name: chart-ci-topograph namespace: topograph - 15: | + 16: | apiVersion: v1 kind: Service metadata: @@ -2164,7 +2282,7 @@ renders values.k8s.gcp-service-account-example.yaml: app.kubernetes.io/instance: chart-ci app.kubernetes.io/name: topograph type: ClusterIP - 16: | + 17: | apiVersion: v1 automountServiceAccountToken: true kind: ServiceAccount @@ -2177,7 +2295,7 @@ renders values.k8s.gcp-service-account-example.yaml: helm.sh/chart: topograph-0.0.0 name: chart-ci-topograph namespace: topograph - 17: | + 18: | apiVersion: v1 kind: Pod metadata: @@ -2220,7 +2338,7 @@ renders values.k8s.gcp-service-account-example.yaml: runAsUser: 65532 seccompProfile: type: RuntimeDefault - 18: | + 19: | apiVersion: v1 kind: Pod metadata: @@ -2274,6 +2392,9 @@ renders values.k8s.ib-example.yaml: kubectl --namespace topograph port-forward $POD_NAME 8080:$CONTAINER_PORT NOTE: node-data-broker applies node annotations once when each broker pod starts. + + NOTE: InfiniBand fabric discovery uses ibnetdiscover; accelerator-domain + discovery source is kubernetes-label. 2: | apiVersion: v1 data: @@ -2378,6 +2499,28 @@ renders values.k8s.ib-example.yaml: name: chart-ci-topograph name: config-volume 4: | + apiVersion: v1 + data: + node-data-broker-config.yaml: |- + provider: + name: infiniband-k8s + params: + accelerator: + kubernetesLabel: + key: nvidia.com/gpu.clique + source: kubernetes-label + healthzPort: 8080 + kind: ConfigMap + metadata: + labels: + app.kubernetes.io/instance: chart-ci + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/name: node-data-broker + app.kubernetes.io/version: v0.0.0 + helm.sh/chart: node-data-broker-0.0.0 + name: chart-ci-topograph-node-data-broker + namespace: topograph + 5: | apiVersion: apps/v1 kind: DaemonSet metadata: @@ -2396,6 +2539,8 @@ renders values.k8s.ib-example.yaml: app.kubernetes.io/name: node-data-broker template: metadata: + annotations: + checksum/config: d514c887dd3e7d421a63cbf0e16b7fe313267988b4a91e6a20eea65e5a5059d5 labels: app.kubernetes.io/instance: chart-ci app.kubernetes.io/managed-by: Helm @@ -2405,10 +2550,9 @@ renders values.k8s.ib-example.yaml: spec: containers: - args: - - --provider=infiniband-k8s + - -c + - /etc/topograph/node-data-broker-config.yaml - -v=3 - - --port=8080 - - --set=useGpuCliqueLabel=true command: - /usr/local/bin/node-data-broker env: @@ -2453,6 +2597,9 @@ renders values.k8s.ib-example.yaml: port: http periodSeconds: 10 volumeMounts: + - mountPath: /etc/topograph + name: config-volume + readOnly: true - mountPath: /sys/class name: sys-class-volume nodeSelector: @@ -2466,11 +2613,15 @@ renders values.k8s.ib-example.yaml: type: RuntimeDefault serviceAccountName: chart-ci-topograph-node-data-broker volumes: + - configMap: + defaultMode: 420 + name: chart-ci-topograph-node-data-broker + name: config-volume - hostPath: path: /sys/class type: Directory name: sys-class-volume - 5: | + 6: | apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: @@ -2483,7 +2634,7 @@ renders values.k8s.ib-example.yaml: verbs: - get - update - 6: | + 7: | apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: @@ -2497,7 +2648,7 @@ renders values.k8s.ib-example.yaml: kind: ServiceAccount name: chart-ci-topograph-node-data-broker namespace: topograph - 7: | + 8: | apiVersion: v1 automountServiceAccountToken: true kind: ServiceAccount @@ -2510,7 +2661,7 @@ renders values.k8s.ib-example.yaml: helm.sh/chart: node-data-broker-0.0.0 name: chart-ci-topograph-node-data-broker namespace: topograph - 8: | + 9: | apiVersion: v1 data: node-observer-config.yaml: |- @@ -2518,7 +2669,10 @@ renders values.k8s.ib-example.yaml: provider: name: infiniband-k8s params: - useGpuCliqueLabel: true + accelerator: + kubernetesLabel: + key: nvidia.com/gpu.clique + source: kubernetes-label engine: name: k8s apiServer: @@ -2541,7 +2695,7 @@ renders values.k8s.ib-example.yaml: helm.sh/chart: node-observer-0.0.0 name: chart-ci-topograph-node-observer namespace: topograph - 9: | + 10: | apiVersion: apps/v1 kind: Deployment metadata: @@ -2562,7 +2716,7 @@ renders values.k8s.ib-example.yaml: template: metadata: annotations: - checksum/config: 12ded19ccdc256a879be87750e73e8df0458ba73f2df1995312e94f790dedfd5 + checksum/config: 9e19c577f18b59e7b090e402b725767d075607550a9eff0a5ec763eb31f1717d labels: app.kubernetes.io/instance: chart-ci app.kubernetes.io/managed-by: Helm @@ -2613,7 +2767,7 @@ renders values.k8s.ib-example.yaml: defaultMode: 420 name: chart-ci-topograph-node-observer name: config-volume - 10: | + 11: | apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: @@ -2640,7 +2794,7 @@ renders values.k8s.ib-example.yaml: verbs: - list - watch - 11: | + 12: | apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: @@ -2654,7 +2808,7 @@ renders values.k8s.ib-example.yaml: kind: ServiceAccount name: chart-ci-topograph-node-observer namespace: topograph - 12: | + 13: | apiVersion: v1 automountServiceAccountToken: true kind: ServiceAccount @@ -2667,7 +2821,7 @@ renders values.k8s.ib-example.yaml: helm.sh/chart: node-observer-0.0.0 name: chart-ci-topograph-node-observer namespace: topograph - 13: | + 14: | apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: @@ -2704,7 +2858,7 @@ renders values.k8s.ib-example.yaml: - daemonsets verbs: - get - 14: | + 15: | apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: @@ -2718,7 +2872,7 @@ renders values.k8s.ib-example.yaml: kind: ServiceAccount name: chart-ci-topograph namespace: topograph - 15: | + 16: | apiVersion: v1 kind: Service metadata: @@ -2740,7 +2894,7 @@ renders values.k8s.ib-example.yaml: app.kubernetes.io/instance: chart-ci app.kubernetes.io/name: topograph type: ClusterIP - 16: | + 17: | apiVersion: v1 automountServiceAccountToken: true kind: ServiceAccount @@ -2753,7 +2907,7 @@ renders values.k8s.ib-example.yaml: helm.sh/chart: topograph-0.0.0 name: chart-ci-topograph namespace: topograph - 17: | + 18: | apiVersion: v1 kind: Pod metadata: @@ -2796,7 +2950,7 @@ renders values.k8s.ib-example.yaml: runAsUser: 65532 seccompProfile: type: RuntimeDefault - 18: | + 19: | apiVersion: v1 kind: Pod metadata: @@ -2958,6 +3112,26 @@ renders values.slinky.block-example.yaml: name: chart-ci-topograph name: config-volume 4: | + apiVersion: v1 + data: + node-data-broker-config.yaml: |- + provider: + name: aws + params: + nodeSelector: + slurmCluster: my-cluster + healthzPort: 8080 + kind: ConfigMap + metadata: + labels: + app.kubernetes.io/instance: chart-ci + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/name: node-data-broker + app.kubernetes.io/version: v0.0.0 + helm.sh/chart: node-data-broker-0.0.0 + name: chart-ci-topograph-node-data-broker + namespace: topograph + 5: | apiVersion: apps/v1 kind: DaemonSet metadata: @@ -2976,6 +3150,8 @@ renders values.slinky.block-example.yaml: app.kubernetes.io/name: node-data-broker template: metadata: + annotations: + checksum/config: 33bfa8a8dcabba211a36e76d9381c9ae2d89a290f76522dfe78cd2345942441e labels: app.kubernetes.io/instance: chart-ci app.kubernetes.io/managed-by: Helm @@ -2985,9 +3161,9 @@ renders values.slinky.block-example.yaml: spec: containers: - args: - - --provider=aws + - -c + - /etc/topograph/node-data-broker-config.yaml - -v=3 - - --port=8080 command: - /usr/local/bin/node-data-broker env: @@ -3030,6 +3206,10 @@ renders values.slinky.block-example.yaml: path: /healthz port: http periodSeconds: 10 + volumeMounts: + - mountPath: /etc/topograph + name: config-volume + readOnly: true securityContext: fsGroup: 65532 runAsGroup: 65532 @@ -3038,7 +3218,12 @@ renders values.slinky.block-example.yaml: seccompProfile: type: RuntimeDefault serviceAccountName: chart-ci-topograph-node-data-broker - 5: | + volumes: + - configMap: + defaultMode: 420 + name: chart-ci-topograph-node-data-broker + name: config-volume + 6: | apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: @@ -3051,7 +3236,7 @@ renders values.slinky.block-example.yaml: verbs: - get - update - 6: | + 7: | apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: @@ -3065,7 +3250,7 @@ renders values.slinky.block-example.yaml: kind: ServiceAccount name: chart-ci-topograph-node-data-broker namespace: topograph - 7: | + 8: | apiVersion: v1 automountServiceAccountToken: true kind: ServiceAccount @@ -3078,7 +3263,7 @@ renders values.slinky.block-example.yaml: helm.sh/chart: node-data-broker-0.0.0 name: chart-ci-topograph-node-data-broker namespace: topograph - 8: | + 9: | apiVersion: v1 data: node-observer-config.yaml: |- @@ -3124,7 +3309,7 @@ renders values.slinky.block-example.yaml: helm.sh/chart: node-observer-0.0.0 name: chart-ci-topograph-node-observer namespace: topograph - 9: | + 10: | apiVersion: apps/v1 kind: Deployment metadata: @@ -3198,7 +3383,7 @@ renders values.slinky.block-example.yaml: defaultMode: 420 name: chart-ci-topograph-node-observer name: config-volume - 10: | + 11: | apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: @@ -3218,7 +3403,7 @@ renders values.slinky.block-example.yaml: verbs: - list - watch - 11: | + 12: | apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: @@ -3232,7 +3417,7 @@ renders values.slinky.block-example.yaml: kind: ServiceAccount name: chart-ci-topograph-node-observer namespace: topograph - 12: | + 13: | apiVersion: v1 automountServiceAccountToken: true kind: ServiceAccount @@ -3245,7 +3430,7 @@ renders values.slinky.block-example.yaml: helm.sh/chart: node-observer-0.0.0 name: chart-ci-topograph-node-observer namespace: topograph - 13: | + 14: | apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: @@ -3284,7 +3469,7 @@ renders values.slinky.block-example.yaml: - create - get - update - 14: | + 15: | apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: @@ -3298,7 +3483,7 @@ renders values.slinky.block-example.yaml: kind: ServiceAccount name: chart-ci-topograph namespace: topograph - 15: | + 16: | apiVersion: v1 kind: Service metadata: @@ -3320,7 +3505,7 @@ renders values.slinky.block-example.yaml: app.kubernetes.io/instance: chart-ci app.kubernetes.io/name: topograph type: ClusterIP - 16: | + 17: | apiVersion: v1 automountServiceAccountToken: true kind: ServiceAccount @@ -3333,7 +3518,7 @@ renders values.slinky.block-example.yaml: helm.sh/chart: topograph-0.0.0 name: chart-ci-topograph namespace: topograph - 17: | + 18: | apiVersion: v1 kind: Pod metadata: @@ -3376,7 +3561,7 @@ renders values.slinky.block-example.yaml: runAsUser: 65532 seccompProfile: type: RuntimeDefault - 18: | + 19: | apiVersion: v1 kind: Pod metadata: @@ -3538,6 +3723,23 @@ renders values.slinky.partition-example.yaml: name: chart-ci-topograph name: config-volume 4: | + apiVersion: v1 + data: + node-data-broker-config.yaml: |- + provider: + name: aws + healthzPort: 8080 + kind: ConfigMap + metadata: + labels: + app.kubernetes.io/instance: chart-ci + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/name: node-data-broker + app.kubernetes.io/version: v0.0.0 + helm.sh/chart: node-data-broker-0.0.0 + name: chart-ci-topograph-node-data-broker + namespace: topograph + 5: | apiVersion: apps/v1 kind: DaemonSet metadata: @@ -3556,6 +3758,8 @@ renders values.slinky.partition-example.yaml: app.kubernetes.io/name: node-data-broker template: metadata: + annotations: + checksum/config: 51fe2be1a13f3d4bc84430f46ef0a1242139b4079e5b12304a29d359de94aa0a labels: app.kubernetes.io/instance: chart-ci app.kubernetes.io/managed-by: Helm @@ -3565,9 +3769,9 @@ renders values.slinky.partition-example.yaml: spec: containers: - args: - - --provider=aws + - -c + - /etc/topograph/node-data-broker-config.yaml - -v=3 - - --port=8080 command: - /usr/local/bin/node-data-broker env: @@ -3610,6 +3814,10 @@ renders values.slinky.partition-example.yaml: path: /healthz port: http periodSeconds: 10 + volumeMounts: + - mountPath: /etc/topograph + name: config-volume + readOnly: true securityContext: fsGroup: 65532 runAsGroup: 65532 @@ -3618,7 +3826,12 @@ renders values.slinky.partition-example.yaml: seccompProfile: type: RuntimeDefault serviceAccountName: chart-ci-topograph-node-data-broker - 5: | + volumes: + - configMap: + defaultMode: 420 + name: chart-ci-topograph-node-data-broker + name: config-volume + 6: | apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: @@ -3631,7 +3844,7 @@ renders values.slinky.partition-example.yaml: verbs: - get - update - 6: | + 7: | apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: @@ -3645,7 +3858,7 @@ renders values.slinky.partition-example.yaml: kind: ServiceAccount name: chart-ci-topograph-node-data-broker namespace: topograph - 7: | + 8: | apiVersion: v1 automountServiceAccountToken: true kind: ServiceAccount @@ -3658,7 +3871,7 @@ renders values.slinky.partition-example.yaml: helm.sh/chart: node-data-broker-0.0.0 name: chart-ci-topograph-node-data-broker namespace: topograph - 8: | + 9: | apiVersion: v1 data: node-observer-config.yaml: |- @@ -3721,7 +3934,7 @@ renders values.slinky.partition-example.yaml: helm.sh/chart: node-observer-0.0.0 name: chart-ci-topograph-node-observer namespace: topograph - 9: | + 10: | apiVersion: apps/v1 kind: Deployment metadata: @@ -3795,7 +4008,7 @@ renders values.slinky.partition-example.yaml: defaultMode: 420 name: chart-ci-topograph-node-observer name: config-volume - 10: | + 11: | apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: @@ -3815,7 +4028,7 @@ renders values.slinky.partition-example.yaml: verbs: - list - watch - 11: | + 12: | apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: @@ -3829,7 +4042,7 @@ renders values.slinky.partition-example.yaml: kind: ServiceAccount name: chart-ci-topograph-node-observer namespace: topograph - 12: | + 13: | apiVersion: v1 automountServiceAccountToken: true kind: ServiceAccount @@ -3842,7 +4055,7 @@ renders values.slinky.partition-example.yaml: helm.sh/chart: node-observer-0.0.0 name: chart-ci-topograph-node-observer namespace: topograph - 13: | + 14: | apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: @@ -3881,7 +4094,7 @@ renders values.slinky.partition-example.yaml: - create - get - update - 14: | + 15: | apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: @@ -3895,7 +4108,7 @@ renders values.slinky.partition-example.yaml: kind: ServiceAccount name: chart-ci-topograph namespace: topograph - 15: | + 16: | apiVersion: v1 kind: Service metadata: @@ -3917,7 +4130,7 @@ renders values.slinky.partition-example.yaml: app.kubernetes.io/instance: chart-ci app.kubernetes.io/name: topograph type: ClusterIP - 16: | + 17: | apiVersion: v1 automountServiceAccountToken: true kind: ServiceAccount @@ -3930,7 +4143,7 @@ renders values.slinky.partition-example.yaml: helm.sh/chart: topograph-0.0.0 name: chart-ci-topograph namespace: topograph - 17: | + 18: | apiVersion: v1 kind: Pod metadata: @@ -3973,7 +4186,7 @@ renders values.slinky.partition-example.yaml: runAsUser: 65532 seccompProfile: type: RuntimeDefault - 18: | + 19: | apiVersion: v1 kind: Pod metadata: @@ -4135,6 +4348,23 @@ renders values.slinky.tree-example.yaml: name: chart-ci-topograph name: config-volume 4: | + apiVersion: v1 + data: + node-data-broker-config.yaml: |- + provider: + name: aws + healthzPort: 8080 + kind: ConfigMap + metadata: + labels: + app.kubernetes.io/instance: chart-ci + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/name: node-data-broker + app.kubernetes.io/version: v0.0.0 + helm.sh/chart: node-data-broker-0.0.0 + name: chart-ci-topograph-node-data-broker + namespace: topograph + 5: | apiVersion: apps/v1 kind: DaemonSet metadata: @@ -4153,6 +4383,8 @@ renders values.slinky.tree-example.yaml: app.kubernetes.io/name: node-data-broker template: metadata: + annotations: + checksum/config: 51fe2be1a13f3d4bc84430f46ef0a1242139b4079e5b12304a29d359de94aa0a labels: app.kubernetes.io/instance: chart-ci app.kubernetes.io/managed-by: Helm @@ -4162,9 +4394,9 @@ renders values.slinky.tree-example.yaml: spec: containers: - args: - - --provider=aws + - -c + - /etc/topograph/node-data-broker-config.yaml - -v=3 - - --port=8080 command: - /usr/local/bin/node-data-broker env: @@ -4207,6 +4439,10 @@ renders values.slinky.tree-example.yaml: path: /healthz port: http periodSeconds: 10 + volumeMounts: + - mountPath: /etc/topograph + name: config-volume + readOnly: true securityContext: fsGroup: 65532 runAsGroup: 65532 @@ -4215,7 +4451,12 @@ renders values.slinky.tree-example.yaml: seccompProfile: type: RuntimeDefault serviceAccountName: chart-ci-topograph-node-data-broker - 5: | + volumes: + - configMap: + defaultMode: 420 + name: chart-ci-topograph-node-data-broker + name: config-volume + 6: | apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: @@ -4228,7 +4469,7 @@ renders values.slinky.tree-example.yaml: verbs: - get - update - 6: | + 7: | apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: @@ -4242,7 +4483,7 @@ renders values.slinky.tree-example.yaml: kind: ServiceAccount name: chart-ci-topograph-node-data-broker namespace: topograph - 7: | + 8: | apiVersion: v1 automountServiceAccountToken: true kind: ServiceAccount @@ -4255,7 +4496,7 @@ renders values.slinky.tree-example.yaml: helm.sh/chart: node-data-broker-0.0.0 name: chart-ci-topograph-node-data-broker namespace: topograph - 8: | + 9: | apiVersion: v1 data: node-observer-config.yaml: |- @@ -4293,7 +4534,7 @@ renders values.slinky.tree-example.yaml: helm.sh/chart: node-observer-0.0.0 name: chart-ci-topograph-node-observer namespace: topograph - 9: | + 10: | apiVersion: apps/v1 kind: Deployment metadata: @@ -4367,7 +4608,7 @@ renders values.slinky.tree-example.yaml: defaultMode: 420 name: chart-ci-topograph-node-observer name: config-volume - 10: | + 11: | apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: @@ -4387,7 +4628,7 @@ renders values.slinky.tree-example.yaml: verbs: - list - watch - 11: | + 12: | apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: @@ -4401,7 +4642,7 @@ renders values.slinky.tree-example.yaml: kind: ServiceAccount name: chart-ci-topograph-node-observer namespace: topograph - 12: | + 13: | apiVersion: v1 automountServiceAccountToken: true kind: ServiceAccount @@ -4414,7 +4655,7 @@ renders values.slinky.tree-example.yaml: helm.sh/chart: node-observer-0.0.0 name: chart-ci-topograph-node-observer namespace: topograph - 13: | + 14: | apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: @@ -4453,7 +4694,7 @@ renders values.slinky.tree-example.yaml: - create - get - update - 14: | + 15: | apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: @@ -4467,7 +4708,7 @@ renders values.slinky.tree-example.yaml: kind: ServiceAccount name: chart-ci-topograph namespace: topograph - 15: | + 16: | apiVersion: v1 kind: Service metadata: @@ -4489,7 +4730,7 @@ renders values.slinky.tree-example.yaml: app.kubernetes.io/instance: chart-ci app.kubernetes.io/name: topograph type: ClusterIP - 16: | + 17: | apiVersion: v1 automountServiceAccountToken: true kind: ServiceAccount @@ -4502,7 +4743,7 @@ renders values.slinky.tree-example.yaml: helm.sh/chart: topograph-0.0.0 name: chart-ci-topograph namespace: topograph - 17: | + 18: | apiVersion: v1 kind: Pod metadata: @@ -4545,7 +4786,7 @@ renders values.slinky.tree-example.yaml: runAsUser: 65532 seccompProfile: type: RuntimeDefault - 18: | + 19: | apiVersion: v1 kind: Pod metadata: diff --git a/charts/topograph/tests/node-data-broker_configmap_test.yaml b/charts/topograph/tests/node-data-broker_configmap_test.yaml new file mode 100644 index 00000000..24ccaf4b --- /dev/null +++ b/charts/topograph/tests/node-data-broker_configmap_test.yaml @@ -0,0 +1,104 @@ +suite: node-data-broker configmap +templates: + - templates/nodeDataBroker/configmap.yaml +release: + name: chart-ci + namespace: topograph +tests: + - it: renders provider configuration and the health port + set: + provider: + name: infiniband-k8s + params: + accelerator: + source: nvidia-smi + nvidiaSmi: + gpuOperatorNamespace: custom-gpu-operator + devicePluginDaemonSet: custom-device-plugin + nodeDataBroker: + port: 18080 + asserts: + - isKind: + of: ConfigMap + - equal: + path: metadata.name + value: chart-ci-topograph-node-data-broker + - matchRegex: + path: data["node-data-broker-config.yaml"] + pattern: "name: infiniband-k8s" + - matchRegex: + path: data["node-data-broker-config.yaml"] + pattern: "source: nvidia-smi" + - matchRegex: + path: data["node-data-broker-config.yaml"] + pattern: "gpuOperatorNamespace: custom-gpu-operator" + - matchRegex: + path: data["node-data-broker-config.yaml"] + pattern: "devicePluginDaemonSet: custom-device-plugin" + - matchRegex: + path: data["node-data-broker-config.yaml"] + pattern: "healthzPort: 18080" + + - it: defaults the nvidia-smi workload location + set: + provider: + name: infiniband-k8s + params: + accelerator: + source: nvidia-smi + asserts: + - matchRegex: + path: data["node-data-broker-config.yaml"] + pattern: "gpuOperatorNamespace: gpu-operator" + - matchRegex: + path: data["node-data-broker-config.yaml"] + pattern: "devicePluginDaemonSet: nvidia-device-plugin-daemonset" + + - it: does not render nvidia-smi defaults when accelerator is omitted + set: + provider: + name: infiniband-k8s + asserts: + - notMatchRegex: + path: data["node-data-broker-config.yaml"] + pattern: "gpuOperatorNamespace:" + - notMatchRegex: + path: data["node-data-broker-config.yaml"] + pattern: "devicePluginDaemonSet:" + + - it: does not render nvidia-smi defaults for an empty accelerator section + set: + provider: + name: infiniband-k8s + params: + accelerator: {} + asserts: + - notMatchRegex: + path: data["node-data-broker-config.yaml"] + pattern: "gpuOperatorNamespace:" + - notMatchRegex: + path: data["node-data-broker-config.yaml"] + pattern: "devicePluginDaemonSet:" + + - it: does not render nvidia-smi defaults for the none source + set: + provider: + name: infiniband-k8s + params: + accelerator: + source: none + asserts: + - notMatchRegex: + path: data["node-data-broker-config.yaml"] + pattern: "gpuOperatorNamespace:" + - notMatchRegex: + path: data["node-data-broker-config.yaml"] + pattern: "devicePluginDaemonSet:" + + - it: is omitted when node-data-broker is disabled + set: + nodeDataBroker: + enabled: false + asserts: + - hasDocuments: + count: 0 diff --git a/charts/topograph/tests/node-data-broker_rbac_test.yaml b/charts/topograph/tests/node-data-broker_rbac_test.yaml index 3efa5e85..b3a63631 100644 --- a/charts/topograph/tests/node-data-broker_rbac_test.yaml +++ b/charts/topograph/tests/node-data-broker_rbac_test.yaml @@ -71,12 +71,15 @@ tests: - hasDocuments: count: 0 - - it: grants pods/exec for infiniband-k8s nvidia-smi discovery + - it: grants pods/exec for explicitly configured infiniband-k8s nvidia-smi discovery templates: - templates/nodeDataBroker/rbac.yaml set: provider: name: infiniband-k8s + params: + accelerator: + source: nvidia-smi documentIndex: 0 asserts: - contains: @@ -110,16 +113,117 @@ tests: resources: [pods/exec] verbs: [create] - - it: drops pods/exec when reusing the GPU clique label + - it: drops nvidia-smi permissions when the accelerator section is absent + templates: + - templates/nodeDataBroker/rbac.yaml + set: + provider: + name: infiniband-k8s + documentIndex: 0 + asserts: + - notContains: + path: rules + content: + apiGroups: [apps] + resources: [daemonsets] + verbs: [get] + - notContains: + path: rules + content: + apiGroups: [""] + resources: [pods] + verbs: [list] + - notContains: + path: rules + content: + apiGroups: [""] + resources: [pods/exec] + verbs: [create] + + - it: drops nvidia-smi permissions when using the Kubernetes label accelerator source templates: - templates/nodeDataBroker/rbac.yaml set: provider: name: infiniband-k8s params: - useGpuCliqueLabel: true + accelerator: + source: kubernetes-label + kubernetesLabel: + key: nvidia.com/gpu.clique documentIndex: 0 asserts: + - notContains: + path: rules + content: + apiGroups: [apps] + resources: [daemonsets] + verbs: [get] + - notContains: + path: rules + content: + apiGroups: [""] + resources: [pods] + verbs: [list] + - notContains: + path: rules + content: + apiGroups: [""] + resources: [pods/exec] + verbs: [create] + + - it: drops nvidia-smi permissions when accelerator discovery is explicitly disabled + templates: + - templates/nodeDataBroker/rbac.yaml + set: + provider: + name: infiniband-k8s + params: + accelerator: + source: none + documentIndex: 0 + asserts: + - notContains: + path: rules + content: + apiGroups: [apps] + resources: [daemonsets] + verbs: [get] + - notContains: + path: rules + content: + apiGroups: [""] + resources: [pods] + verbs: [list] + - notContains: + path: rules + content: + apiGroups: [""] + resources: [pods/exec] + verbs: [create] + + - it: drops nvidia-smi permissions for an empty accelerator section + templates: + - templates/nodeDataBroker/rbac.yaml + set: + provider: + name: infiniband-k8s + params: + accelerator: {} + documentIndex: 0 + asserts: + - notContains: + path: rules + content: + apiGroups: [apps] + resources: [daemonsets] + verbs: [get] + - notContains: + path: rules + content: + apiGroups: [""] + resources: [pods] + verbs: [list] - notContains: path: rules content: diff --git a/charts/topograph/tests/node-data-broker_test.yaml b/charts/topograph/tests/node-data-broker_test.yaml index 94c97780..1389cca7 100644 --- a/charts/topograph/tests/node-data-broker_test.yaml +++ b/charts/topograph/tests/node-data-broker_test.yaml @@ -77,7 +77,7 @@ tests: path: spec.template.spec.serviceAccountName value: existing-node-data-broker-sa - - it: passes the provider and NODE_NAME to the container + - it: mounts its config and passes its path with NODE_NAME templates: - templates/nodeDataBroker/daemonset.yaml set: @@ -103,7 +103,26 @@ tests: asserts: - contains: path: spec.template.spec.containers[0].args - content: --provider=infiniband-k8s + content: -c + - contains: + path: spec.template.spec.containers[0].args + content: /etc/topograph/node-data-broker-config.yaml + - contains: + path: spec.template.spec.containers[0].volumeMounts + content: + name: config-volume + mountPath: /etc/topograph + readOnly: true + - contains: + path: spec.template.spec.volumes + content: + name: config-volume + configMap: + defaultMode: 420 + name: chart-ci-topograph-node-data-broker + - matchRegex: + path: spec.template.metadata.annotations.checksum/config + pattern: "^[a-f0-9]{64}$" - contains: path: spec.template.spec.containers[0].env content: @@ -140,25 +159,6 @@ tests: path: spec.template.spec.containers[0].livenessProbe.httpGet.path value: /healthz - - it: forwards the GPU clique label flag and extra args - templates: - - templates/nodeDataBroker/daemonset.yaml - set: - provider: - name: infiniband-k8s - params: - useGpuCliqueLabel: true - nodeDataBroker: - extraArgs: - - foo=bar - asserts: - - contains: - path: spec.template.spec.containers[0].args - content: --set=useGpuCliqueLabel=true - - contains: - path: spec.template.spec.containers[0].args - content: --set=foo=bar - - it: is omitted when the component is disabled templates: - templates/nodeDataBroker/daemonset.yaml diff --git a/charts/topograph/tests/validation_test.yaml b/charts/topograph/tests/validation_test.yaml index a6511d5c..89437512 100644 --- a/charts/topograph/tests/validation_test.yaml +++ b/charts/topograph/tests/validation_test.yaml @@ -76,3 +76,37 @@ tests: asserts: - failedTemplate: errorMessage: "env.KUBE_BURST is managed by the chart; configure kubeClient.burst instead" + + - it: rejects an unsupported InfiniBand accelerator source + set: + provider: + name: infiniband-k8s + params: + accelerator: + source: invalid + asserts: + - failedTemplate: + errorMessage: 'unsupported provider.params.accelerator.source "invalid"' + + - it: rejects a non-empty InfiniBand accelerator section without source + set: + provider: + name: infiniband-k8s + params: + accelerator: + kubernetesLabel: + key: nvidia.com/gpu.clique + asserts: + - failedTemplate: + errorMessage: "provider.params.accelerator.source must be set when provider.params.accelerator is present" + + - it: rejects the Kubernetes label accelerator source without a key + set: + provider: + name: infiniband-k8s + params: + accelerator: + source: kubernetes-label + asserts: + - failedTemplate: + errorMessage: "provider.params.accelerator.kubernetesLabel.key must be set for source kubernetes-label" diff --git a/charts/topograph/values.k8s.ib-example.yaml b/charts/topograph/values.k8s.ib-example.yaml index 3d2c6326..4c558431 100644 --- a/charts/topograph/values.k8s.ib-example.yaml +++ b/charts/topograph/values.k8s.ib-example.yaml @@ -1,7 +1,10 @@ provider: name: infiniband-k8s params: - useGpuCliqueLabel: true + accelerator: + source: kubernetes-label + kubernetesLabel: + key: nvidia.com/gpu.clique engine: name: k8s diff --git a/charts/topograph/values.schema.json b/charts/topograph/values.schema.json index 033666ea..ee371e11 100644 --- a/charts/topograph/values.schema.json +++ b/charts/topograph/values.schema.json @@ -243,7 +243,6 @@ "properties": { "enabled": { "type": "boolean" }, "port": { "type": "integer", "minimum": 1, "maximum": 65535 }, - "extraArgs": { "type": "array", "items": { "type": "string" } }, "startupProbe": { "type": "object" }, "serviceAccount": { "type": "object" }, "rbac": { "type": "object" }, diff --git a/charts/topograph/values.slinky.ib.block-example.yaml b/charts/topograph/values.slinky.ib.block-example.yaml index 9b8fca43..d538800c 100644 --- a/charts/topograph/values.slinky.ib.block-example.yaml +++ b/charts/topograph/values.slinky.ib.block-example.yaml @@ -12,10 +12,12 @@ provider: # so Topograph only discovers nodes that Slinky can schedule. nodeSelector: slurmCluster: my-cluster - # Reads accelerator/block domain IDs from the GPU Operator's - # nvidia.com/gpu.clique node label instead of Topograph's - # topograph.nvidia.com/cluster-id annotation. - useGpuCliqueLabel: true + # Accelerator-domain discovery is independent of InfiniBand fabric + # discovery. This source reads the GPU Operator's gpu.clique label. + accelerator: + source: kubernetes-label + kubernetesLabel: + key: nvidia.com/gpu.clique engine: # Writes generated Slurm topology data into a Slinky ConfigMap. name: slinky @@ -38,7 +40,7 @@ engine: # be larger power-of-two multiples of the previous size. blockSizes: [4] # For block topology, use nvidia.com/gpu.clique as the block-domain - # source. Keep this aligned with provider.params.useGpuCliqueLabel. + # source. Keep this aligned with provider.params.accelerator. useGpuCliqueLabel: true # Key inside the target ConfigMap that Slinky mounts as topology.conf. topologyConfigPath: topology.conf diff --git a/charts/topograph/values.yaml b/charts/topograph/values.yaml index 96d0ab0e..b5cbf4d3 100644 --- a/charts/topograph/values.yaml +++ b/charts/topograph/values.yaml @@ -6,10 +6,21 @@ provider: # name: "aws", "oci", "gcp", "nebius", "nscale", "lambdai", "netq", "infiniband-k8s", "dra" or "test". name: test # params: - # # For infiniband-k8s, use an existing Kubernetes node label as the - # # accelerator-domain source instead of running nvidia-smi through the - # # GPU Operator device-plugin DaemonSet. - # useGpuCliqueLabel: true + # accelerator: + # # Accelerator-domain discovery is independent of fabric discovery. + # # Sources supported by infiniband-k8s: nvidia-smi, + # # kubernetes-label, and none. Omitting the accelerator section disables + # # accelerator discovery; an empty accelerator section is equivalent to + # # source: none. A non-empty section must set source explicitly. + # source: kubernetes-label + # kubernetesLabel: + # # Required for the kubernetes-label source; there is no default key. + # key: nvidia.com/gpu.clique + # nvidiaSmi: + # # Optional; defaults used by the Helm-managed node-data-broker are + # # shown below. + # gpuOperatorNamespace: gpu-operator + # devicePluginDaemonSet: nvidia-device-plugin-daemonset engine: # name: "k8s", "nfd", "slinky", "slurm" or "graph" @@ -307,9 +318,6 @@ nodeDataBroker: # Port serving the /healthz endpoint after node annotations are applied. port: 8080 - extraArgs: [] - # - key=val - startupProbe: failureThreshold: 30 periodSeconds: 10 diff --git a/cmd/node-data-broker/main.go b/cmd/node-data-broker/main.go index c71fb80b..b9f3cf89 100644 --- a/cmd/node-data-broker/main.go +++ b/cmd/node-data-broker/main.go @@ -14,7 +14,6 @@ import ( "net/http" "os" "os/signal" - "strings" "syscall" "time" @@ -24,8 +23,10 @@ import ( "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" "k8s.io/klog/v2" + "sigs.k8s.io/yaml" "github.com/NVIDIA/topograph/internal/version" + "github.com/NVIDIA/topograph/pkg/accelerator" "github.com/NVIDIA/topograph/pkg/providers/aws" "github.com/NVIDIA/topograph/pkg/providers/dra" "github.com/NVIDIA/topograph/pkg/providers/gcp" @@ -33,31 +34,32 @@ import ( "github.com/NVIDIA/topograph/pkg/providers/lambdai" "github.com/NVIDIA/topograph/pkg/providers/nebius" "github.com/NVIDIA/topograph/pkg/providers/oci" + "github.com/NVIDIA/topograph/pkg/topology" ) const ( - defaultPort = 8080 + defaultConfigPath = "/etc/topograph/node-data-broker-config.yaml" readHeaderTimeout = 5 * time.Second shutdownTimeout = 5 * time.Second ) type nodeBroker struct { - clientset kubernetes.Interface - config *rest.Config - provider string - sets []string - nodeName string + clientset kubernetes.Interface + restConfig *rest.Config + config nodeDataBrokerConfig + nodeName string +} + +type nodeDataBrokerConfig struct { + Provider topology.Provider `yaml:"provider"` + HealthzPort int `yaml:"healthzPort"` } func main() { - var provider string var ver bool - var sets []string - var port int - pflag.StringVar(&provider, "provider", "", "API provider") + var configPath string pflag.BoolVar(&ver, "version", false, "show the version") - pflag.StringArrayVar(&sets, "set", []string{}, "extra key=value parameters") - pflag.IntVar(&port, "port", defaultPort, "port for the health HTTP server") + pflag.StringVarP(&configPath, "config", "c", defaultConfigPath, "config file") klog.InitFlags(nil) pflag.CommandLine.AddGoFlagSet(flag.CommandLine) @@ -69,13 +71,36 @@ func main() { os.Exit(0) } - if err := mainInternal(provider, sets, port); err != nil { + config, err := newNodeDataBrokerConfig(configPath) + if err != nil { + klog.Error(err.Error()) + os.Exit(1) + } + if err := mainInternal(config); err != nil { klog.Error(err.Error()) os.Exit(1) } } -func mainInternal(provider string, sets []string, port int) error { +func newNodeDataBrokerConfig(path string) (nodeDataBrokerConfig, error) { + data, err := os.ReadFile(path) + if err != nil { + return nodeDataBrokerConfig{}, fmt.Errorf("failed to read node-data-broker config %q: %w", path, err) + } + var config nodeDataBrokerConfig + if err := yaml.Unmarshal(data, &config); err != nil { + return nodeDataBrokerConfig{}, fmt.Errorf("failed to decode node-data-broker config %q: %w", path, err) + } + if config.Provider.Name == "" { + return nodeDataBrokerConfig{}, fmt.Errorf("must specify provider.name") + } + if config.HealthzPort <= 0 { + return nodeDataBrokerConfig{}, fmt.Errorf("must specify a positive healthzPort") + } + return config, nil +} + +func mainInternal(brokerConfig nodeDataBrokerConfig) error { ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) defer stop() @@ -85,11 +110,10 @@ func mainInternal(provider string, sets []string, port int) error { } broker := &nodeBroker{ - clientset: clientset, - config: config, - provider: provider, - sets: sets, - nodeName: os.Getenv("NODE_NAME"), + clientset: clientset, + restConfig: config, + config: brokerConfig, + nodeName: os.Getenv("NODE_NAME"), } if err := broker.apply(ctx); err != nil { @@ -98,7 +122,7 @@ func mainInternal(provider string, sets []string, port int) error { // Keep the DaemonSet pod Running by serving a health endpoint until the pod // is terminated. - return serveHealth(ctx, port) + return serveHealth(ctx, brokerConfig.HealthzPort) } func newInClusterClientset() (kubernetes.Interface, *rest.Config, error) { @@ -116,18 +140,13 @@ func newInClusterClientset() (kubernetes.Interface, *rest.Config, error) { } func (b *nodeBroker) apply(ctx context.Context) error { - klog.InfoS("Applying node annotations", "provider", b.provider, "extras", b.sets) - - extras, err := getExtras(b.sets) - if err != nil { - return err - } + klog.InfoS("Applying node annotations", "provider", b.config.Provider.Name) - annotations, err := getAnnotations(ctx, b.clientset, b.config, b.provider, b.nodeName, extras) + annotations, err := b.getAnnotations(ctx) if err != nil { return err } - klog.Infof("adding annotations %v in node %s for provider %s", annotations, b.nodeName, b.provider) + klog.Infof("adding annotations %v in node %s for provider %s", annotations, b.nodeName, b.config.Provider.Name) node, err := b.clientset.CoreV1().Nodes().Get(ctx, b.nodeName, metav1.GetOptions{}) if err != nil { @@ -182,26 +201,8 @@ func healthHandler() http.Handler { return mux } -func getExtras(sets []string) (map[string]string, error) { - extras := make(map[string]string) - for _, kv := range sets { - parts := strings.SplitN(kv, "=", 2) - if len(parts) == 2 { - key, val := parts[0], parts[1] - if len(key) == 0 || len(val) == 0 { - return nil, fmt.Errorf("invalid value %q for '--set': key/value cannot be empty", kv) - } - extras[key] = val - } else { - return nil, fmt.Errorf("invalid value %q for '--set': expected format '='", kv) - } - } - - return extras, nil -} - -func getAnnotations(ctx context.Context, client kubernetes.Interface, config *rest.Config, provider, nodeName string, extras map[string]string) (map[string]string, error) { - switch provider { +func (b *nodeBroker) getAnnotations(ctx context.Context) (map[string]string, error) { + switch b.config.Provider.Name { case aws.NAME: return aws.GetNodeAnnotations(ctx) case gcp.NAME: @@ -211,15 +212,16 @@ func getAnnotations(ctx context.Context, client kubernetes.Interface, config *re case nebius.NAME: return nebius.GetNodeAnnotations(ctx) case dra.NAME: - return dra.GetNodeAnnotations(ctx, nodeName) + return dra.GetNodeAnnotations(ctx, b.nodeName) case infiniband.NAME_K8S: - return infiniband.GetNodeAnnotations(ctx, client, config, nodeName, extras) + section := accelerator.SectionFromProviderParams(b.config.Provider.Params) + return infiniband.GetNodeAnnotations(ctx, b.clientset, b.restConfig, b.nodeName, section) case lambdai.NAME: - return lambdai.GetNodeAnnotations(ctx, client, nodeName) + return lambdai.GetNodeAnnotations(ctx, b.clientset, b.nodeName) case "": return nil, fmt.Errorf("must set provider") default: - return nil, fmt.Errorf("unsupported provider %q", provider) + return nil, fmt.Errorf("unsupported provider %q", b.config.Provider.Name) } } diff --git a/cmd/node-data-broker/main_test.go b/cmd/node-data-broker/main_test.go index 70c7ace7..950b2736 100644 --- a/cmd/node-data-broker/main_test.go +++ b/cmd/node-data-broker/main_test.go @@ -10,65 +10,18 @@ import ( "io" "net/http" "net/http/httptest" + "os" + "path/filepath" "testing" "time" "github.com/stretchr/testify/require" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" -) - -func TestGetExtras(t *testing.T) { - tests := []struct { - name string - sets []string - extras map[string]string - err string - }{ - { - name: "Case 1: empty input", - sets: []string{}, - extras: map[string]string{}, - }, - { - name: "Case 2: single valid key=value", - sets: []string{"a=b"}, - extras: map[string]string{"a": "b"}, - }, - { - name: "Case 3: multiple valid key=value", - sets: []string{"a=b", "c=d"}, - extras: map[string]string{"a": "b", "c": "d"}, - }, - { - name: "Case 4: invalid format", - sets: []string{"foo"}, - err: `invalid value "foo" for '--set': expected format '='`, - }, - { - name: "Case 5: empty key", - sets: []string{"=bar"}, - err: `invalid value "=bar" for '--set': key/value cannot be empty`, - }, - { - name: "Case 6: empty value", - sets: []string{"foo="}, - err: `invalid value "foo=" for '--set': key/value cannot be empty`, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - extras, err := getExtras(tt.sets) - if len(tt.err) != 0 { - require.EqualError(t, err, tt.err) - } else { - require.NoError(t, err) - require.Equal(t, tt.extras, extras) - } - }) - } -} + "github.com/NVIDIA/topograph/pkg/providers/infiniband" + "github.com/NVIDIA/topograph/pkg/topology" +) func TestGetAnnotations(t *testing.T) { ctx := context.TODO() @@ -90,10 +43,107 @@ func TestGetAnnotations(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - _, err := getAnnotations(ctx, nil, nil, tt.provider, "", nil) + broker := &nodeBroker{ + config: nodeDataBrokerConfig{Provider: topology.Provider{Name: tt.provider}}, + } + _, err := broker.getAnnotations(ctx) require.EqualError(t, err, tt.err) }) } + + t.Run("invalid accelerator section", func(t *testing.T) { + broker := &nodeBroker{ + config: nodeDataBrokerConfig{ + Provider: topology.Provider{ + Name: infiniband.NAME_K8S, + Params: map[string]any{ + "accelerator": "invalid", + }, + }, + }, + } + _, err := broker.getAnnotations(ctx) + require.EqualError(t, err, "accelerator section must be an object with a source") + }) + + t.Run("null accelerator section", func(t *testing.T) { + broker := &nodeBroker{ + config: nodeDataBrokerConfig{ + Provider: topology.Provider{ + Name: infiniband.NAME_K8S, + Params: map[string]any{ + "accelerator": nil, + }, + }, + }, + } + _, err := broker.getAnnotations(ctx) + require.EqualError(t, err, "accelerator section must be an object with a source") + }) + + t.Run("empty accelerator section disables discovery", func(t *testing.T) { + broker := &nodeBroker{ + nodeName: "node-1", + config: nodeDataBrokerConfig{ + Provider: topology.Provider{ + Name: infiniband.NAME_K8S, + Params: map[string]any{ + "accelerator": map[string]any{}, + }, + }, + }, + } + annotations, err := broker.getAnnotations(ctx) + require.NoError(t, err) + require.Equal(t, map[string]string{ + topology.KeyNodeInstance: "node-1", + topology.KeyNodeRegion: "local", + }, annotations) + }) +} + +func TestNewNodeDataBrokerConfig(t *testing.T) { + dir := t.TempDir() + configPath := filepath.Join(dir, "node-data-broker-config.yaml") + require.NoError(t, os.WriteFile(configPath, []byte(` +provider: + name: infiniband-k8s + params: + accelerator: + source: none +healthzPort: 18080 +`), 0o600)) + + config, err := newNodeDataBrokerConfig(configPath) + require.NoError(t, err) + require.Equal(t, infiniband.NAME_K8S, config.Provider.Name) + require.Equal(t, map[string]any{ + "accelerator": map[string]any{"source": "none"}, + }, config.Provider.Params) + require.Equal(t, 18080, config.HealthzPort) + + _, err = newNodeDataBrokerConfig(filepath.Join(dir, "missing.yaml")) + require.ErrorContains(t, err, "failed to read node-data-broker config") + + invalidPath := filepath.Join(dir, "invalid.yaml") + require.NoError(t, os.WriteFile(invalidPath, []byte("provider: ["), 0o600)) + _, err = newNodeDataBrokerConfig(invalidPath) + require.ErrorContains(t, err, "failed to decode node-data-broker config") + + missingProviderPath := filepath.Join(dir, "missing-provider.yaml") + require.NoError(t, os.WriteFile(missingProviderPath, []byte("healthzPort: 8080"), 0o600)) + _, err = newNodeDataBrokerConfig(missingProviderPath) + require.EqualError(t, err, "must specify provider.name") + + missingPortPath := filepath.Join(dir, "missing-port.yaml") + require.NoError(t, os.WriteFile(missingPortPath, []byte("provider:\n name: test"), 0o600)) + _, err = newNodeDataBrokerConfig(missingPortPath) + require.EqualError(t, err, "must specify a positive healthzPort") + + negativePortPath := filepath.Join(dir, "negative-port.yaml") + require.NoError(t, os.WriteFile(negativePortPath, []byte("provider:\n name: test\nhealthzPort: -1"), 0o600)) + _, err = newNodeDataBrokerConfig(negativePortPath) + require.EqualError(t, err, "must specify a positive healthzPort") } func TestMergeNodeAnnotations(t *testing.T) { diff --git a/docs/api.md b/docs/api.md index ddf8453b..66024df4 100644 --- a/docs/api.md +++ b/docs/api.md @@ -72,7 +72,10 @@ Topograph exposes three endpoints for interacting with the service. Below are th - **name**: (optional) A string specifying the Service Provider, such as `aws`, `oci`, `gcp`, `nebius`, `nscale`, `netq`, `dra`, `infiniband-k8s`, `infiniband-bm` or `test`. This parameter will override the provider set in the topograph config. - **creds**: (optional) A key-value map with provider-specific parameters for authentication. - **params**: (optional) A key-value map with provider-specific parameters. The `test` provider uses these parameters for response simulation; for complete behavior and examples, see [Test Mode and Test Provider](./providers/test.md). - - **useGpuCliqueLabel**: (optional) Used in: [`infiniband-k8s`]. If `true`, reads the GPU Operator's `nvidia.com/gpu.clique` node label as the accelerator-domain source instead of using the `topograph.nvidia.com/cluster-id` node annotation. + - **accelerator**: (optional) Used in: [`infiniband-k8s`, `infiniband-bm`]. Configures accelerator-domain discovery independently of network-fabric discovery. Omitting this section or setting it to an empty object disables accelerator-domain discovery. + - **source**: (required when `accelerator` is non-empty) `nvidia-smi`, `kubernetes-label` (`infiniband-k8s` only), or `none`. An empty object is equivalent to `source: none`. + - **kubernetesLabel.key**: (required for `kubernetes-label`) Kubernetes Node label read as the accelerator-domain ID. No default is assumed. + - For `infiniband-k8s`, a request with `source: nvidia-smi` reads accelerator-domain annotations previously collected by the node-data-broker. The request does not run `nvidia-smi` or reconfigure the broker. Deploy the broker with the same accelerator source before sending the request; see [Helm node-data-broker settings](./providers/infiniband.md#helm-node-data-broker-settings). - **engine**: (optional) Selects the topology output and provides any engine-specific parameters. - **name**: (optional) A string specifying the topology output, either `slurm`, `k8s`, `nfd`, `slinky`, or `graph`. This parameter will override the engine set in the topograph config. - **params**: (optional) A key-value map with engine-specific parameters. diff --git a/docs/architecture.md b/docs/architecture.md index 72800b63..6c4716a3 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -26,7 +26,7 @@ The Node Data Broker is also used when Topograph is deployed in a Kubernetes clu ### 4. Provider -The Provider interfaces with CSPs or on-premises tools to retrieve topology-related data from the cluster and converts it into an internal representation. +The Provider interfaces with CSPs or on-premises tools to retrieve topology-related data from the cluster and converts it into an internal representation. Providers may compose network-fabric discovery with an independent accelerator-domain source; for example, the InfiniBand provider can combine `ibnetdiscover` with `nvidia-smi`, a Kubernetes Node label, or no accelerator source. ### 5. Engine diff --git a/docs/engines/k8s.md b/docs/engines/k8s.md index 222ca202..a22a0470 100644 --- a/docs/engines/k8s.md +++ b/docs/engines/k8s.md @@ -77,7 +77,7 @@ Topograph treats `nvidia.com/gpu.clique` as the authoritative accelerator node l In addition to NVLink domain membership, Topograph provides the full IB switch hierarchy as numbered fabric tiers, giving schedulers both dimensions simultaneously. -For `infiniband-k8s`, operators can set `provider.params.useGpuCliqueLabel: true` so the provider reads the GPU Operator's existing clique label instead of collecting the same value through a `nvidia-smi` exec in the GPU Operator device-plugin DaemonSet. +For `infiniband-k8s`, operators can set `provider.params.accelerator.source: kubernetes-label` with `kubernetesLabel.key: nvidia.com/gpu.clique` so the provider reads the existing label instead of collecting the same value through a `nvidia-smi` exec in the GPU Operator device-plugin DaemonSet. ## Use of Topograph diff --git a/docs/providers/infiniband.md b/docs/providers/infiniband.md index ff411c6c..7ed53ad8 100644 --- a/docs/providers/infiniband.md +++ b/docs/providers/infiniband.md @@ -17,7 +17,7 @@ For **Multi-Node NVLink (MNNVL) Kubernetes clusters** (e.g. GB200 NVL72), do not |---|---|---| | **Auth** | None | In-cluster service account | | **Node access** | `pdsh` (SSH-based) | Kubernetes pod exec | -| **NVLink clique source** | `nvidia-smi` via pdsh | Node annotations (set by node-data-broker), or a configured Kubernetes node label | +| **Accelerator-domain source** | Configurable: `nvidia-smi` via pdsh, or none | Configurable: `nvidia-smi`, Kubernetes Node label, or none | | **Target environment** | Bare-metal / Slurm | Kubernetes | Both variants are presently single-region only (multi-region requests return a `400 Bad Request` error). No CSP credentials are required. @@ -41,17 +41,17 @@ See the engine documentation (`docs/engines/`) for details on each output format - `pdsh` must be installed on the node running Topograph and able to reach at least one node per IB fabric segment — Topograph discovers the full fabric from a single entry point per segment, so every node does not need to be reachable via pdsh - `ibnetdiscover` must be available on cluster nodes (invoked via `pdsh` with `sudo`) — part of the standard `infiniband-diags` package (`dnf install infiniband-diags` / `apt install infiniband-diags`), expected to already be present on any properly configured IB system -- NVIDIA GPU driver required on nodes with NVLink-connected GPUs — used to collect NVLink clique IDs via `nvidia-smi`. Nodes without NVLink are included in the IB switch tree but excluded from block topology. +- NVIDIA GPU driver required on nodes with NVLink-connected GPUs when `accelerator.source: nvidia-smi` is configured. Nodes without accelerator-domain data are still included in the IB switch tree. ### How It Works 1. Runs `sudo ibnetdiscover` via `pdsh` on one node per IB fabric segment to map the full switch tree -2. On NVIDIA GPU nodes: runs `nvidia-smi -q | grep "ClusterUUID\|CliqueId" | sort -u` via `pdsh` across all nodes to collect NVLink clique IDs. The resulting `accelerator` label value is `ClusterUUID.CliqueId` — the same format as `nvidia.com/gpu.clique` set by the GPU Operator device plugin on MNNVL systems. +2. When the `accelerator` section selects `nvidia-smi`, runs `nvidia-smi --query-gpu=fabric.clusterUuid,fabric.cliqueId --format=csv,noheader` via `pdsh` across all nodes to collect NVLink partition IDs. Identical rows returned for multiple GPUs are merged, unavailable (`N/A`) fields are rejected, and the CSV pair is normalized to `ClusterUUID.CliqueId` — the same format as `nvidia.com/gpu.clique` set by the GPU Operator device plugin on MNNVL systems. 3. Combines the switch tree and any NVLink clique data into the topology graph ### Configuration -No credentials or parameters are required. Set `provider: infiniband-bm` in your Topograph config: +No credentials are required. Set `provider: infiniband-bm` in your Topograph config: ```yaml http: @@ -62,6 +62,18 @@ provider: infiniband-bm engine: slurm ``` +Accelerator discovery is independent of InfiniBand fabric discovery and is +disabled when the `accelerator` section is omitted or empty. Bare-metal +deployments support `nvidia-smi` or `none`: + +```yaml +provider: + name: infiniband-bm + params: + accelerator: + source: none +``` + ### Verifying the Output After triggering topology generation, query the result endpoint: @@ -79,14 +91,14 @@ For the Slurm engine, verify the generated `topology.conf` reflects the expected ### Prerequisites -- Topograph deployed via Helm — the node-data-broker DaemonSet (a component of the main Topograph chart, enabled by default) collects NVLink clique IDs from each node and stores them as Kubernetes node annotations (`topograph.nvidia.com/cluster-id`). If `useGpuCliqueLabel` is enabled, Topograph reads `nvidia.com/gpu.clique` directly instead and the node-data-broker skips NVLink clique collection. +- Topograph deployed via Helm — when `accelerator.source` is `nvidia-smi`, the node-data-broker DaemonSet collects NVLink partition IDs from each node and stores them as Kubernetes node annotations (`topograph.nvidia.com/cluster-id`). With `kubernetes-label` or `none`, the broker skips that collection. - The default **`ghcr.io/nvidia/topograph`** image includes **`ibnetdiscover`** (Alpine `rdma-core`). No separate InfiniBand image is required. IB deployments typically run the broker **privileged** and mount host **`/sys/class`** so `ibnetdiscover` can reach IB devices — see [`values.k8s.ib-example.yaml`](../../charts/topograph/values.k8s.ib-example.yaml). - NVIDIA GPU Operator — standard on NVIDIA GPU Kubernetes clusters; manages the device plugin DaemonSet used to read NVLink clique IDs. Required only for NVLink domain discovery; on clusters without NVLink-connected GPUs this does not apply and the provider will still discover the IB switch tree. ### How It Works 1. Runs `ibnetdiscover` by exec-ing into a node-data-broker pod on each node to map the switch tree -2. On NVIDIA GPU nodes: reads NVLink clique IDs from the `topograph.nvidia.com/cluster-id` node annotations set by the node-data-broker. If `useGpuCliqueLabel` is enabled, it reads `nvidia.com/gpu.clique` directly instead. The accelerator ID is `ClusterUUID.CliqueId` — the same format as `nvidia.com/gpu.clique` set by the GPU Operator device plugin on MNNVL systems. When the k8s engine sees `nvidia.com/gpu.clique` already present on a node, it does not write the duplicate accelerator label for that node. +2. When `provider.params.accelerator` is non-empty, resolves accelerator domains independently using its `source`: `nvidia-smi` reads the broker-written `topograph.nvidia.com/cluster-id` annotation, `kubernetes-label` reads a configured Node label, and `none` explicitly disables accelerator discovery. Omitting the section or using an empty object also disables it. NVL partition IDs use `ClusterUUID.CliqueId`, the same format as `nvidia.com/gpu.clique`. 3. Combines the switch tree and any NVLink clique data into the topology graph ### Configuration @@ -106,31 +118,53 @@ engine: k8s ### Parameters -The following optional parameter can be passed in the topology request payload: +#### Topology request parameters + +The following optional parameters can be passed in the topology request payload: | Parameter | Type | Default | Description | |---|---|---|---| | `nodeSelector` | `map[string]string` | — | Label selector to filter which nodes participate in topology discovery | -| `useGpuCliqueLabel` | `bool` | `false` | Use `nvidia.com/gpu.clique` as the accelerator-domain ID source instead of the `topograph.nvidia.com/cluster-id` annotation. | +| `accelerator` | `object` | — | Enables and configures accelerator-domain discovery. When omitted or empty, no accelerator domains are discovered. | +| `accelerator.source` | `string` | — | Required when the `accelerator` section is non-empty. Accelerator-domain source: `nvidia-smi`, `kubernetes-label`, or `none`. | +| `accelerator.kubernetesLabel.key` | `string` | — | Required for the `kubernetes-label` source. Kubernetes Node label read as the accelerator-domain ID; no default is assumed. | + +For a manual request, keep `accelerator.source` consistent with the source configured for the deployed node-data-broker. In particular, `source: nvidia-smi` reads the broker-written `topograph.nvidia.com/cluster-id` annotation; it does not execute `nvidia-smi` or change the broker's GPU Operator workload target. -With Helm, configure `useGpuCliqueLabel` under `provider.params`. The chart also passes it to the node-data-broker container so it skips NVLink clique collection instead of exec-ing into the GPU Operator device-plugin DaemonSet to run `nvidia-smi`: +#### Helm node-data-broker settings + +The following settings select the GPU Operator workload used by the node-data-broker when `provider.params.accelerator.source` is `nvidia-smi`. They are deployment settings and cannot be changed by a topology request: + +| Helm value | Type | Default | Description | +|---|---|---|---| +| `provider.params.accelerator.nvidiaSmi.gpuOperatorNamespace` | `string` | `gpu-operator` | Namespace containing the GPU Operator device-plugin DaemonSet. | +| `provider.params.accelerator.nvidiaSmi.devicePluginDaemonSet` | `string` | `nvidia-device-plugin-daemonset` | Device-plugin DaemonSet used for `nvidia-smi` execution. | + +With Helm, configure the accelerator source under `provider.params`. The chart writes the provider configuration to both the node-data-broker and node-observer ConfigMaps, keeping chart-generated topology requests aligned with broker collection. When the source is `nvidia-smi`, omitted `gpuOperatorNamespace` and `devicePluginDaemonSet` values are rendered as `gpu-operator` and `nvidia-device-plugin-daemonset`, respectively. The chart also drops the broker's GPU Operator pod-exec permissions when the source is not `nvidia-smi`: ```yaml provider: name: infiniband-k8s params: - useGpuCliqueLabel: true + accelerator: + source: kubernetes-label + kubernetesLabel: + key: nvidia.com/gpu.clique engine: name: k8s ``` -When `useGpuCliqueLabel` is not set, the node-data-broker uses the GPU Operator device-plugin DaemonSet as before. To override the GPU Operator namespace or device plugin DaemonSet name (defaults: `gpu-operator` and `nvidia-device-plugin-daemonset`), set these via `nodeDataBroker.extraArgs` in your Helm values — they are node-data-broker arguments, not provider request parameters: +To use a non-default GPU Operator namespace or device-plugin DaemonSet: ```yaml -nodeDataBroker: - extraArgs: - - gpu-operator-namespace=my-namespace - - device-plugin-daemonset=my-daemonset +provider: + name: infiniband-k8s + params: + accelerator: + source: nvidia-smi + nvidiaSmi: + gpuOperatorNamespace: my-namespace + devicePluginDaemonSet: my-daemonset ``` The node-data-broker applies node annotations once when its pod starts. Restart the broker pod to re-apply them after relevant node or provider metadata changes. diff --git a/docs/reference/node-labels.md b/docs/reference/node-labels.md index 72660e7c..5f72c571 100644 --- a/docs/reference/node-labels.md +++ b/docs/reference/node-labels.md @@ -40,8 +40,8 @@ types: | `nebius` | No | Yes | | `nscale` | Yes | Yes | | `netq` | Yes (NMX `DomainUUID`) | Yes (Spectrum-X switch hierarchy) | -| `infiniband-bm` | Yes (`ClusterUUID.CliqueId`) | Yes (IB switch hierarchy) | -| `infiniband-k8s` | Yes (`ClusterUUID.CliqueId`) | Yes (IB switch hierarchy) | +| `infiniband-bm` | Optional (`ClusterUUID.CliqueId` when configured) | Yes (IB switch hierarchy) | +| `infiniband-k8s` | Optional (`ClusterUUID.CliqueId` when configured) | Yes (IB switch hierarchy) | The OCI API provider can publish both accelerator hierarchy levels when `additionalData.locationDetails.rack` is available. Other providers currently @@ -51,7 +51,7 @@ The DRA provider is intentionally omitted: its supported use is with the Slinky engine, where it converts existing `nvidia.com/gpu.clique` labels into Slurm `topology/block` domains rather than writing Kubernetes topology labels. -**Relationship to `nvidia.com/gpu.clique`**: Some GPU Operator deployments expose `nvidia.com/gpu.clique` on nodes with Multi-Node NVLink (MNNVL) GPUs; it is not guaranteed to be present on every MNNVL cluster. The k8s engine treats that label as authoritative when present and does not write Topograph's configured accelerator domain or sub-domain labels for that node, regardless of whether the selected provider also returned accelerator topology from API data. For Slinky block topology, setting `engine.params.useGpuCliqueLabel: true` makes the Slinky engine build `topology/block` domains from `nvidia.com/gpu.clique` instead of provider accelerator-domain data. For `infiniband-k8s`, setting `provider.params.useGpuCliqueLabel: true` also makes the provider read that existing node label instead of collecting the same value through `nvidia-smi`. The `netq` provider uses a `DomainUUID` from the NMX management API — a different identifier that refers to the same physical domain but cannot be compared as a string. +**Relationship to `nvidia.com/gpu.clique`**: Some GPU Operator deployments expose `nvidia.com/gpu.clique` on nodes with Multi-Node NVLink (MNNVL) GPUs; it is not guaranteed to be present on every MNNVL cluster. The k8s engine treats that label as authoritative when present and does not write Topograph's configured accelerator domain or sub-domain labels for that node, regardless of whether the selected provider also returned accelerator topology from API data. For Slinky block topology, setting `engine.params.useGpuCliqueLabel: true` makes the Slinky engine build `topology/block` domains from `nvidia.com/gpu.clique` instead of provider accelerator-domain data. For `infiniband-k8s`, setting `provider.params.accelerator.source: kubernetes-label` with `kubernetesLabel.key: nvidia.com/gpu.clique` selects the same label without collecting a duplicate value through `nvidia-smi`. The `netq` provider uses a `DomainUUID` from the NMX management API — a different identifier that refers to the same physical domain but cannot be compared as a string. [NVIDIA Fabric Manager](https://docs.nvidia.com/datacenter/tesla/fabric-manager-user-guide/) runs at node init on MNNVL-capable hardware, discovers the NVLink fabric across GPUs, and registers each GPU with [NVML](https://docs.nvidia.com/deploy/nvml-api/) (NVIDIA Management Library — a C API that exposes per-GPU state). The GPU Operator's IMEX labeler writes `nvidia.com/gpu.clique` only once NVML reports the node's fabric state as `GPU_FABRIC_STATE_COMPLETED` — meaning Fabric Manager finished initialization successfully and the node is part of an NVLink domain. diff --git a/pkg/accelerator/accelerator.go b/pkg/accelerator/accelerator.go new file mode 100644 index 00000000..e981edfe --- /dev/null +++ b/pkg/accelerator/accelerator.go @@ -0,0 +1,217 @@ +/* + * Copyright 2026 NVIDIA CORPORATION + * SPDX-License-Identifier: Apache-2.0 + */ + +// Package accelerator discovers accelerator-domain assignments independently +// of the network fabric a provider discovers. +package accelerator + +import ( + "context" + "encoding/json" + "fmt" + "reflect" + "strings" + + internalconfig "github.com/NVIDIA/topograph/internal/config" + "github.com/NVIDIA/topograph/pkg/topology" +) + +const ( + SourceNvidiaSMI = "nvidia-smi" + SourceKubernetesLabel = "kubernetes-label" + SourceNone = "none" + + DefaultGPUOperatorNamespace = "gpu-operator" + DefaultDevicePluginDaemonSet = "nvidia-device-plugin-daemonset" +) + +type Config struct { + Source string `mapstructure:"source"` + KubernetesLabel KubernetesLabelConfig `mapstructure:"kubernetesLabel"` + NvidiaSMI NvidiaSMIConfig `mapstructure:"nvidiaSmi"` +} + +type KubernetesLabelConfig struct { + Key string `mapstructure:"key"` +} + +type NvidiaSMIConfig struct { + GPUOperatorNamespace string `mapstructure:"gpuOperatorNamespace"` + DevicePluginDaemonSet string `mapstructure:"devicePluginDaemonSet"` +} + +// Section is the accelerator section extracted from provider parameters. It +// retains whether the section was omitted so omission can be distinguished +// from an explicitly configured null value. +type Section struct { + value any + present bool +} + +// SectionFromProviderParams extracts the accelerator section without parsing +// source-specific fields in the provider. +func SectionFromProviderParams(providerParams map[string]any) Section { + value, present := providerParams["accelerator"] + return Section{value: value, present: present} +} + +// DecodeSection decodes an accelerator section transported as JSON. An empty +// value represents an omitted section. +func DecodeSection(encoded string) (Section, error) { + if strings.TrimSpace(encoded) == "" { + return Section{}, nil + } + + var value any + if err := json.Unmarshal([]byte(encoded), &value); err != nil { + return Section{}, fmt.Errorf("could not decode accelerator section: %w", err) + } + return Section{value: value, present: true}, nil +} + +// ParseConfig parses and validates the accelerator section of provider +// parameters. An omitted or empty section disables accelerator discovery. +func ParseConfig(section Section) (Config, error) { + config := Config{Source: SourceNone} + if !section.present { + config.SetDefaults() + return config, nil + } + if section.value == nil { + return Config{}, fmt.Errorf("accelerator section must be an object with a source") + } + + value := reflect.ValueOf(section.value) + if value.Kind() != reflect.Map { + return Config{}, fmt.Errorf("accelerator section must be an object with a source") + } + if value.Len() != 0 { + config = Config{} + if err := internalconfig.Decode(section.value, &config); err != nil { + return Config{}, err + } + } + + config.SetDefaults() + if err := config.Validate(); err != nil { + return Config{}, err + } + return config, nil +} + +func (c *Config) SetDefaults() { + c.Source = strings.ToLower(strings.TrimSpace(c.Source)) + c.KubernetesLabel.Key = strings.TrimSpace(c.KubernetesLabel.Key) + c.NvidiaSMI.GPUOperatorNamespace = strings.TrimSpace(c.NvidiaSMI.GPUOperatorNamespace) + if c.NvidiaSMI.GPUOperatorNamespace == "" { + c.NvidiaSMI.GPUOperatorNamespace = DefaultGPUOperatorNamespace + } + c.NvidiaSMI.DevicePluginDaemonSet = strings.TrimSpace(c.NvidiaSMI.DevicePluginDaemonSet) + if c.NvidiaSMI.DevicePluginDaemonSet == "" { + c.NvidiaSMI.DevicePluginDaemonSet = DefaultDevicePluginDaemonSet + } +} + +func (c Config) Validate() error { + if err := ValidateSource(c.Source); err != nil { + return err + } + if c.Source == SourceKubernetesLabel { + if strings.TrimSpace(c.KubernetesLabel.Key) == "" { + return fmt.Errorf("accelerator kubernetesLabel.key must be set for source %q", SourceKubernetesLabel) + } + } + return nil +} + +func ValidateSource(source string) error { + if source == "" { + return fmt.Errorf("accelerator source must be set") + } + switch source { + case SourceNvidiaSMI, SourceNone: + return nil + case SourceKubernetesLabel: + return nil + default: + return fmt.Errorf("unsupported accelerator source %q", source) + } +} + +func NewNoneDiscoverer() Discoverer { + return noneDiscoverer{} +} + +type Target struct { + InstanceID string + HostName string + Labels map[string]string + Annotations map[string]string +} + +type Assignment struct { + DomainID string + SubDomainID string +} + +type Assignments map[string]Assignment + +type Discoverer interface { + Discover(context.Context, []Target) (Assignments, error) +} + +type metadataDiscoverer struct { + key string + value func(Target, string) string +} + +// NewKubernetesDiscoverer parses provider parameters and returns a discoverer +// that resolves accelerator domains from Kubernetes Node metadata. +func NewKubernetesDiscoverer(section Section) (Discoverer, error) { + config, err := ParseConfig(section) + if err != nil { + return nil, err + } + + switch config.Source { + case SourceNvidiaSMI: + return &metadataDiscoverer{ + key: topology.KeyGpuClusterID, + value: func(target Target, key string) string { + return target.Annotations[key] + }, + }, nil + case SourceKubernetesLabel: + return &metadataDiscoverer{ + key: config.KubernetesLabel.Key, + value: func(target Target, key string) string { + return target.Labels[key] + }, + }, nil + case SourceNone: + return noneDiscoverer{}, nil + default: + return nil, fmt.Errorf("unsupported accelerator source %q", config.Source) + } +} + +func (d *metadataDiscoverer) Discover(_ context.Context, targets []Target) (Assignments, error) { + assignments := make(Assignments) + for _, target := range targets { + domainID := strings.TrimSpace(d.value(target, d.key)) + if domainID == "" { + continue + } + assignments[target.InstanceID] = Assignment{DomainID: domainID} + } + + return assignments, nil +} + +type noneDiscoverer struct{} + +func (noneDiscoverer) Discover(context.Context, []Target) (Assignments, error) { + return make(Assignments), nil +} diff --git a/pkg/accelerator/accelerator_test.go b/pkg/accelerator/accelerator_test.go new file mode 100644 index 00000000..568aa9c4 --- /dev/null +++ b/pkg/accelerator/accelerator_test.go @@ -0,0 +1,358 @@ +/* + * Copyright 2026 NVIDIA CORPORATION + * SPDX-License-Identifier: Apache-2.0 + */ + +package accelerator + +import ( + "context" + "errors" + "testing" + + "github.com/stretchr/testify/require" + kubernetesfake "k8s.io/client-go/kubernetes/fake" + "k8s.io/client-go/rest" + + "github.com/NVIDIA/topograph/pkg/topology" +) + +const testKubernetesLabel = "example.com/accelerator-domain" + +func configuredSection(value any) Section { + return SectionFromProviderParams(map[string]any{"accelerator": value}) +} + +func TestConfigDefaultsAndValidation(t *testing.T) { + config := Config{} + config.SetDefaults() + + require.Empty(t, config.Source) + require.Empty(t, config.KubernetesLabel.Key) + require.Equal(t, DefaultGPUOperatorNamespace, config.NvidiaSMI.GPUOperatorNamespace) + require.Equal(t, DefaultDevicePluginDaemonSet, config.NvidiaSMI.DevicePluginDaemonSet) + require.EqualError(t, config.Validate(), "accelerator source must be set") + + config.Source = "invalid" + require.EqualError(t, config.Validate(), `unsupported accelerator source "invalid"`) + + config.Source = SourceKubernetesLabel + require.EqualError(t, config.Validate(), `accelerator kubernetesLabel.key must be set for source "kubernetes-label"`) +} + +func TestParseConfig(t *testing.T) { + tests := []struct { + name string + params map[string]any + source string + labelKey string + gpuNamespace string + devicePlugin string + err string + }{ + {name: "omitted section", source: SourceNone}, + {name: "empty section", params: map[string]any{"accelerator": map[string]any{}}, source: SourceNone}, + { + name: "null section", + params: map[string]any{"accelerator": nil}, + err: "accelerator section must be an object with a source", + }, + { + name: "non-object section", + params: map[string]any{"accelerator": "nvidia-smi"}, + err: "accelerator section must be an object with a source", + }, + { + name: "non-empty section requires source", + params: map[string]any{"accelerator": map[string]any{ + "nvidiaSmi": map[string]any{"gpuOperatorNamespace": "gpu-operator"}, + }}, + err: "accelerator source must be set", + }, + { + name: "label source requires key", + params: map[string]any{"accelerator": map[string]any{"source": SourceKubernetesLabel}}, + err: `accelerator kubernetesLabel.key must be set for source "kubernetes-label"`, + }, + { + name: "configured label source", + params: map[string]any{"accelerator": map[string]any{ + "source": SourceKubernetesLabel, + "kubernetesLabel": map[string]any{ + "key": " example.com/domain ", + }, + }}, + source: SourceKubernetesLabel, + labelKey: "example.com/domain", + }, + { + name: "configured nvidia-smi source", + params: map[string]any{"accelerator": map[string]any{ + "source": " NVIDIA-SMI ", + "nvidiaSmi": map[string]any{ + "gpuOperatorNamespace": "custom-operator", + "devicePluginDaemonSet": "custom-plugin", + }, + }}, + source: SourceNvidiaSMI, + gpuNamespace: "custom-operator", + devicePlugin: "custom-plugin", + }, + { + name: "invalid source", + params: map[string]any{"accelerator": map[string]any{"source": "invalid"}}, + err: `unsupported accelerator source "invalid"`, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + config, err := ParseConfig(SectionFromProviderParams(test.params)) + if test.err != "" { + require.EqualError(t, err, test.err) + return + } + require.NoError(t, err) + require.Equal(t, test.source, config.Source) + require.Equal(t, test.labelKey, config.KubernetesLabel.Key) + expectedNamespace := test.gpuNamespace + if expectedNamespace == "" { + expectedNamespace = DefaultGPUOperatorNamespace + } + expectedDevicePlugin := test.devicePlugin + if expectedDevicePlugin == "" { + expectedDevicePlugin = DefaultDevicePluginDaemonSet + } + require.Equal(t, expectedNamespace, config.NvidiaSMI.GPUOperatorNamespace) + require.Equal(t, expectedDevicePlugin, config.NvidiaSMI.DevicePluginDaemonSet) + }) + } +} + +func TestDecodeSection(t *testing.T) { + section, err := DecodeSection(`{"source":"nvidia-smi","nvidiaSmi":{"gpuOperatorNamespace":"custom"}}`) + require.NoError(t, err) + config, err := ParseConfig(section) + require.NoError(t, err) + require.Equal(t, SourceNvidiaSMI, config.Source) + require.Equal(t, "custom", config.NvidiaSMI.GPUOperatorNamespace) + require.Equal(t, DefaultDevicePluginDaemonSet, config.NvidiaSMI.DevicePluginDaemonSet) + + section, err = DecodeSection("") + require.NoError(t, err) + config, err = ParseConfig(section) + require.NoError(t, err) + require.Equal(t, SourceNone, config.Source) + + _, err = DecodeSection("{") + require.ErrorContains(t, err, "could not decode accelerator section") + + section, err = DecodeSection("null") + require.NoError(t, err) + _, err = ParseConfig(section) + require.EqualError(t, err, "accelerator section must be an object with a source") +} + +func TestKubernetesDiscoverer(t *testing.T) { + targets := []Target{ + { + InstanceID: "instance-1", + HostName: "node-1", + Labels: map[string]string{testKubernetesLabel: "label-domain"}, + Annotations: map[string]string{topology.KeyGpuClusterID: "annotation-domain"}, + }, + {InstanceID: "instance-2", HostName: "node-2"}, + } + + tests := []struct { + name string + params map[string]any + assignments Assignments + }{ + { + name: "nvidia-smi annotation", + params: map[string]any{"accelerator": map[string]any{ + "source": SourceNvidiaSMI, + }}, + assignments: Assignments{ + "instance-1": {DomainID: "annotation-domain"}, + }, + }, + { + name: "Kubernetes label", + params: map[string]any{"accelerator": map[string]any{ + "source": SourceKubernetesLabel, + "kubernetesLabel": map[string]any{ + "key": testKubernetesLabel, + }, + }}, + assignments: Assignments{ + "instance-1": {DomainID: "label-domain"}, + }, + }, + { + name: "none", + assignments: Assignments{}, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + discoverer, err := NewKubernetesDiscoverer(SectionFromProviderParams(test.params)) + require.NoError(t, err) + + assignments, err := discoverer.Discover(context.Background(), targets) + require.NoError(t, err) + require.Equal(t, test.assignments, assignments) + }) + } +} + +type fakeCommandRunner struct { + outputs map[string]string + err error +} + +func (r fakeCommandRunner) Run(context.Context, string, []Target) (map[string]string, error) { + return r.outputs, r.err +} + +type commandRunnerFunc func(context.Context, string, []Target) (map[string]string, error) + +func (f commandRunnerFunc) Run(ctx context.Context, command string, targets []Target) (map[string]string, error) { + return f(ctx, command, targets) +} + +func TestCommandDiscoverer(t *testing.T) { + targets := []Target{{InstanceID: "instance-1", HostName: "node-1"}} + + discoverer, err := NewCommandDiscoverer(SectionFromProviderParams(nil), nil) + require.NoError(t, err) + assignments, err := discoverer.Discover(context.Background(), targets) + require.NoError(t, err) + require.Empty(t, assignments) + + discoverer, err = NewCommandDiscoverer(configuredSection(map[string]any{ + "source": SourceNvidiaSMI, + }), fakeCommandRunner{outputs: map[string]string{"node-1": "uuid, 7"}}) + require.NoError(t, err) + assignments, err = discoverer.Discover(context.Background(), targets) + require.NoError(t, err) + require.Equal(t, Assignments{"instance-1": {DomainID: "uuid.7"}}, assignments) + + _, err = NewCommandDiscoverer(configuredSection(map[string]any{ + "source": SourceKubernetesLabel, + "kubernetesLabel": map[string]any{ + "key": testKubernetesLabel, + }, + }), nil) + require.EqualError(t, err, `accelerator source "kubernetes-label" is not supported by command discovery`) +} + +func TestKubernetesNodeDiscovererWithoutCollection(t *testing.T) { + sections := []Section{ + SectionFromProviderParams(nil), + configuredSection(map[string]any{ + "source": SourceKubernetesLabel, + "kubernetesLabel": map[string]any{ + "key": testKubernetesLabel, + }, + }), + configuredSection(map[string]any{"source": SourceNone}), + } + for _, section := range sections { + discoverer, err := NewKubernetesNodeDiscoverer(section, nil, nil) + require.NoError(t, err) + + assignments, err := discoverer.Discover(context.Background(), []Target{{InstanceID: "node-1", HostName: "node-1"}}) + require.NoError(t, err) + require.Empty(t, assignments) + } + + _, err := NewKubernetesNodeDiscoverer(configuredSection(map[string]any{"source": "invalid"}), nil, nil) + require.EqualError(t, err, `unsupported accelerator source "invalid"`) + + _, err = NewKubernetesNodeDiscoverer(configuredSection(map[string]any{"source": SourceNvidiaSMI}), nil, nil) + require.EqualError(t, err, "k8s client is required for nvidia-smi discovery") +} + +func TestKubernetesNodeDiscovererUsesSectionConfig(t *testing.T) { + section, err := DecodeSection(`{ + "source":"nvidia-smi", + "nvidiaSmi":{ + "gpuOperatorNamespace":"custom-operator", + "devicePluginDaemonSet":"custom-plugin" + } + }`) + require.NoError(t, err) + + discoverer, err := NewKubernetesNodeDiscoverer(section, kubernetesfake.NewClientset(), &rest.Config{}) + require.NoError(t, err) + nvidiaDiscoverer, ok := discoverer.(*nvidiaSMIDiscoverer) + require.True(t, ok) + runner, ok := nvidiaDiscoverer.runner.(*kubernetesNvidiaSMIRunner) + require.True(t, ok) + require.Equal(t, "custom-operator", runner.namespace) + require.Equal(t, "custom-plugin", runner.daemonSet) +} + +func TestNvidiaSMIDiscoverer(t *testing.T) { + targets := []Target{{InstanceID: "instance-1", HostName: "node-1"}} + discoverer, err := NewNvidiaSMIDiscoverer(Config{Source: SourceNvidiaSMI}, fakeCommandRunner{outputs: map[string]string{ + "node-1": "uuid, 7\nuuid, 7\n", + }}) + require.NoError(t, err) + + assignments, err := discoverer.Discover(context.Background(), targets) + require.NoError(t, err) + require.Equal(t, Assignments{"instance-1": {DomainID: "uuid.7"}}, assignments) + + discoverer, err = NewNvidiaSMIDiscoverer(Config{Source: SourceNvidiaSMI}, fakeCommandRunner{err: errors.New("command failed")}) + require.NoError(t, err) + _, err = discoverer.Discover(context.Background(), targets) + require.EqualError(t, err, "failed to query NVL partition IDs: command failed") + + ctx, cancel := context.WithCancel(context.Background()) + cancel() + var observedContextErr error + discoverer, err = NewNvidiaSMIDiscoverer(Config{Source: SourceNvidiaSMI}, commandRunnerFunc( + func(ctx context.Context, _ string, _ []Target) (map[string]string, error) { + observedContextErr = ctx.Err() + return nil, ctx.Err() + }, + )) + require.NoError(t, err) + _, err = discoverer.Discover(ctx, targets) + require.ErrorIs(t, observedContextErr, context.Canceled) + require.ErrorIs(t, err, context.Canceled) +} + +func TestParseNvidiaSMIOutput(t *testing.T) { + tests := []struct { + name string + output string + partition string + err string + }{ + {name: "duplicates", output: "uuid, 7\nuuid , 7\n", partition: "uuid.7"}, + {name: "missing", err: "missing NVL partition ID"}, + {name: "missing UUID", output: ", 7", err: "missing ClusterUUID"}, + {name: "missing clique", output: "uuid, ", err: "missing CliqueId"}, + {name: "malformed CSV", output: "uuid", err: `expected ClusterUUID and CliqueId CSV fields, got "uuid"`}, + {name: "N/A UUID", output: "N/A, 7", err: "ClusterUUID is N/A"}, + {name: "N/A clique", output: "uuid, N/A", err: "CliqueId is N/A"}, + {name: "ambiguous", output: "uuid, 7\nuuid, 8", err: "ambiguous NVL partition IDs: uuid.7, uuid.8"}, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + partition, err := ParseNvidiaSMIOutput(test.output) + if test.err != "" { + require.EqualError(t, err, test.err) + return + } + require.NoError(t, err) + require.Equal(t, test.partition, partition) + }) + } +} diff --git a/pkg/accelerator/kubernetes.go b/pkg/accelerator/kubernetes.go new file mode 100644 index 00000000..955c466b --- /dev/null +++ b/pkg/accelerator/kubernetes.go @@ -0,0 +1,87 @@ +/* + * Copyright 2026 NVIDIA CORPORATION + * SPDX-License-Identifier: Apache-2.0 + */ + +package accelerator + +import ( + "context" + "fmt" + "strings" + + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" + "k8s.io/klog/v2" + + internalK8s "github.com/NVIDIA/topograph/internal/k8s" +) + +// NewKubernetesNodeDiscoverer returns the node-local discoverer used by the +// node-data-broker. Sources that read existing Kubernetes metadata require no +// node-local collection and therefore return an empty discoverer. +func NewKubernetesNodeDiscoverer(section Section, client kubernetes.Interface, restConfig *rest.Config) (Discoverer, error) { + config, err := ParseConfig(section) + if err != nil { + return nil, err + } + + switch config.Source { + case SourceNvidiaSMI: + if client == nil { + return nil, fmt.Errorf("k8s client is required for nvidia-smi discovery") + } + if restConfig == nil { + return nil, fmt.Errorf("k8s REST config is required for nvidia-smi discovery") + } + return NewNvidiaSMIDiscoverer(config, &kubernetesNvidiaSMIRunner{ + client: client, + config: restConfig, + namespace: config.NvidiaSMI.GPUOperatorNamespace, + daemonSet: config.NvidiaSMI.DevicePluginDaemonSet, + }) + case SourceKubernetesLabel, SourceNone: + return NewNoneDiscoverer(), nil + default: + return nil, fmt.Errorf("unsupported accelerator source %q", config.Source) + } +} + +type kubernetesNvidiaSMIRunner struct { + client kubernetes.Interface + config *rest.Config + namespace string + daemonSet string +} + +func (r *kubernetesNvidiaSMIRunner) Run(ctx context.Context, command string, targets []Target) (map[string]string, error) { + outputs := make(map[string]string) + for _, target := range targets { + pods, err := internalK8s.GetDaemonSetPods(ctx, r.client, r.daemonSet, r.namespace, target.HostName) + if err != nil { + return nil, err + } + + switch len(pods.Items) { + case 0: + klog.Infof("no %s on %s node", r.daemonSet, target.HostName) + case 1: + output, err := internalK8s.ExecInPod( + ctx, + r.client, + r.config, + pods.Items[0].Name, + r.namespace, + strings.Fields(command), + ) + if err != nil { + return nil, fmt.Errorf("failed to query NVL partition ID: %w", err) + } + outputs[target.HostName] = output.String() + default: + return nil, fmt.Errorf("expected 1 %s pod, got %d", r.daemonSet, len(pods.Items)) + } + } + + return outputs, nil +} diff --git a/pkg/accelerator/nvidia_smi.go b/pkg/accelerator/nvidia_smi.go new file mode 100644 index 00000000..f9ecbef6 --- /dev/null +++ b/pkg/accelerator/nvidia_smi.go @@ -0,0 +1,136 @@ +/* + * Copyright 2026 NVIDIA CORPORATION + * SPDX-License-Identifier: Apache-2.0 + */ + +package accelerator + +import ( + "context" + "fmt" + "sort" + "strings" +) + +const NvidiaSMICommand = "nvidia-smi --query-gpu=fabric.clusterUuid,fabric.cliqueId --format=csv,noheader" + +type CommandRunner interface { + Run(context.Context, string, []Target) (map[string]string, error) +} + +type nvidiaSMIDiscoverer struct { + runner CommandRunner +} + +// NewCommandDiscoverer parses provider parameters and returns a discoverer +// suitable for environments that can execute commands on accelerator nodes. +func NewCommandDiscoverer(section Section, runner CommandRunner) (Discoverer, error) { + config, err := ParseConfig(section) + if err != nil { + return nil, err + } + + switch config.Source { + case SourceNvidiaSMI: + return NewNvidiaSMIDiscoverer(config, runner) + case SourceNone: + return NewNoneDiscoverer(), nil + default: + return nil, fmt.Errorf("accelerator source %q is not supported by command discovery", config.Source) + } +} + +func NewNvidiaSMIDiscoverer(config Config, runner CommandRunner) (Discoverer, error) { + config.SetDefaults() + if err := config.Validate(); err != nil { + return nil, err + } + if config.Source != SourceNvidiaSMI { + return nil, fmt.Errorf("accelerator source %q cannot use an nvidia-smi command runner", config.Source) + } + if runner == nil { + return nil, fmt.Errorf("nvidia-smi command runner is required") + } + + return &nvidiaSMIDiscoverer{runner: runner}, nil +} + +func (d *nvidiaSMIDiscoverer) Discover(ctx context.Context, targets []Target) (Assignments, error) { + outputs, err := d.runner.Run(ctx, NvidiaSMICommand, targets) + if err != nil { + return nil, fmt.Errorf("failed to query NVL partition IDs: %w", err) + } + + assignments := make(Assignments) + for _, target := range targets { + output := strings.TrimSpace(outputs[target.HostName]) + if output == "" { + continue + } + + partitionID, err := ParseNvidiaSMIOutput(output) + if err != nil { + return nil, fmt.Errorf("invalid nvidia-smi output for node %q: %w", target.HostName, err) + } + assignments[target.InstanceID] = Assignment{DomainID: partitionID} + } + + return assignments, nil +} + +func ParseNvidiaSMIOutput(output string) (string, error) { + partitions := make(map[string]struct{}) + for line := range strings.Lines(output) { + line = strings.TrimSpace(line) + if line == "" { + continue + } + + partitionID, err := parseNVLPartitionID(line) + if err != nil { + return "", err + } + partitions[partitionID] = struct{}{} + } + + if len(partitions) == 0 { + return "", fmt.Errorf("missing NVL partition ID") + } + + partitionIDs := make([]string, 0, len(partitions)) + for partitionID := range partitions { + partitionIDs = append(partitionIDs, partitionID) + } + sort.Strings(partitionIDs) + + if len(partitionIDs) != 1 { + return "", fmt.Errorf("ambiguous NVL partition IDs: %s", strings.Join(partitionIDs, ", ")) + } + + return partitionIDs[0], nil +} + +func parseNVLPartitionID(line string) (string, error) { + fields := strings.Split(line, ",") + if len(fields) != 2 { + return "", fmt.Errorf("expected ClusterUUID and CliqueId CSV fields, got %q", line) + } + + clusterUUID := strings.TrimSpace(fields[0]) + if clusterUUID == "" { + return "", fmt.Errorf("missing ClusterUUID") + } + if clusterUUID == "N/A" { + return "", fmt.Errorf("ClusterUUID is N/A") + } + + cliqueID := strings.TrimSpace(fields[1]) + if cliqueID == "" { + return "", fmt.Errorf("missing CliqueId") + } + if cliqueID == "N/A" { + return "", fmt.Errorf("CliqueId is N/A") + } + + return clusterUUID + "." + cliqueID, nil +} diff --git a/pkg/engines/slinky/engine.go b/pkg/engines/slinky/engine.go index cb69e3c4..aedab504 100644 --- a/pkg/engines/slinky/engine.go +++ b/pkg/engines/slinky/engine.go @@ -154,7 +154,6 @@ func getParameters(params engines.Config) (*Params, error) { if len(p.ConfigUpdateMode) != 0 && p.ConfigUpdateMode != ConfigUpdateModeNone && p.ConfigUpdateMode != ConfigUpdateModeSkeletonOnly { return nil, fmt.Errorf("invalid configUpdateMode: %s, must be either %s, or %s", p.ConfigUpdateMode, ConfigUpdateModeNone, ConfigUpdateModeSkeletonOnly) } - sel, err := metav1.LabelSelectorAsSelector(&p.PodSelector) if err != nil { return nil, err diff --git a/pkg/providers/infiniband/bm.go b/pkg/providers/infiniband/bm.go index 981c7aa7..19dfb24d 100644 --- a/pkg/providers/infiniband/bm.go +++ b/pkg/providers/infiniband/bm.go @@ -9,91 +9,76 @@ import ( "bufio" "bytes" "context" - "fmt" "strings" - "k8s.io/klog/v2" - "github.com/NVIDIA/topograph/internal/exec" + "github.com/NVIDIA/topograph/pkg/accelerator" "github.com/NVIDIA/topograph/pkg/topology" ) -type Cluster struct { - node string - UUID string - cliqueID string +type IBNetDiscoverBM struct{} + +func (h *IBNetDiscoverBM) Run(ctx context.Context, node string) (*bytes.Buffer, error) { + return exec.Pdsh(ctx, "sudo ibnetdiscover", []string{node}, "-N") } -type IBNetDiscoverBM struct{} +type pdshNvidiaSMIRunner struct{} -func (c *Cluster) ID() (string, error) { - if len(c.UUID) == 0 { - return "", fmt.Errorf("missing ClusterUUID for node %q", c.node) +func (pdshNvidiaSMIRunner) Run(ctx context.Context, command string, targets []accelerator.Target) (map[string]string, error) { + nodes := make([]string, 0, len(targets)) + for _, target := range targets { + nodes = append(nodes, target.HostName) } - if len(c.cliqueID) == 0 { - return "", fmt.Errorf("missing CliqueId for node %q", c.node) - } - return c.UUID + "." + c.cliqueID, nil -} -func (h *IBNetDiscoverBM) Run(ctx context.Context, node string) (*bytes.Buffer, error) { - return exec.Pdsh(ctx, "sudo ibnetdiscover", []string{node}, "-N") + stdout, err := exec.Pdsh(ctx, command, nodes) + if err != nil { + return nil, err + } + return parsePdshNvidiaSMIOutput(stdout) } -func populateDomainsFromPdshOutput(stdout *bytes.Buffer) (topology.DomainMap, error) { - clusters := make(map[string]*Cluster) - invalid := make(map[string]bool) +func parsePdshNvidiaSMIOutput(stdout *bytes.Buffer) (map[string]string, error) { + outputs := make(map[string]string) scanner := bufio.NewScanner(stdout) for scanner.Scan() { nodeLine := scanner.Text() - arr := strings.Split(nodeLine, ":") - if len(arr) < 3 { - klog.V(4).Infof("skipping malformed ibnetdiscover line: %q", nodeLine) + arr := strings.SplitN(nodeLine, ":", 2) + if len(arr) != 2 { continue } nodeName := strings.TrimSpace(arr[0]) - idName := strings.TrimSpace(arr[1]) - val := strings.TrimSpace(arr[2]) - cluster, ok := clusters[nodeName] - if !ok { - cluster = &Cluster{node: nodeName} - clusters[nodeName] = cluster - } - switch idName { - case "CliqueId": - setID(nodeName, idName, &cluster.cliqueID, val, invalid) - case "ClusterUUID": - setID(nodeName, idName, &cluster.UUID, val, invalid) - } + outputs[nodeName] += strings.TrimSpace(arr[1]) + "\n" } if err := scanner.Err(); err != nil { return nil, err } - // delete invalid nodes - for nodeName := range invalid { - delete(clusters, nodeName) - } + return outputs, nil +} - domainMap := topology.NewDomainMap() - for nodeName, cluster := range clusters { - clusterID, err := cluster.ID() - if err != nil { - return nil, err +func acceleratorTargets(cis []topology.ComputeInstances) []accelerator.Target { + targets := make([]accelerator.Target, 0) + for _, ci := range cis { + for instanceID, hostName := range ci.Instances { + targets = append(targets, accelerator.Target{InstanceID: instanceID, HostName: hostName}) } - domainMap.AddHost(clusterID, nodeName, nodeName) } - - klog.V(4).Info(domainMap.String()) - - return domainMap, nil + return targets } -func setID(nodename, idname string, id *string, val string, invalid map[string]bool) { - if len(*id) == 0 { - *id = val - } else { - klog.Warningf("Ambiguous %s %q, %q for node %q", idname, *id, val, nodename) - invalid[nodename] = true +func domainMapFromAssignments(assignments accelerator.Assignments, targets []accelerator.Target) topology.DomainMap { + domainMap := topology.NewDomainMap() + for _, target := range targets { + assignment, ok := assignments[target.InstanceID] + if !ok { + continue + } + domainMap.AddHostInfo(&topology.HostInfo{ + Domain: assignment.DomainID, + SubDomain: assignment.SubDomainID, + InstanceID: target.InstanceID, + HostName: target.HostName, + }) } + return domainMap } diff --git a/pkg/providers/infiniband/bm_test.go b/pkg/providers/infiniband/bm_test.go index 0d474af6..eafc8402 100644 --- a/pkg/providers/infiniband/bm_test.go +++ b/pkg/providers/infiniband/bm_test.go @@ -11,113 +11,48 @@ import ( "github.com/stretchr/testify/require" + "github.com/NVIDIA/topograph/pkg/accelerator" "github.com/NVIDIA/topograph/pkg/topology" ) -func TestPopulateDomainsFromPdshOutput(t *testing.T) { - nvOutput := `node-10: CliqueId : 4000000004 - node-10: ClusterUUID : 50000000-0000-0000-0000-000000000005 - node-07: CliqueId : 4000000005 - node-07: ClusterUUID : 50000000-0000-0000-0000-000000000004 - node-11: CliqueId : 50000000-0000-0000-0000-000000000003 - node-11: CliqueId : N/A - node-11: ClusterUUID : 4000000003 - node-11: ClusterUUID : N/A - node-08: CliqueId : 4000000005 - node-08: ClusterUUID : 50000000-0000-0000-0000-000000000004 - node-09: CliqueId : 4000000005 - node-09: ClusterUUID : 50000000-0000-0000-0000-000000000005 -` - domainMap := topology.DomainMap{ - "50000000-0000-0000-0000-000000000004.4000000005": map[string]*topology.HostInfo{"node-07": {Domain: "50000000-0000-0000-0000-000000000004.4000000005", HostName: "node-07", InstanceID: "node-07"}, "node-08": {Domain: "50000000-0000-0000-0000-000000000004.4000000005", HostName: "node-08", InstanceID: "node-08"}}, - "50000000-0000-0000-0000-000000000005.4000000004": map[string]*topology.HostInfo{"node-10": {Domain: "50000000-0000-0000-0000-000000000005.4000000004", HostName: "node-10", InstanceID: "node-10"}}, - "50000000-0000-0000-0000-000000000005.4000000005": map[string]*topology.HostInfo{"node-09": {Domain: "50000000-0000-0000-0000-000000000005.4000000005", HostName: "node-09", InstanceID: "node-09"}}, - } +func TestParsePdshNvidiaSMIOutput(t *testing.T) { + output := bytes.NewBufferString(`node-1: uuid-1, 7 +node-1: uuid-1, 7 +malformed +node-2: uuid-2, 8 +`) + + outputs, err := parsePdshNvidiaSMIOutput(output) + require.NoError(t, err) + require.Equal(t, map[string]string{ + "node-1": "uuid-1, 7\nuuid-1, 7\n", + "node-2": "uuid-2, 8\n", + }, outputs) +} - testCases := []struct { - name string - nvOutput string - domains topology.DomainMap - err string - }{ - { - name: "Case 1: missing CliqueId", - nvOutput: ` node-10: ClusterUUID : 50000000-0000-0000-0000-000000000005`, - err: `missing CliqueId for node "node-10"`, - }, - { - name: "Case 2: missing ClusterUUID", - nvOutput: `node-10: CliqueId : 4000000004 - node-10: ClusterUUID : 50000000-0000-0000-0000-000000000005 - node-07: CliqueId : 4000000005 -`, - err: `missing ClusterUUID for node "node-07"`, - }, - { - name: "Case 3: valid input", - nvOutput: nvOutput, - domains: domainMap, +func TestAcceleratorTargetsAndDomainMap(t *testing.T) { + targets := acceleratorTargets([]topology.ComputeInstances{{ + Instances: map[string]string{ + "instance-1": "node-1", + "instance-2": "node-2", }, - { - name: "Case 4: malformed line (missing second colon) is skipped gracefully", - nvOutput: "node1:CliqueId\n", - domains: topology.NewDomainMap(), + }}) + require.ElementsMatch(t, []accelerator.Target{ + {InstanceID: "instance-1", HostName: "node-1"}, + {InstanceID: "instance-2", HostName: "node-2"}, + }, targets) + + domainMap := domainMapFromAssignments(accelerator.Assignments{ + "instance-1": {DomainID: "domain-1", SubDomainID: "partition-1"}, + }, targets) + require.Equal(t, topology.DomainMap{ + "domain-1": { + "node-1": { + Domain: "domain-1", + SubDomain: "partition-1", + InstanceID: "instance-1", + HostName: "node-1", + }, }, - } - - for _, tc := range testCases { - t.Run(tc.name, func(t *testing.T) { - domains, err := populateDomainsFromPdshOutput(bytes.NewBufferString(tc.nvOutput)) - if len(tc.err) != 0 { - require.EqualError(t, err, tc.err) - } else { - require.NoError(t, err) - require.Equal(t, tc.domains, domains) - } - }) - } -} - -func TestSetID(t *testing.T) { - clusters := map[string]*Cluster{ - "node1": {node: "node1"}, - "node2": {node: "node2"}, - "node3": {node: "node3"}, - } - invalid := make(map[string]bool) - - input := []struct { - nodename string - idname string - val string - }{ - {nodename: "node1", idname: "ID", val: "ID1"}, - {nodename: "node1", idname: "UUID", val: "UUID1"}, - {nodename: "node2", idname: "ID", val: "ID2"}, - {nodename: "node2", idname: "UUID", val: "UUID2"}, - {nodename: "node2", idname: "ID", val: "N/A"}, - {nodename: "node2", idname: "UUID", val: "N/A"}, - {nodename: "node3", idname: "ID", val: "ID3"}, - {nodename: "node3", idname: "UUID", val: "UUID3"}, - } - - for _, i := range input { - cluster := clusters[i.nodename] - switch i.idname { - case "ID": - setID(i.nodename, i.idname, &cluster.cliqueID, i.val, invalid) - case "UUID": - setID(i.nodename, i.idname, &cluster.UUID, i.val, invalid) - } - } - - resClusters := map[string]*Cluster{ - "node1": {node: "node1", UUID: "UUID1", cliqueID: "ID1"}, - "node2": {node: "node2", UUID: "UUID2", cliqueID: "ID2"}, - "node3": {node: "node3", UUID: "UUID3", cliqueID: "ID3"}, - } - resInvalid := map[string]bool{"node2": true} - - require.Equal(t, resClusters, clusters) - require.Equal(t, resInvalid, invalid) + }, domainMap) } diff --git a/pkg/providers/infiniband/common.go b/pkg/providers/infiniband/common.go index 197e6022..e67558bd 100644 --- a/pkg/providers/infiniband/common.go +++ b/pkg/providers/infiniband/common.go @@ -17,10 +17,6 @@ import ( "github.com/NVIDIA/topograph/pkg/topology" ) -const ( - cmdClusterID = `nvidia-smi -q | grep "ClusterUUID\|CliqueId" | sort -u` -) - type IBNetDiscover interface { Run(context.Context, string) (*bytes.Buffer, error) } diff --git a/pkg/providers/infiniband/k8s.go b/pkg/providers/infiniband/k8s.go index 7b3a4929..12cd6d9e 100644 --- a/pkg/providers/infiniband/k8s.go +++ b/pkg/providers/infiniband/k8s.go @@ -6,30 +6,20 @@ package infiniband import ( - "bufio" "bytes" "context" "fmt" "os" - "strings" "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" "k8s.io/klog/v2" "github.com/NVIDIA/topograph/internal/k8s" + "github.com/NVIDIA/topograph/pkg/accelerator" "github.com/NVIDIA/topograph/pkg/topology" ) -const ( - gpuOperatorNamespaceArg = "gpu-operator-namespace" - devicePluginDaemonSetArg = "device-plugin-daemonset" - useGPUCliqueLabelArg = "useGpuCliqueLabel" - - defaultGpuOperatorNamespace = "gpu-operator" - defaultDevicePluginDaemonSet = "nvidia-device-plugin-daemonset" -) - type IBNetDiscoverK8S struct { config *rest.Config client *kubernetes.Clientset @@ -57,95 +47,23 @@ func (h *IBNetDiscoverK8S) Run(ctx context.Context, node string) (*bytes.Buffer, return k8s.ExecInPod(ctx, h.client, h.config, pods.Items[0].Name, dataBrokerNamespace, []string{"ibnetdiscover"}) } -func GetGpuClusterID(ctx context.Context, client kubernetes.Interface, config *rest.Config, hostname string, overrides map[string]string) (string, error) { - ds, namespace := getDevicePluginInfo(overrides) - - pods, err := k8s.GetDaemonSetPods(ctx, client, ds, namespace, hostname) - if err != nil { - return "", err - } - - switch len(pods.Items) { - case 0: - klog.Infof("no %s on %s node", ds, hostname) - return "", nil - case 1: - cmd := []string{"sh", "-c", cmdClusterID} - buf, err := k8s.ExecInPod(ctx, client, config, pods.Items[0].Name, namespace, cmd) - if err != nil { - return "", err - } - return parseClusterID(buf.String()) - default: - return "", fmt.Errorf("expected 1 %s pod, got %d", ds, len(pods.Items)) - } -} - -func getDevicePluginInfo(overrides map[string]string) (daemonset string, namespace string) { - var ok bool - if daemonset, ok = overrides[devicePluginDaemonSetArg]; !ok { - daemonset = defaultDevicePluginDaemonSet - } - if namespace, ok = overrides[gpuOperatorNamespaceArg]; !ok { - namespace = defaultGpuOperatorNamespace - } - - return -} - -func parseClusterID(txt string) (string, error) { - klog.V(4).Infof("ClusterID output: %q", txt) - var cliqueId, clusterUUID string - scanner := bufio.NewScanner(strings.NewReader(txt)) - for scanner.Scan() { - line := scanner.Text() - arr := strings.Split(line, ":") - if len(arr) < 2 { - continue - } - switch strings.TrimSpace(arr[0]) { - case "CliqueId": - cliqueId = strings.TrimSpace(arr[1]) - case "ClusterUUID": - clusterUUID = strings.TrimSpace(arr[1]) - } - } - - if err := scanner.Err(); err != nil { - return "", fmt.Errorf("failed to scan %q: %v", txt, err) - } - - if len(clusterUUID) == 0 { - return "", fmt.Errorf("missing ClusterUUID") - } - - if len(cliqueId) == 0 { - return "", fmt.Errorf("missing CliqueId") - } - - klog.V(4).InfoS("Cluster ID", "clusterUUID", clusterUUID, "cliqueId", cliqueId) - return clusterUUID + "." + cliqueId, nil -} - -func GetNodeAnnotations(ctx context.Context, client kubernetes.Interface, config *rest.Config, hostname string, overrides map[string]string) (map[string]string, error) { +func GetNodeAnnotations(ctx context.Context, client kubernetes.Interface, config *rest.Config, hostname string, section accelerator.Section) (map[string]string, error) { annotations := map[string]string{ topology.KeyNodeInstance: hostname, topology.KeyNodeRegion: "local", } - if useGPUCliqueLabel(overrides) { - return annotations, nil + discoverer, err := accelerator.NewKubernetesNodeDiscoverer(section, client, config) + if err != nil { + return nil, err } - if clusterID, err := GetGpuClusterID(ctx, client, config, hostname, overrides); err != nil { - klog.Warningf("No clusterID for node %s: %v", hostname, err) - } else if clusterID != "" { - annotations[topology.KeyGpuClusterID] = clusterID + assignments, err := discoverer.Discover(ctx, []accelerator.Target{{InstanceID: hostname, HostName: hostname}}) + if err != nil { + klog.Warningf("No accelerator domain for node %s: %v", hostname, err) + } else if assignment, ok := assignments[hostname]; ok { + annotations[topology.KeyGpuClusterID] = assignment.DomainID } return annotations, nil } - -func useGPUCliqueLabel(overrides map[string]string) bool { - return strings.EqualFold(strings.TrimSpace(overrides[useGPUCliqueLabelArg]), "true") -} diff --git a/pkg/providers/infiniband/k8s_test.go b/pkg/providers/infiniband/k8s_test.go index 4d5451a7..8bf414ed 100644 --- a/pkg/providers/infiniband/k8s_test.go +++ b/pkg/providers/infiniband/k8s_test.go @@ -11,116 +11,31 @@ import ( "github.com/stretchr/testify/require" + "github.com/NVIDIA/topograph/pkg/accelerator" "github.com/NVIDIA/topograph/pkg/topology" ) -func TestGetDevicePluginInfo(t *testing.T) { - tests := []struct { - name string - overrides map[string]string - ns string - ds string - }{ - { - name: "Case 1: no overrides uses defaults", - ns: defaultGpuOperatorNamespace, - ds: defaultDevicePluginDaemonSet, - }, - { - name: "Case 2: override namespace only", - overrides: map[string]string{ - gpuOperatorNamespaceArg: "custom-ns", - }, - ns: "custom-ns", - ds: defaultDevicePluginDaemonSet, - }, - { - name: "Case 3: override daemonset only", - overrides: map[string]string{ - devicePluginDaemonSetArg: "custom-ds", - }, - ns: defaultGpuOperatorNamespace, - ds: "custom-ds", - }, - { - name: "Case 4: override both", - overrides: map[string]string{ - gpuOperatorNamespaceArg: "custom-ns", - devicePluginDaemonSetArg: "custom-ds", - }, - ns: "custom-ns", - ds: "custom-ds", - }, - { - name: "Case 5: irrelevant keys ignored", - overrides: map[string]string{ - "other": "value", - }, - ns: defaultGpuOperatorNamespace, - ds: defaultDevicePluginDaemonSet, - }, - } +func TestGetNodeAnnotationsWithoutCollection(t *testing.T) { + ctx := context.TODO() - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - ds, ns := getDevicePluginInfo(tt.overrides) - require.Equal(t, tt.ds, ds) - require.Equal(t, tt.ns, ns) - }) + sections := []string{ + "", + `{"source":"kubernetes-label","kubernetesLabel":{"key":"example.com/domain"}}`, + `{"source":"none"}`, } -} - -func TestParseClusterID(t *testing.T) { - tests := []struct { - name string - input string - clusterID string - err string - }{ - { - name: "Case 1: missing ClusterUUID", - err: "missing ClusterUUID", - }, - { - name: "Case 2: missing CliqueId", - input: " ClusterUUID : 0000-0000-0000-0000-000000000000", - err: "missing CliqueId", - }, - { - name: "Case 3: valid input", - input: ` - CliqueId : 0 - ClusterUUID : 00000000-0000-0000-0000-000000000000 -`, - clusterID: "00000000-0000-0000-0000-000000000000.0", - }, - { - name: "Case 4: malformed line (no colon) is skipped", - input: "CliqueId\n", - err: "missing ClusterUUID", - }, + for _, encodedSection := range sections { + section, err := accelerator.DecodeSection(encodedSection) + require.NoError(t, err) + annotations, err := GetNodeAnnotations(ctx, nil, nil, "node-1", section) + require.NoError(t, err) + require.Equal(t, map[string]string{ + topology.KeyNodeInstance: "node-1", + topology.KeyNodeRegion: "local", + }, annotations) } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - clusterID, err := parseClusterID(tt.input) - if len(tt.err) != 0 { - require.EqualError(t, err, tt.err) - } else { - require.NoError(t, err) - require.Equal(t, tt.clusterID, clusterID) - } - }) - } -} - -func TestGetNodeAnnotationsWithGPUCliqueLabel(t *testing.T) { - ctx := context.TODO() - - annotations, err := GetNodeAnnotations(ctx, nil, nil, "node-1", map[string]string{useGPUCliqueLabelArg: "true"}) + section, err := accelerator.DecodeSection(`{"source":"invalid"}`) require.NoError(t, err) - require.Equal(t, map[string]string{ - topology.KeyNodeInstance: "node-1", - topology.KeyNodeRegion: "local", - }, annotations) + _, err = GetNodeAnnotations(ctx, nil, nil, "node-1", section) + require.EqualError(t, err, `unsupported accelerator source "invalid"`) } diff --git a/pkg/providers/infiniband/provider_bm.go b/pkg/providers/infiniband/provider_bm.go index 058d1961..dd0b1087 100644 --- a/pkg/providers/infiniband/provider_bm.go +++ b/pkg/providers/infiniband/provider_bm.go @@ -10,22 +10,32 @@ import ( "fmt" "net/http" - "github.com/NVIDIA/topograph/internal/exec" "github.com/NVIDIA/topograph/internal/httperr" + "github.com/NVIDIA/topograph/pkg/accelerator" "github.com/NVIDIA/topograph/pkg/providers" "github.com/NVIDIA/topograph/pkg/topology" ) const NAME_BM = "infiniband-bm" -type ProviderBM struct{} +type ProviderBM struct { + accelerator accelerator.Discoverer +} func NamedLoaderBM() (string, providers.Loader) { return NAME_BM, LoaderBM } -func LoaderBM(_ context.Context, _ providers.Config) (providers.Provider, *httperr.Error) { - return &ProviderBM{}, nil +func LoaderBM(_ context.Context, providerConfig providers.Config) (providers.Provider, *httperr.Error) { + discoverer, err := accelerator.NewCommandDiscoverer( + accelerator.SectionFromProviderParams(providerConfig.Params), + pdshNvidiaSMIRunner{}, + ) + if err != nil { + return nil, httperr.NewError(http.StatusBadRequest, err.Error()) + } + + return &ProviderBM{accelerator: discoverer}, nil } func (p *ProviderBM) GenerateTopologyConfig(ctx context.Context, _ *int, cis []topology.ComputeInstances) (*topology.Graph, *httperr.Error) { @@ -33,17 +43,12 @@ func (p *ProviderBM) GenerateTopologyConfig(ctx context.Context, _ *int, cis []t return nil, httperr.NewError(http.StatusBadRequest, "on-prem does not support multi-region topology requests") } - nodes := topology.GetNodeNameList(cis) - - output, err := exec.Pdsh(ctx, cmdClusterID, nodes) - if err != nil { - return nil, httperr.NewError(http.StatusInternalServerError, err.Error()) - } - - domainMap, err := populateDomainsFromPdshOutput(output) + targets := acceleratorTargets(cis) + assignments, err := p.accelerator.Discover(ctx, targets) if err != nil { - return nil, httperr.NewError(http.StatusInternalServerError, fmt.Sprintf("failed to populate NVL domains: %v", err)) + return nil, httperr.NewError(http.StatusInternalServerError, fmt.Sprintf("failed to discover accelerator domains: %v", err)) } + domainMap := domainMapFromAssignments(assignments, targets) treeRoot, err := getIbTree(ctx, cis, &IBNetDiscoverBM{}) if err != nil { diff --git a/pkg/providers/infiniband/provider_bm_test.go b/pkg/providers/infiniband/provider_bm_test.go new file mode 100644 index 00000000..319d6d91 --- /dev/null +++ b/pkg/providers/infiniband/provider_bm_test.go @@ -0,0 +1,76 @@ +/* + * Copyright 2026 NVIDIA CORPORATION + * SPDX-License-Identifier: Apache-2.0 + */ + +package infiniband + +import ( + "context" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/NVIDIA/topograph/pkg/accelerator" + "github.com/NVIDIA/topograph/pkg/providers" +) + +func TestLoaderBMAcceleratorSource(t *testing.T) { + tests := []struct { + name string + params map[string]any + err string + }{ + {name: "no accelerator discovery by default"}, + { + name: "empty accelerator section disables discovery", + params: map[string]any{"accelerator": map[string]any{}}, + }, + { + name: "null accelerator section", + params: map[string]any{"accelerator": nil}, + err: "accelerator section must be an object with a source", + }, + { + name: "non-empty accelerator section requires source", + params: map[string]any{"accelerator": map[string]any{ + "nvidiaSmi": map[string]any{"gpuOperatorNamespace": "gpu-operator"}, + }}, + err: "accelerator source must be set", + }, + { + name: "nvidia-smi", + params: map[string]any{"accelerator": map[string]any{ + "source": accelerator.SourceNvidiaSMI, + }}, + }, + { + name: "none", + params: map[string]any{"accelerator": map[string]any{ + "source": accelerator.SourceNone, + }}, + }, + { + name: "Kubernetes label is unsupported", + params: map[string]any{"accelerator": map[string]any{ + "source": accelerator.SourceKubernetesLabel, + "kubernetesLabel": map[string]any{ + "key": "example.com/domain", + }, + }}, + err: `accelerator source "kubernetes-label" is not supported by command discovery`, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + provider, httpErr := LoaderBM(context.Background(), providers.Config{Params: test.params}) + if test.err != "" { + require.EqualError(t, httpErr, test.err) + return + } + require.Nil(t, httpErr) + require.IsType(t, &ProviderBM{}, provider) + }) + } +} diff --git a/pkg/providers/infiniband/provider_k8s.go b/pkg/providers/infiniband/provider_k8s.go index 8cdae8df..4ed3844a 100644 --- a/pkg/providers/infiniband/provider_k8s.go +++ b/pkg/providers/infiniband/provider_k8s.go @@ -9,9 +9,7 @@ import ( "context" "fmt" "net/http" - "strings" - corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" "k8s.io/client-go/kubernetes" @@ -20,6 +18,7 @@ import ( "github.com/NVIDIA/topograph/internal/config" "github.com/NVIDIA/topograph/internal/httperr" "github.com/NVIDIA/topograph/internal/k8s" + "github.com/NVIDIA/topograph/pkg/accelerator" "github.com/NVIDIA/topograph/pkg/providers" "github.com/NVIDIA/topograph/pkg/topology" ) @@ -27,19 +26,16 @@ import ( const NAME_K8S = "infiniband-k8s" type ProviderK8S struct { - config *rest.Config - client *kubernetes.Clientset - params *Params + config *rest.Config + client *kubernetes.Clientset + params *Params + accelerator accelerator.Discoverer } type Params struct { // NodeSelector (optional) specifies nodes participating in the topology NodeSelector map[string]string `mapstructure:"nodeSelector"` - // UseGPUCliqueLabel uses the GPU Operator's nvidia.com/gpu.clique node label - // as the accelerator domain ID instead of Topograph's node annotation. - UseGPUCliqueLabel bool `mapstructure:"useGpuCliqueLabel"` - // derived fields nodeListOpt *metav1.ListOptions } @@ -53,6 +49,12 @@ func LoaderK8S(ctx context.Context, config providers.Config) (providers.Provider if err != nil { return nil, httperr.NewError(http.StatusBadRequest, err.Error()) } + acceleratorDiscoverer, err := accelerator.NewKubernetesDiscoverer( + accelerator.SectionFromProviderParams(config.Params), + ) + if err != nil { + return nil, httperr.NewError(http.StatusBadRequest, err.Error()) + } cfg, err := rest.InClusterConfig() if err != nil { @@ -65,9 +67,10 @@ func LoaderK8S(ctx context.Context, config providers.Config) (providers.Provider } return &ProviderK8S{ - config: cfg, - client: client, - params: p, + config: cfg, + client: client, + params: p, + accelerator: acceleratorDiscoverer, }, nil } @@ -96,12 +99,20 @@ func (p *ProviderK8S) GenerateTopologyConfig(ctx context.Context, _ *int, cis [] return nil, httperr.NewError(http.StatusBadGateway, err.Error()) } - domainMap := topology.NewDomainMap() + targets := make([]accelerator.Target, 0, len(nodes.Items)) for _, node := range nodes.Items { - if clusterID := getGPUClusterID(node, p.params.UseGPUCliqueLabel); clusterID != "" { - domainMap.AddHost(clusterID, node.Name, node.Name) - } + targets = append(targets, accelerator.Target{ + InstanceID: node.Name, + HostName: node.Name, + Labels: node.Labels, + Annotations: node.Annotations, + }) } + assignments, err := p.accelerator.Discover(ctx, targets) + if err != nil { + return nil, httperr.NewError(http.StatusBadGateway, fmt.Sprintf("failed to discover accelerator domains: %v", err)) + } + domainMap := domainMapFromAssignments(assignments, targets) ibnetdiscover := NewIBNetDiscoverK8S(p.config, p.client) treeRoot, err := getIbTree(ctx, cis, ibnetdiscover) @@ -114,11 +125,3 @@ func (p *ProviderK8S) GenerateTopologyConfig(ctx context.Context, _ *int, cis [] Domains: domainMap, }, nil } - -func getGPUClusterID(node corev1.Node, useGPUCliqueLabel bool) string { - if useGPUCliqueLabel { - return strings.TrimSpace(node.Labels[topology.KeyNvidiaGPUClique]) - } - - return strings.TrimSpace(node.Annotations[topology.KeyGpuClusterID]) -} diff --git a/pkg/providers/infiniband/provider_k8s_test.go b/pkg/providers/infiniband/provider_k8s_test.go index 6208f36a..d49c67a8 100644 --- a/pkg/providers/infiniband/provider_k8s_test.go +++ b/pkg/providers/infiniband/provider_k8s_test.go @@ -1,5 +1,5 @@ /* - * Copyright 2025 NVIDIA CORPORATION + * Copyright 2025-2026 NVIDIA CORPORATION * SPDX-License-Identifier: Apache-2.0 */ @@ -9,80 +9,42 @@ import ( "testing" "github.com/stretchr/testify/require" - corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - - "github.com/NVIDIA/topograph/pkg/topology" ) func TestGetParameters(t *testing.T) { - testCases := []struct { - name string - params map[string]any - ret *Params - err string + tests := []struct { + name string + params map[string]any + labelSelector string + err string }{ + {name: "no parameters"}, { - name: "Case 1: no params", - params: nil, - ret: &Params{}, - }, - { - name: "Case 2: bad params", + name: "bad node selector", params: map[string]any{"nodeSelector": .1}, - err: "could not decode configuration: 1 error(s) decoding:\n\n* 'nodeSelector' expected a map, got 'float64'", - }, - { - name: "Case 3: valid input", - params: map[string]any{"nodeSelector": map[string]string{"key": "val"}}, - ret: &Params{ - NodeSelector: map[string]string{"key": "val"}, - nodeListOpt: &metav1.ListOptions{ - LabelSelector: "key=val", - }, - }, - }, - { - name: "Case 4: valid GPU clique label toggle", - params: map[string]any{"useGpuCliqueLabel": true}, - ret: &Params{ - UseGPUCliqueLabel: true, - }, + err: "could not decode configuration", }, { - name: "Case 5: valid GPU clique label toggle from string", - params: map[string]any{"useGpuCliqueLabel": "true"}, - ret: &Params{ - UseGPUCliqueLabel: true, - }, + name: "node selector", + params: map[string]any{"nodeSelector": map[string]string{"key": "val"}}, + labelSelector: "key=val", }, } - for _, tc := range testCases { - t.Run(tc.name, func(t *testing.T) { - p, err := getParameters(tc.params) - if len(tc.err) != 0 { - require.ErrorContains(t, err, tc.err) + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + params, err := getParameters(test.params) + if test.err != "" { + require.ErrorContains(t, err, test.err) + return + } + require.NoError(t, err) + if test.labelSelector == "" { + require.Nil(t, params.nodeListOpt) } else { - require.NoError(t, err) - require.Equal(t, tc.ret, p) + require.Equal(t, &metav1.ListOptions{LabelSelector: test.labelSelector}, params.nodeListOpt) } }) } } - -func TestGetGPUClusterID(t *testing.T) { - node := corev1.Node{ - ObjectMeta: metav1.ObjectMeta{ - Labels: map[string]string{ - topology.KeyNvidiaGPUClique: "label-domain.0", - }, - Annotations: map[string]string{ - topology.KeyGpuClusterID: "annotation-domain.0", - }, - }, - } - - require.Equal(t, "annotation-domain.0", getGPUClusterID(node, false)) - require.Equal(t, "label-domain.0", getGPUClusterID(node, true)) -}