From 8a495f3fa88173948b10e2167d33148cf4ed83b0 Mon Sep 17 00:00:00 2001 From: Alexander Korobkov Date: Wed, 10 Jun 2026 10:27:23 -0600 Subject: [PATCH 01/28] feat(helm): nico machine-a-tron --- helm/charts/nico-machine-a-tron/Chart.yaml | 12 + helm/charts/nico-machine-a-tron/README.md | 133 ++++++++++ .../templates/_helpers.tpl | 99 ++++++++ .../templates/certificate.yaml | 31 +++ .../templates/configmap.yaml | 81 +++++++ .../templates/deployment.yaml | 121 ++++++++++ .../templates/external-service.yaml | 22 ++ .../templates/metrics-service.yaml | 15 ++ .../nico-machine-a-tron/templates/pvc.yaml | 18 ++ .../nico-machine-a-tron/templates/rbac.yaml | 27 +++ .../templates/service-account.yaml | 7 + .../templates/service-monitor.yaml | 11 + .../templates/service.yaml | 22 ++ .../tests/certificate_test.yaml | 21 ++ .../tests/configmap_test.yaml | 43 ++++ helm/charts/nico-machine-a-tron/values.yaml | 227 ++++++++++++++++++ 16 files changed, 890 insertions(+) create mode 100644 helm/charts/nico-machine-a-tron/Chart.yaml create mode 100644 helm/charts/nico-machine-a-tron/README.md create mode 100644 helm/charts/nico-machine-a-tron/templates/_helpers.tpl create mode 100644 helm/charts/nico-machine-a-tron/templates/certificate.yaml create mode 100644 helm/charts/nico-machine-a-tron/templates/configmap.yaml create mode 100644 helm/charts/nico-machine-a-tron/templates/deployment.yaml create mode 100644 helm/charts/nico-machine-a-tron/templates/external-service.yaml create mode 100644 helm/charts/nico-machine-a-tron/templates/metrics-service.yaml create mode 100644 helm/charts/nico-machine-a-tron/templates/pvc.yaml create mode 100644 helm/charts/nico-machine-a-tron/templates/rbac.yaml create mode 100644 helm/charts/nico-machine-a-tron/templates/service-account.yaml create mode 100644 helm/charts/nico-machine-a-tron/templates/service-monitor.yaml create mode 100644 helm/charts/nico-machine-a-tron/templates/service.yaml create mode 100644 helm/charts/nico-machine-a-tron/tests/certificate_test.yaml create mode 100644 helm/charts/nico-machine-a-tron/tests/configmap_test.yaml create mode 100644 helm/charts/nico-machine-a-tron/values.yaml diff --git a/helm/charts/nico-machine-a-tron/Chart.yaml b/helm/charts/nico-machine-a-tron/Chart.yaml new file mode 100644 index 0000000000..5f6ceac8d6 --- /dev/null +++ b/helm/charts/nico-machine-a-tron/Chart.yaml @@ -0,0 +1,12 @@ +apiVersion: v2 +name: nico-machine-a-tron +description: Helm chart for Machine-A-Tron - a mock machine simulator for NICo development +type: application +version: 0.1.0 +appVersion: "latest" +keywords: + - nico + - machine-a-tron + - mock + - testing + - development diff --git a/helm/charts/nico-machine-a-tron/README.md b/helm/charts/nico-machine-a-tron/README.md new file mode 100644 index 0000000000..87aa04e108 --- /dev/null +++ b/helm/charts/nico-machine-a-tron/README.md @@ -0,0 +1,133 @@ +# Machine-A-Tron Helm Chart + +Helm chart for deploying Machine-A-Tron - a mock machine simulator for NICo development and testing. + +## Overview + +Machine-A-Tron creates simulated bare-metal machines that behave like real hosts, allowing developers to: +- Test NICo without physical hardware +- Simulate multiple hosts, DPUs, switches and power shelves +- Develop and debug the full machine lifecycle + +## Prerequisites + +- Kubernetes 1.19+ +- Helm 3.2.0+ +- cert-manager for TLS certificate management +- NICo API server deployed and accessible + +## Installation + +```bash +# Install with default values +helm install machine-a-tron ./helm/charts/nico-machine-a-tron + +# Install with custom values, eg. with 10 hosts, 2 DPUs each +helm install machine-a-tron ./helm/charts/nico-machine-a-tron \ + --set machines.config.hostCount=10 \ + --set machines.config.dpuPerHostCount=2 + +# Install with a values file +helm install machine-a-tron ./helm/charts/nico-machine-a-tron -f my-values.yaml +``` + +## Configuration + +### Key Configuration Options + +| Parameter | Description | Default | +|-----------|-------------|---------| +| `machineATron.nicoApiUrl` | URL of the NICo API server | `https://nico-api:443` | +| `machineATron.bmcMockPort` | Port for BMC mock service | `2000` | +| `machineATron.useSingleBmcMock` | Use header-based BMC routing (required for k8s) | `true` | +| `machineATron.usePxeApi` | Use PXE API instead of direct server | `true` | +| `machines.config.hostCount` | Number of mock hosts to create | `3` | +| `machines.config.dpuPerHostCount` | DPUs per host | `1` | +| `machines.config.vpcCount` | Number of VPCs to create | `0` | +| `persistence.enabled` | Enable persistent storage for machine state | `false` | + +### Machine Configuration + +The `machines` section supports **multiple named groups** with different hardware types: + +```yaml +machines: + # Remove default section if not needed + config: null + + # Dell hosts with 2 DPUs each + dell-hosts: + hwType: DellR760XA + hostCount: 10 + dpuPerHostCount: 2 + oobDhcpRelayAddress: "192.168.192.1" + adminDhcpRelayAddress: "192.168.176.1" + + # Gigabyte hosts with 1 DPU each + gigabyte-hosts: + hwType: GigabyteG493 + hostCount: 5 + dpuPerHostCount: 1 + oobDhcpRelayAddress: "192.168.192.1" + adminDhcpRelayAddress: "192.168.176.1" + + # Power shelves (no DPUs) + power-shelves: + hwType: LiteOnPowerShelf + hostCount: 2 + dpuPerHostCount: 0 + oobDhcpRelayAddress: "192.168.192.1" + adminDhcpRelayAddress: "192.168.176.1" +``` + +### Hardware Types + +Supported `hwType` values: +- `DellR760XA` (default) +- `DellR760` +- `GigabyteG493` +- `LiteOnPowerShelf` +- `NvidiaSwitchNd5200Ld` + +### NICo Site Configuration + +For Machine-A-Tron to work correctly, NICo must be configured to route Redfish calls through the mock: + +```toml +[site_explorer] +override_target_port = 2000 +override_target_host = "nico-machine-a-tron" # k8s service name +enabled = true +create_machines = true +``` + +## Persistence + +Enable persistence to preserve machine state across pod restarts: + +```yaml +persistence: + enabled: true + storageClass: "standard" + size: 1Gi +``` + +## External Access + +To expose the BMC mock externally (eg. for local development): + +```yaml +externalService: + enabled: true + type: LoadBalancer +``` + +## Monitoring + +Enable Prometheus ServiceMonitor: + +```yaml +serviceMonitor: + enabled: true + interval: 30s +``` diff --git a/helm/charts/nico-machine-a-tron/templates/_helpers.tpl b/helm/charts/nico-machine-a-tron/templates/_helpers.tpl new file mode 100644 index 0000000000..f209b26618 --- /dev/null +++ b/helm/charts/nico-machine-a-tron/templates/_helpers.tpl @@ -0,0 +1,99 @@ +{{/* +Allow the release namespace to be overridden for multi-namespace deployments. +*/}} +{{- define "nico-machine-a-tron.namespace" -}} +{{- default .Release.Namespace .Values.namespaceOverride | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Resource name prefix. Defaults to the chart name; override with nameOverride. +*/}} +{{- define "nico-machine-a-tron.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{- define "nico-machine-a-tron.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +What image to use: Use subchart-local image if defined, fall back on global +image. In devspace deployments, {{ include "nico-machine-a-tron.name" . }} gets its own image. +In other deployments, the main nico image contains all binaries. +*/}} +{{- define "nico-machine-a-tron.image" -}} +{{- if not (eq (toString (.Values.image.repository | default "")) "") }} +{{- .Values.image.repository }}:{{ .Values.image.tag | default "latest" }} +{{- else if .Values.global.image }} +{{- .Values.global.image.repository }}:{{ .Values.global.image.tag }} +{{- else }} +{{- "nico:latest" }} +{{- end }} +{{- end }} + +{{- define "nico-machine-a-tron.labels" -}} +helm.sh/chart: {{ include "nico-machine-a-tron.chart" . }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +app.kubernetes.io/part-of: site-controller +app.kubernetes.io/name: {{ include "nico-machine-a-tron.name" . }} +app.kubernetes.io/component: machine-a-tron +{{- end }} + +{{- define "nico-machine-a-tron.selectorLabels" -}} +app.kubernetes.io/name: {{ include "nico-machine-a-tron.name" . }} +app.kubernetes.io/component: machine-a-tron +{{- end }} + +{{- define "nico-machine-a-tron.certificateSpec" -}} +duration: {{ .global.certificate.duration }} +renewBefore: {{ .global.certificate.renewBefore }} +commonName: {{ printf "%s.%s.svc.cluster.local" (.cert.serviceName | default .svcName) (.cert.identityNamespace | default .namespace) }} +dnsNames: +{{- if .cert.dnsNames }} +{{- range .cert.dnsNames }} + - {{ . }} +{{- end }} +{{- else }} + - {{ printf "%s.%s.svc.cluster.local" (.cert.serviceName | default .svcName) (.cert.identityNamespace | default .namespace) }} +{{- if ne (toString .cert.includeShortDnsName) "false" }} + - {{ printf "%s.%s" (.cert.serviceName | default .svcName) (.cert.identityNamespace | default .namespace) }} +{{- end }} +{{- range .cert.extraDnsNames | default list }} + - {{ . }} +{{- end }} +{{- end }} +uris: +{{- if .cert.uris }} +{{- range .cert.uris }} + - {{ . }} +{{- end }} +{{- else }} + - {{ printf "spiffe://%s/%s/sa/%s" .global.spiffe.trustDomain (.cert.identityNamespace | default .namespace) (.cert.spiffeServiceName | default .cert.serviceName | default .svcName) }} +{{- range .cert.extraUris | default list }} + - {{ . }} +{{- end }} +{{- end }} +privateKey: + algorithm: {{ .global.certificate.privateKey.algorithm }} + size: {{ .global.certificate.privateKey.size }} +issuerRef: + kind: {{ .global.certificate.issuerRef.kind }} + name: {{ .global.certificate.issuerRef.name }} + group: {{ .global.certificate.issuerRef.group }} +secretName: {{ .name }} +{{- end }} + +{{- define "nico-machine-a-tron.serviceMonitorSpec" -}} +endpoints: + - honorLabels: false + interval: {{ .monitor.interval }} + port: {{ .port }} + scheme: http + scrapeTimeout: {{ .monitor.scrapeTimeout }} +namespaceSelector: + matchNames: + - {{ .namespace }} +selector: + matchLabels: + app.kubernetes.io/metrics: {{ .name }} +{{- end }} diff --git a/helm/charts/nico-machine-a-tron/templates/certificate.yaml b/helm/charts/nico-machine-a-tron/templates/certificate.yaml new file mode 100644 index 0000000000..093eb8176e --- /dev/null +++ b/helm/charts/nico-machine-a-tron/templates/certificate.yaml @@ -0,0 +1,31 @@ +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: {{ include "nico-machine-a-tron.name" . }}-certificate + namespace: {{ include "nico-machine-a-tron.namespace" . }} + labels: + {{- include "nico-machine-a-tron.labels" . | nindent 4 }} +spec: + duration: {{ .Values.global.certificate.duration }} + renewBefore: {{ .Values.global.certificate.renewBefore }} + secretName: {{ include "nico-machine-a-tron.name" . }}-certificate + commonName: {{ printf "%s.%s.svc.cluster.local" (include "nico-machine-a-tron.name" .) (include "nico-machine-a-tron.namespace" .) }} + dnsNames: + - {{ printf "%s.%s.svc.cluster.local" (include "nico-machine-a-tron.name" .) (include "nico-machine-a-tron.namespace" .) }} + - {{ printf "%s.%s" (include "nico-machine-a-tron.name" .) (include "nico-machine-a-tron.namespace" .) }} + - {{ printf "%s-bmc-mock.%s.svc.cluster.local" (include "nico-machine-a-tron.name" .) (include "nico-machine-a-tron.namespace" .) }} + {{- range .Values.certificate.extraDnsNames | default list }} + - {{ . }} + {{- end }} + uris: + - {{ printf "spiffe://%s/%s/sa/%s" .Values.global.spiffe.trustDomain (include "nico-machine-a-tron.namespace" .) (include "nico-machine-a-tron.name" .) }} + {{- range .Values.certificate.extraUris | default list }} + - {{ . }} + {{- end }} + privateKey: + algorithm: {{ .Values.global.certificate.privateKey.algorithm }} + size: {{ .Values.global.certificate.privateKey.size }} + issuerRef: + kind: {{ .Values.global.certificate.issuerRef.kind }} + name: {{ .Values.global.certificate.issuerRef.name }} + group: {{ .Values.global.certificate.issuerRef.group }} diff --git a/helm/charts/nico-machine-a-tron/templates/configmap.yaml b/helm/charts/nico-machine-a-tron/templates/configmap.yaml new file mode 100644 index 0000000000..a9201ab3d8 --- /dev/null +++ b/helm/charts/nico-machine-a-tron/templates/configmap.yaml @@ -0,0 +1,81 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "nico-machine-a-tron.name" . }}-config-files + namespace: {{ include "nico-machine-a-tron.namespace" . }} + labels: + {{- include "nico-machine-a-tron.labels" . | nindent 4 }} +data: + mat.toml: | + {{- if .Values.configFiles.matConfig }} + {{- .Values.configFiles.matConfig | nindent 4 }} + {{- else }} + # Machine-A-Tron configuration - generated by Helm + carbide_api_url = {{ .Values.machineATron.nicoApiUrl | quote }} + interface = {{ .Values.machineATron.interface | quote }} + tui_enabled = {{ .Values.machineATron.tuiEnabled }} + {{- if .Values.machineATron.logFile }} + log_file = {{ .Values.machineATron.logFile | quote }} + {{- end }} + bmc_mock_port = {{ .Values.machineATron.bmcMockPort }} + use_single_bmc_mock = {{ .Values.machineATron.useSingleBmcMock }} + mock_bmc_ssh_server = {{ .Values.machineATron.mockBmcSshServer }} + {{- if .Values.machineATron.mockBmcSshPort }} + mock_bmc_ssh_port = {{ .Values.machineATron.mockBmcSshPort }} + {{- end }} + {{- if .Values.machineATron.bmcMockHostTar }} + bmc_mock_host_tar = {{ .Values.machineATron.bmcMockHostTar | quote }} + {{- end }} + {{- if .Values.machineATron.bmcMockDpuTar }} + bmc_mock_dpu_tar = {{ .Values.machineATron.bmcMockDpuTar | quote }} + {{- end }} + use_pxe_api = {{ .Values.machineATron.usePxeApi }} + {{- if .Values.machineATron.pxeServerHost }} + pxe_server_host = {{ .Values.machineATron.pxeServerHost | quote }} + {{- end }} + {{- if .Values.machineATron.pxeServerPort }} + pxe_server_port = {{ .Values.machineATron.pxeServerPort | quote }} + {{- end }} + {{- if .Values.machineATron.configureBmcProxyHost }} + configure_carbide_bmc_proxy_host = {{ .Values.machineATron.configureBmcProxyHost | quote }} + {{- end }} + {{- if .Values.persistence.enabled }} + persist_dir = {{ .Values.machineATron.persistDir | quote }} + {{- end }} + cleanup_on_quit = {{ .Values.machineATron.cleanupOnQuit }} + register_expected_machines = {{ .Values.machineATron.registerExpectedMachines }} + {{- if .Values.machineATron.hostBmcPassword }} + host_bmc_password = {{ .Values.machineATron.hostBmcPassword | quote }} + {{- end }} + {{- if .Values.machineATron.dpuBmcPassword }} + dpu_bmc_password = {{ .Values.machineATron.dpuBmcPassword | quote }} + {{- end }} + api_refresh_interval = {{ .Values.machineATron.apiRefreshInterval | quote }} + + {{- range $sectionName, $section := .Values.machines }} + {{- if $section }} + + [machines.{{ $sectionName }}] + {{- if $section.hwType }} + hw_type = {{ $section.hwType | quote }} + {{- end }} + host_count = {{ $section.hostCount | default 1 }} + vpc_count = {{ $section.vpcCount | default 0 }} + subnets_per_vpc = {{ $section.subnetsPerVpc | default 0 }} + dpu_per_host_count = {{ $section.dpuPerHostCount | default 1 }} + dpu_reboot_delay = {{ $section.dpuRebootDelay | default 1 }} + host_reboot_delay = {{ $section.hostRebootDelay | default 1 }} + scout_run_interval = {{ $section.scoutRunInterval | default "60s" | quote }} + run_interval_working = {{ $section.runIntervalWorking | default "1s" | quote }} + run_interval_idle = {{ $section.runIntervalIdle | default "10s" | quote }} + network_status_run_interval = {{ $section.networkStatusRunInterval | default "20s" | quote }} + template_dir = {{ $section.templateDir | default "/opt/machine-a-tron/templates" | quote }} + oob_dhcp_relay_address = {{ $section.oobDhcpRelayAddress | default "192.168.192.1" | quote }} + admin_dhcp_relay_address = {{ $section.adminDhcpRelayAddress | default "192.168.176.1" | quote }} + {{- if $section.networkVirtualizationType }} + network_virtualization_type = {{ $section.networkVirtualizationType | quote }} + {{- end }} + dpus_in_nic_mode = {{ $section.dpusInNicMode | default false }} + {{- end }} + {{- end }} + {{- end }} diff --git a/helm/charts/nico-machine-a-tron/templates/deployment.yaml b/helm/charts/nico-machine-a-tron/templates/deployment.yaml new file mode 100644 index 0000000000..6a69ed7e1e --- /dev/null +++ b/helm/charts/nico-machine-a-tron/templates/deployment.yaml @@ -0,0 +1,121 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "nico-machine-a-tron.name" . }} + namespace: {{ include "nico-machine-a-tron.namespace" . }} + labels: + {{- include "nico-machine-a-tron.labels" . | nindent 4 }} + annotations: + {{- tpl (toYaml .Values.annotations) . | nindent 4 }} +spec: + replicas: {{ .Values.replicas }} + strategy: + type: Recreate + selector: + matchLabels: + {{- include "nico-machine-a-tron.selectorLabels" . | nindent 6 }} + template: + metadata: + labels: + {{- include "nico-machine-a-tron.labels" . | nindent 8 }} + annotations: + kubectl.kubernetes.io/default-container: {{ include "nico-machine-a-tron.name" . }} + spec: + automountServiceAccountToken: {{ .Values.automountServiceAccountToken }} + serviceAccountName: {{ include "nico-machine-a-tron.name" . }} + {{- with .Values.global.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + containers: + - name: {{ include "nico-machine-a-tron.name" . }} + image: "{{ include "nico-machine-a-tron.image" . }}" + imagePullPolicy: {{ (.Values.global.image).pullPolicy | default .Values.image.pullPolicy | default "IfNotPresent" }} + command: + {{- toYaml .Values.command | nindent 12 }} + env: + - name: VAULT_ROLE_ID + valueFrom: + secretKeyRef: + name: {{ .Values.envFrom.vaultApproleTokens.secretName }} + key: VAULT_ROLE_ID + optional: true + - name: VAULT_SECRET_ID + valueFrom: + secretKeyRef: + name: {{ .Values.envFrom.vaultApproleTokens.secretName }} + key: VAULT_SECRET_ID + optional: true + - name: VAULT_TOKEN + valueFrom: + secretKeyRef: + name: {{ .Values.envFrom.vaultToken.secretName }} + key: token + optional: true + - name: VAULT_ADDR + valueFrom: + configMapKeyRef: + name: {{ .Values.envFrom.vaultClusterInfo.configMapName }} + key: VAULT_SERVICE + optional: true + - name: MACHINE_A_TRON_CONFIG_PATH + value: "/etc/nico/machine-a-tron/mat.toml" + - name: FORGE_ROOT_CA_PATH + value: "/var/run/secrets/spiffe.io/ca.crt" + - name: CLIENT_CERT_PATH + value: "/var/run/secrets/spiffe.io/tls.crt" + - name: CLIENT_KEY_PATH + value: "/var/run/secrets/spiffe.io/tls.key" + - name: RUST_BACKTRACE + value: {{ .Values.env.RUST_BACKTRACE | quote }} + - name: RUST_LIB_BACKTRACE + value: {{ .Values.env.RUST_LIB_BACKTRACE | quote }} + {{- with .Values.extraEnv }} + {{- toYaml . | nindent 12 }} + {{- end }} + ports: + - name: redfish + containerPort: {{ .Values.service.bmcMock.port }} + - name: ssh + containerPort: {{ .Values.machineATron.mockBmcSshPort }} + - name: metrics + containerPort: {{ .Values.service.metrics.port }} + livenessProbe: + {{- toYaml .Values.livenessProbe | nindent 12 }} + readinessProbe: + {{- toYaml .Values.readinessProbe | nindent 12 }} + volumeMounts: + - name: {{ include "nico-machine-a-tron.name" . }}-config-files + mountPath: /etc/forge/machine-a-tron + - name: {{ include "nico-machine-a-tron.name" . }}-config-files + mountPath: /etc/nico/machine-a-tron + - name: spiffe + mountPath: "/var/run/secrets/spiffe.io" + readOnly: true + - name: nico-roots + mountPath: "/var/run/secrets/forge-roots" + readOnly: true + - name: nico-roots + mountPath: "/var/run/secrets/nico-roots" + readOnly: true + {{- if .Values.persistence.enabled }} + - name: data + mountPath: {{ .Values.machineATron.persistDir }} + {{- end }} + resources: + {{- toYaml .Values.resources | nindent 12 }} + volumes: + - name: {{ include "nico-machine-a-tron.name" . }}-config-files + configMap: + name: {{ include "nico-machine-a-tron.name" . }}-config-files + - name: spiffe + secret: + secretName: {{ include "nico-machine-a-tron.name" . }}-certificate + - name: nico-roots + secret: + secretName: nico-roots + {{- if .Values.persistence.enabled }} + - name: data + persistentVolumeClaim: + claimName: {{ include "nico-machine-a-tron.name" . }}-data + {{- end }} diff --git a/helm/charts/nico-machine-a-tron/templates/external-service.yaml b/helm/charts/nico-machine-a-tron/templates/external-service.yaml new file mode 100644 index 0000000000..18c07eddfd --- /dev/null +++ b/helm/charts/nico-machine-a-tron/templates/external-service.yaml @@ -0,0 +1,22 @@ +{{- if .Values.externalService.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: {{ include "nico-machine-a-tron.name" . }}-external + namespace: {{ include "nico-machine-a-tron.namespace" . }} + labels: + {{- include "nico-machine-a-tron.labels" . | nindent 4 }} + {{- with .Values.externalService.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + type: {{ .Values.externalService.type }} + externalTrafficPolicy: {{ .Values.externalService.externalTrafficPolicy }} + selector: + {{- include "nico-machine-a-tron.selectorLabels" . | nindent 4 }} + ports: + - port: {{ .Values.service.bmcMock.port }} + name: bmc-mock + protocol: TCP +{{- end }} diff --git a/helm/charts/nico-machine-a-tron/templates/metrics-service.yaml b/helm/charts/nico-machine-a-tron/templates/metrics-service.yaml new file mode 100644 index 0000000000..f7dafc029a --- /dev/null +++ b/helm/charts/nico-machine-a-tron/templates/metrics-service.yaml @@ -0,0 +1,15 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ include "nico-machine-a-tron.name" . }}-metrics + namespace: {{ include "nico-machine-a-tron.namespace" . }} + labels: + {{- include "nico-machine-a-tron.labels" . | nindent 4 }} + app.kubernetes.io/metrics: {{ include "nico-machine-a-tron.name" . }} +spec: + selector: + {{- include "nico-machine-a-tron.selectorLabels" . | nindent 4 }} + ports: + - port: {{ .Values.service.metrics.port }} + name: metrics + protocol: TCP diff --git a/helm/charts/nico-machine-a-tron/templates/pvc.yaml b/helm/charts/nico-machine-a-tron/templates/pvc.yaml new file mode 100644 index 0000000000..7ac4d8de03 --- /dev/null +++ b/helm/charts/nico-machine-a-tron/templates/pvc.yaml @@ -0,0 +1,18 @@ +{{- if .Values.persistence.enabled }} +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: {{ include "nico-machine-a-tron.name" . }}-data + namespace: {{ include "nico-machine-a-tron.namespace" . }} + labels: + {{- include "nico-machine-a-tron.labels" . | nindent 4 }} +spec: + accessModes: + {{- toYaml .Values.persistence.accessModes | nindent 4 }} + {{- if .Values.persistence.storageClass }} + storageClassName: {{ .Values.persistence.storageClass | quote }} + {{- end }} + resources: + requests: + storage: {{ .Values.persistence.size }} +{{- end }} diff --git a/helm/charts/nico-machine-a-tron/templates/rbac.yaml b/helm/charts/nico-machine-a-tron/templates/rbac.yaml new file mode 100644 index 0000000000..5cc13d0f4b --- /dev/null +++ b/helm/charts/nico-machine-a-tron/templates/rbac.yaml @@ -0,0 +1,27 @@ +kind: Role +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: {{ include "nico-machine-a-tron.name" . }} + namespace: {{ include "nico-machine-a-tron.namespace" . }} + labels: + {{- include "nico-machine-a-tron.labels" . | nindent 4 }} +rules: + - apiGroups: ["cert-manager.io"] + resources: ["certificaterequests"] + verbs: ["create"] +--- +kind: RoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: {{ include "nico-machine-a-tron.name" . }} + namespace: {{ include "nico-machine-a-tron.namespace" . }} + labels: + {{- include "nico-machine-a-tron.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: {{ include "nico-machine-a-tron.name" . }} +subjects: + - kind: ServiceAccount + name: {{ include "nico-machine-a-tron.name" . }} + namespace: {{ include "nico-machine-a-tron.namespace" . }} diff --git a/helm/charts/nico-machine-a-tron/templates/service-account.yaml b/helm/charts/nico-machine-a-tron/templates/service-account.yaml new file mode 100644 index 0000000000..d7a1fe8a92 --- /dev/null +++ b/helm/charts/nico-machine-a-tron/templates/service-account.yaml @@ -0,0 +1,7 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "nico-machine-a-tron.name" . }} + namespace: {{ include "nico-machine-a-tron.namespace" . }} + labels: + {{- include "nico-machine-a-tron.labels" . | nindent 4 }} diff --git a/helm/charts/nico-machine-a-tron/templates/service-monitor.yaml b/helm/charts/nico-machine-a-tron/templates/service-monitor.yaml new file mode 100644 index 0000000000..36c039fc0b --- /dev/null +++ b/helm/charts/nico-machine-a-tron/templates/service-monitor.yaml @@ -0,0 +1,11 @@ +{{- if .Values.serviceMonitor.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ include "nico-machine-a-tron.name" . }} + namespace: {{ include "nico-machine-a-tron.namespace" . }} + labels: + {{- include "nico-machine-a-tron.labels" . | nindent 4 }} +spec: + {{- include "nico-machine-a-tron.serviceMonitorSpec" (dict "name" (include "nico-machine-a-tron.name" .) "monitor" .Values.serviceMonitor "port" "metrics" "namespace" (include "nico-machine-a-tron.namespace" .)) | nindent 2 }} +{{- end }} diff --git a/helm/charts/nico-machine-a-tron/templates/service.yaml b/helm/charts/nico-machine-a-tron/templates/service.yaml new file mode 100644 index 0000000000..808e60db53 --- /dev/null +++ b/helm/charts/nico-machine-a-tron/templates/service.yaml @@ -0,0 +1,22 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ include "nico-machine-a-tron.name" . }}-bmc-mock + namespace: {{ include "nico-machine-a-tron.namespace" . }} + labels: + {{- include "nico-machine-a-tron.labels" . | nindent 4 }} + app: {{ include "nico-machine-a-tron.name" . }} +spec: + selector: + {{- include "nico-machine-a-tron.selectorLabels" . | nindent 4 }} + ports: + - port: {{ .Values.service.bmcMock.port }} + name: redfish + targetPort: redfish + protocol: TCP + {{- if .Values.machineATron.mockBmcSshServer }} + - port: {{ .Values.service.ssh.port }} + name: ssh + targetPort: ssh + protocol: TCP + {{- end }} diff --git a/helm/charts/nico-machine-a-tron/tests/certificate_test.yaml b/helm/charts/nico-machine-a-tron/tests/certificate_test.yaml new file mode 100644 index 0000000000..7b67933546 --- /dev/null +++ b/helm/charts/nico-machine-a-tron/tests/certificate_test.yaml @@ -0,0 +1,21 @@ +suite: certificate tests +templates: + - certificate.yaml +tests: + - it: should create a certificate with default values + asserts: + - isKind: + of: Certificate + - equal: + path: metadata.name + value: nico-machine-a-tron-certificate + + - it: should use custom certificate settings + set: + certificate: + serviceName: custom-service + extraDnsNames: + - extra.example.com + asserts: + - isKind: + of: Certificate diff --git a/helm/charts/nico-machine-a-tron/tests/configmap_test.yaml b/helm/charts/nico-machine-a-tron/tests/configmap_test.yaml new file mode 100644 index 0000000000..cba00bc599 --- /dev/null +++ b/helm/charts/nico-machine-a-tron/tests/configmap_test.yaml @@ -0,0 +1,43 @@ +suite: configmap tests +templates: + - configmap.yaml +tests: + - it: should create configmap with default machine config + asserts: + - isKind: + of: ConfigMap + - equal: + path: metadata.name + value: nico-machine-a-tron-config-files + - matchRegex: + path: data["mat.toml"] + pattern: "host_count = 3" + - matchRegex: + path: data["mat.toml"] + pattern: "use_single_bmc_mock = true" + + - it: should configure custom host count + set: + machines: + config: + hostCount: 10 + dpuPerHostCount: 2 + asserts: + - matchRegex: + path: data["mat.toml"] + pattern: "host_count = 10" + - matchRegex: + path: data["mat.toml"] + pattern: "dpu_per_host_count = 2" + + - it: should allow config override + set: + configFiles: + matConfig: | + carbide_api_url = "https://custom-api:443" + [machines.custom] + host_count = 5 + asserts: + - matchRegex: + path: data["mat.toml"] + pattern: "custom-api" diff --git a/helm/charts/nico-machine-a-tron/values.yaml b/helm/charts/nico-machine-a-tron/values.yaml new file mode 100644 index 0000000000..e98a6fb339 --- /dev/null +++ b/helm/charts/nico-machine-a-tron/values.yaml @@ -0,0 +1,227 @@ +## ============================================================================= +## nico-machine-a-tron - Mock machine simulator for NICo development/testing +## ============================================================================= + +global: + certificate: + duration: 720h0m0s + renewBefore: 360h0m0s + privateKey: + algorithm: ECDSA + size: 384 + issuerRef: + kind: ClusterIssuer + name: vault-nico-issuer + group: cert-manager.io + spiffe: + trustDomain: nico.local + labels: + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/part-of: site-controller + +image: + repository: "" + tag: "" + pullPolicy: IfNotPresent + +namespaceOverride: "" + +replicas: 1 +automountServiceAccountToken: true +annotations: + configmap.reloader.stakater.com/reload: '{{ include "nico-machine-a-tron.name" . }}-config-files' + +resources: + limits: + cpu: 4 + memory: 4Gi + requests: + cpu: 500m + memory: 1Gi + +service: + bmcMock: + port: 1266 + ssh: + port: 22 + targetPort: 2222 + metrics: + port: 9090 + +certificate: + serviceName: "" + spiffeServiceName: "" + identityNamespace: "" + dnsNames: [] + uris: [] + ## Extra DNS name for the bmc-mock service is added automatically in _helpers.tpl + extraDnsNames: [] + extraUris: [] + +command: + - /bin/sh + - -c + - | + if [ -x /opt/machine-a-tron/bin/machine-a-tron ]; then + exec /opt/machine-a-tron/bin/machine-a-tron + elif [ -x /opt/nico/machine-a-tron ]; then + exec /opt/nico/machine-a-tron + else + exec /opt/carbide/machine-a-tron + fi + +env: + RUST_BACKTRACE: "full" + RUST_LIB_BACKTRACE: "0" + +extraEnv: [] + +envFrom: + vaultApproleTokens: + secretName: nico-vault-approle-tokens + vaultToken: + secretName: nico-vault-token + vaultClusterInfo: + configMapName: vault-cluster-info + +livenessProbe: + tcpSocket: + port: redfish + initialDelaySeconds: 30 + periodSeconds: 30 + failureThreshold: 3 + successThreshold: 1 + timeoutSeconds: 10 + +readinessProbe: + tcpSocket: + port: redfish + initialDelaySeconds: 5 + periodSeconds: 10 + failureThreshold: 3 + successThreshold: 1 + timeoutSeconds: 5 + +serviceMonitor: + enabled: false + interval: 30s + scrapeTimeout: 25s + +## Machine-a-tron specific configuration +machineATron: + ## URL of the NICo API server + nicoApiUrl: "https://nico-api.nico-system.svc.cluster.local:1079" + + ## Network interface for BMC mock IP binding (not used in k8s single-mock mode) + interface: "NOTUSED" + + ## TUI is disabled in kubernetes - logs go to stdout + tuiEnabled: false + + ## Log file path (empty = stdout) + logFile: "" + + ## BMC mock port (1266 is the devspace default) + bmcMockPort: 1266 + + ## Use single BMC mock with header-based routing (required for k8s) + useSingleBmcMock: true + + ## Enable mock BMC SSH server + mockBmcSshServer: true + mockBmcSshPort: 2222 + + ## BMC mock tar files (templates for Redfish responses) + bmcMockHostTar: "/opt/machine-a-tron/dell_poweredge_r750.tar.gz" + bmcMockDpuTar: "/opt/machine-a-tron/nvidia_dpu.tar.gz" + + ## Use PXE API instead of direct PXE server + usePxeApi: true + + ## PXE server settings (only needed if usePxeApi: false) + pxeServerHost: "" + pxeServerPort: "" + + ## If set, machine-a-tron will configure NICo to use this as bmc_proxy host + ## Combined with bmcMockPort to form host:port + configureBmcProxyHost: "" + + ## Directory for persisting machine state between restarts + persistDir: "/tmp/machine-a-tron-data" + + ## Delete created machines from API on quit + cleanupOnQuit: false + + ## Auto-register mock hosts as expected machines + registerExpectedMachines: true + + ## Override BMC passwords (leave empty for factory defaults) + hostBmcPassword: "" + dpuBmcPassword: "" + + ## API refresh interval + apiRefreshInterval: "2s" + +## Machine configuration sections +## Each named section defines a group of mock machines with specific settings +## You can define multiple sections with different hardware types +## Example: +## machines: +## dell-hosts: +## hwType: DellR760XA +## hostCount: 10 +## dpuPerHostCount: 2 +## ... +## gigabyte-hosts: +## hwType: GigabyteG493 +## hostCount: 5 +## dpuPerHostCount: 1 +## ... +machines: + ## Default section - can be renamed or have additional sections added + config: + ## Hardware type: DellR760XA, DellR760, GigabyteG493, LiteOnPowerShelf, NvidiaSwitchNd5200Ld + hwType: DellR760XA + ## Number of host machines to create + hostCount: 10 + ## Number of VPCs to create + vpcCount: 0 + ## Number of subnets per VPC + subnetsPerVpc: 0 + ## Number of DPUs per host + dpuPerHostCount: 2 + ## Reboot delay in seconds (short for dev/test) + dpuRebootDelay: 1 + hostRebootDelay: 1 + ## State machine intervals (faster for dev) + scoutRunInterval: "60s" + runIntervalWorking: "1s" + runIntervalIdle: "10s" + networkStatusRunInterval: "20s" + ## Template directory for BMC mock data + templateDir: "/opt/machine-a-tron/templates" + ## DHCP relay addresses (must match site config) + oobDhcpRelayAddress: "192.168.192.1" + adminDhcpRelayAddress: "192.168.176.1" + ## Network virtualization type: etv, etv_nvue, or fnn + networkVirtualizationType: "" + ## DPUs run in NIC mode (no PXE boot) + dpusInNicMode: false + +## Persistence volume for machine state +persistence: + enabled: false + storageClass: "" + size: 1Gi + accessModes: + - ReadWriteOnce + +externalService: + enabled: false + type: LoadBalancer + externalTrafficPolicy: Local + annotations: {} + +configFiles: + ## Override the entire mat.toml config (leave empty to use generated config) + matConfig: "" From 06de0646dde783fe9b52e8cb057b3d4f4791f1dd Mon Sep 17 00:00:00 2001 From: Alexander Korobkov Date: Wed, 10 Jun 2026 10:47:01 -0600 Subject: [PATCH 02/28] fix(helm): nico machine-a-tron command --- helm/charts/nico-machine-a-tron/values.yaml | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/helm/charts/nico-machine-a-tron/values.yaml b/helm/charts/nico-machine-a-tron/values.yaml index e98a6fb339..29c99d7734 100644 --- a/helm/charts/nico-machine-a-tron/values.yaml +++ b/helm/charts/nico-machine-a-tron/values.yaml @@ -61,14 +61,7 @@ certificate: command: - /bin/sh - -c - - | - if [ -x /opt/machine-a-tron/bin/machine-a-tron ]; then - exec /opt/machine-a-tron/bin/machine-a-tron - elif [ -x /opt/nico/machine-a-tron ]; then - exec /opt/nico/machine-a-tron - else - exec /opt/carbide/machine-a-tron - fi + - /opt/machine-a-tron/bin/machine-a-tron env: RUST_BACKTRACE: "full" From 65457c05ad0bf4302854c5af82f139fb974c136a Mon Sep 17 00:00:00 2001 From: Alexander Korobkov Date: Wed, 10 Jun 2026 12:55:49 -0600 Subject: [PATCH 03/28] fix(helm): nico machine-a-tron config --- dev/deployment/devspace/machine-a-tron.yaml | 3 -- helm/charts/nico-machine-a-tron/README.md | 41 +++++++++++-------- .../templates/configmap.yaml | 13 ------ helm/charts/nico-machine-a-tron/values.yaml | 24 ++++------- 4 files changed, 31 insertions(+), 50 deletions(-) diff --git a/dev/deployment/devspace/machine-a-tron.yaml b/dev/deployment/devspace/machine-a-tron.yaml index d825bcd748..135a4b860e 100644 --- a/dev/deployment/devspace/machine-a-tron.yaml +++ b/dev/deployment/devspace/machine-a-tron.yaml @@ -12,9 +12,6 @@ data: carbide_api_url = "https://nico-api.nico-system.svc.cluster.local:1079" interface = "NOTUSED" tui_enabled = false - use_pxe_api = true - bmc_mock_host_tar = "/opt/machine-a-tron/dell_poweredge_r750.tar.gz" - bmc_mock_dpu_tar = "/opt/machine-a-tron/nvidia_dpu.tar.gz" bmc_mock_port = 1266 use_single_bmc_mock = true mock_bmc_ssh_server = true diff --git a/helm/charts/nico-machine-a-tron/README.md b/helm/charts/nico-machine-a-tron/README.md index 87aa04e108..54e876caf3 100644 --- a/helm/charts/nico-machine-a-tron/README.md +++ b/helm/charts/nico-machine-a-tron/README.md @@ -37,12 +37,11 @@ helm install machine-a-tron ./helm/charts/nico-machine-a-tron -f my-values.yaml | Parameter | Description | Default | |-----------|-------------|---------| -| `machineATron.nicoApiUrl` | URL of the NICo API server | `https://nico-api:443` | -| `machineATron.bmcMockPort` | Port for BMC mock service | `2000` | +| `machineATron.nicoApiUrl` | URL of the NICo API server | `https://nico-api.nico-system.svc.cluster.local:1079` | +| `machineATron.bmcMockPort` | Port for BMC mock service | `1266` | | `machineATron.useSingleBmcMock` | Use header-based BMC routing (required for k8s) | `true` | -| `machineATron.usePxeApi` | Use PXE API instead of direct server | `true` | -| `machines.config.hostCount` | Number of mock hosts to create | `3` | -| `machines.config.dpuPerHostCount` | DPUs per host | `1` | +| `machines.config.hostCount` | Number of mock hosts to create | `10` | +| `machines.config.dpuPerHostCount` | DPUs per host | `2` | | `machines.config.vpcCount` | Number of VPCs to create | `0` | | `persistence.enabled` | Enable persistent storage for machine state | `false` | @@ -57,15 +56,15 @@ machines: # Dell hosts with 2 DPUs each dell-hosts: - hwType: DellR760XA + hwType: dell_poweredge_r750 hostCount: 10 dpuPerHostCount: 2 oobDhcpRelayAddress: "192.168.192.1" adminDhcpRelayAddress: "192.168.176.1" - # Gigabyte hosts with 1 DPU each - gigabyte-hosts: - hwType: GigabyteG493 + # NVIDIA DGX H100 hosts + dgx-hosts: + hwType: nvidia_dgx_h100 hostCount: 5 dpuPerHostCount: 1 oobDhcpRelayAddress: "192.168.192.1" @@ -73,7 +72,7 @@ machines: # Power shelves (no DPUs) power-shelves: - hwType: LiteOnPowerShelf + hwType: liteon_power_shelf hostCount: 2 dpuPerHostCount: 0 oobDhcpRelayAddress: "192.168.192.1" @@ -82,12 +81,18 @@ machines: ### Hardware Types -Supported `hwType` values: -- `DellR760XA` (default) -- `DellR760` -- `GigabyteG493` -- `LiteOnPowerShelf` -- `NvidiaSwitchNd5200Ld` +Supported `hwType` values (from `HostHardwareType` enum in `crates/bmc-mock/src/lib.rs`): + +- `dell_poweredge_r750` (default) +- `wiwynn_gb200_nvl` +- `lenovo_gb300_nvl` +- `nvidia_dgx_gb300` +- `supermicro_gb300_nvl` +- `liteon_power_shelf` +- `nvidia_switch_nd5200_ld` +- `nvidia_dgx_h100` +- `generic_ami` +- `generic_supermicro` ### NICo Site Configuration @@ -95,8 +100,8 @@ For Machine-A-Tron to work correctly, NICo must be configured to route Redfish c ```toml [site_explorer] -override_target_port = 2000 -override_target_host = "nico-machine-a-tron" # k8s service name +override_target_port = 1266 +override_target_host = "nico-machine-a-tron-bmc-mock" # k8s service name enabled = true create_machines = true ``` diff --git a/helm/charts/nico-machine-a-tron/templates/configmap.yaml b/helm/charts/nico-machine-a-tron/templates/configmap.yaml index a9201ab3d8..1ca86b6486 100644 --- a/helm/charts/nico-machine-a-tron/templates/configmap.yaml +++ b/helm/charts/nico-machine-a-tron/templates/configmap.yaml @@ -23,19 +23,6 @@ data: {{- if .Values.machineATron.mockBmcSshPort }} mock_bmc_ssh_port = {{ .Values.machineATron.mockBmcSshPort }} {{- end }} - {{- if .Values.machineATron.bmcMockHostTar }} - bmc_mock_host_tar = {{ .Values.machineATron.bmcMockHostTar | quote }} - {{- end }} - {{- if .Values.machineATron.bmcMockDpuTar }} - bmc_mock_dpu_tar = {{ .Values.machineATron.bmcMockDpuTar | quote }} - {{- end }} - use_pxe_api = {{ .Values.machineATron.usePxeApi }} - {{- if .Values.machineATron.pxeServerHost }} - pxe_server_host = {{ .Values.machineATron.pxeServerHost | quote }} - {{- end }} - {{- if .Values.machineATron.pxeServerPort }} - pxe_server_port = {{ .Values.machineATron.pxeServerPort | quote }} - {{- end }} {{- if .Values.machineATron.configureBmcProxyHost }} configure_carbide_bmc_proxy_host = {{ .Values.machineATron.configureBmcProxyHost | quote }} {{- end }} diff --git a/helm/charts/nico-machine-a-tron/values.yaml b/helm/charts/nico-machine-a-tron/values.yaml index 29c99d7734..d98f2f69d6 100644 --- a/helm/charts/nico-machine-a-tron/values.yaml +++ b/helm/charts/nico-machine-a-tron/values.yaml @@ -124,17 +124,6 @@ machineATron: mockBmcSshServer: true mockBmcSshPort: 2222 - ## BMC mock tar files (templates for Redfish responses) - bmcMockHostTar: "/opt/machine-a-tron/dell_poweredge_r750.tar.gz" - bmcMockDpuTar: "/opt/machine-a-tron/nvidia_dpu.tar.gz" - - ## Use PXE API instead of direct PXE server - usePxeApi: true - - ## PXE server settings (only needed if usePxeApi: false) - pxeServerHost: "" - pxeServerPort: "" - ## If set, machine-a-tron will configure NICo to use this as bmc_proxy host ## Combined with bmcMockPort to form host:port configureBmcProxyHost: "" @@ -161,20 +150,23 @@ machineATron: ## Example: ## machines: ## dell-hosts: -## hwType: DellR760XA +## hwType: dell_poweredge_r750 ## hostCount: 10 ## dpuPerHostCount: 2 ## ... -## gigabyte-hosts: -## hwType: GigabyteG493 +## dgx-hosts: +## hwType: nvidia_dgx_h100 ## hostCount: 5 ## dpuPerHostCount: 1 ## ... machines: ## Default section - can be renamed or have additional sections added config: - ## Hardware type: DellR760XA, DellR760, GigabyteG493, LiteOnPowerShelf, NvidiaSwitchNd5200Ld - hwType: DellR760XA + ## Hardware type (serde names from HostHardwareType enum): + ## dell_poweredge_r750 (default), wiwynn_gb200_nvl, lenovo_gb300_nvl, + ## nvidia_dgx_gb300, supermicro_gb300_nvl, liteon_power_shelf, + ## nvidia_switch_nd5200_ld, nvidia_dgx_h100, generic_ami, generic_supermicro + hwType: dell_poweredge_r750 ## Number of host machines to create hostCount: 10 ## Number of VPCs to create From 39a1d0367006bbe5145e55bae5a84a04450c740a Mon Sep 17 00:00:00 2001 From: Alexander Korobkov Date: Wed, 10 Jun 2026 13:02:02 -0600 Subject: [PATCH 04/28] fix(helm): nico machine-a-tron requirements --- helm/charts/nico-machine-a-tron/README.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/helm/charts/nico-machine-a-tron/README.md b/helm/charts/nico-machine-a-tron/README.md index 54e876caf3..56194bb509 100644 --- a/helm/charts/nico-machine-a-tron/README.md +++ b/helm/charts/nico-machine-a-tron/README.md @@ -11,8 +11,8 @@ Machine-A-Tron creates simulated bare-metal machines that behave like real hosts ## Prerequisites -- Kubernetes 1.19+ -- Helm 3.2.0+ +- Kubernetes 1.27+ +- Helm 3.12+ - cert-manager for TLS certificate management - NICo API server deployed and accessible @@ -53,7 +53,7 @@ The `machines` section supports **multiple named groups** with different hardwar machines: # Remove default section if not needed config: null - + # Dell hosts with 2 DPUs each dell-hosts: hwType: dell_poweredge_r750 @@ -61,7 +61,7 @@ machines: dpuPerHostCount: 2 oobDhcpRelayAddress: "192.168.192.1" adminDhcpRelayAddress: "192.168.176.1" - + # NVIDIA DGX H100 hosts dgx-hosts: hwType: nvidia_dgx_h100 @@ -69,7 +69,7 @@ machines: dpuPerHostCount: 1 oobDhcpRelayAddress: "192.168.192.1" adminDhcpRelayAddress: "192.168.176.1" - + # Power shelves (no DPUs) power-shelves: hwType: liteon_power_shelf From 747dc659896ab792cc646dcf1091216ea66edb44 Mon Sep 17 00:00:00 2001 From: Alexander Korobkov Date: Wed, 10 Jun 2026 13:13:17 -0600 Subject: [PATCH 05/28] fix(helm): nico machine-a-tron machines config --- helm/charts/nico-machine-a-tron/README.md | 3 --- helm/charts/nico-machine-a-tron/values.yaml | 6 ++++-- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/helm/charts/nico-machine-a-tron/README.md b/helm/charts/nico-machine-a-tron/README.md index 56194bb509..743bfe1372 100644 --- a/helm/charts/nico-machine-a-tron/README.md +++ b/helm/charts/nico-machine-a-tron/README.md @@ -51,9 +51,6 @@ The `machines` section supports **multiple named groups** with different hardwar ```yaml machines: - # Remove default section if not needed - config: null - # Dell hosts with 2 DPUs each dell-hosts: hwType: dell_poweredge_r750 diff --git a/helm/charts/nico-machine-a-tron/values.yaml b/helm/charts/nico-machine-a-tron/values.yaml index d98f2f69d6..9c939318f2 100644 --- a/helm/charts/nico-machine-a-tron/values.yaml +++ b/helm/charts/nico-machine-a-tron/values.yaml @@ -160,8 +160,10 @@ machineATron: ## dpuPerHostCount: 1 ## ... machines: - ## Default section - can be renamed or have additional sections added - config: + ## Machine group section. The name "dell-hosts" is arbitrary - you can rename it + ## or add more sections (e.g., "dgx-hosts", "power-shelves") to simulate different + ## hardware types. Each section creates a separate group of mock machines. + dell-hosts: ## Hardware type (serde names from HostHardwareType enum): ## dell_poweredge_r750 (default), wiwynn_gb200_nvl, lenovo_gb300_nvl, ## nvidia_dgx_gb300, supermicro_gb300_nvl, liteon_power_shelf, From e753dc7ef8706b047e5b42107ff5e91dbdb6604b Mon Sep 17 00:00:00 2001 From: Alexander Korobkov Date: Mon, 22 Jun 2026 12:18:09 -0600 Subject: [PATCH 06/28] feat(helm): nico machine-a-tron graceful shutdown support --- helm/charts/nico-machine-a-tron/README.md | 22 ++++++++++++++++--- .../templates/deployment.yaml | 1 + helm/charts/nico-machine-a-tron/values.yaml | 5 +++++ 3 files changed, 25 insertions(+), 3 deletions(-) diff --git a/helm/charts/nico-machine-a-tron/README.md b/helm/charts/nico-machine-a-tron/README.md index 743bfe1372..c9724110b8 100644 --- a/helm/charts/nico-machine-a-tron/README.md +++ b/helm/charts/nico-machine-a-tron/README.md @@ -40,9 +40,11 @@ helm install machine-a-tron ./helm/charts/nico-machine-a-tron -f my-values.yaml | `machineATron.nicoApiUrl` | URL of the NICo API server | `https://nico-api.nico-system.svc.cluster.local:1079` | | `machineATron.bmcMockPort` | Port for BMC mock service | `1266` | | `machineATron.useSingleBmcMock` | Use header-based BMC routing (required for k8s) | `true` | -| `machines.config.hostCount` | Number of mock hosts to create | `10` | -| `machines.config.dpuPerHostCount` | DPUs per host | `2` | -| `machines.config.vpcCount` | Number of VPCs to create | `0` | +| `machineATron.cleanupOnQuit` | Delete created machines from API on quit | `false` | +| `terminationGracePeriodSeconds` | Graceful shutdown timeout | `60` | +| `machines..hostCount` | Number of mock hosts to create | `10` | +| `machines..dpuPerHostCount` | DPUs per host | `2` | +| `machines..hwType` | Hardware type to simulate | `dell_poweredge_r750` | | `persistence.enabled` | Enable persistent storage for machine state | `false` | ### Machine Configuration @@ -103,6 +105,20 @@ enabled = true create_machines = true ``` +## Graceful Shutdown + +The chart supports graceful shutdown when pods are deleted. This gives machine-a-tron +time to clean up resources: + +```yaml +# Shutdown timeout +terminationGracePeriodSeconds: 60 + +machineATron: + # Set to true to delete created machines from NICo API on shutdown + cleanupOnQuit: true +``` + ## Persistence Enable persistence to preserve machine state across pod restarts: diff --git a/helm/charts/nico-machine-a-tron/templates/deployment.yaml b/helm/charts/nico-machine-a-tron/templates/deployment.yaml index 6a69ed7e1e..86758584b9 100644 --- a/helm/charts/nico-machine-a-tron/templates/deployment.yaml +++ b/helm/charts/nico-machine-a-tron/templates/deployment.yaml @@ -23,6 +23,7 @@ spec: spec: automountServiceAccountToken: {{ .Values.automountServiceAccountToken }} serviceAccountName: {{ include "nico-machine-a-tron.name" . }} + terminationGracePeriodSeconds: {{ .Values.terminationGracePeriodSeconds }} {{- with .Values.global.imagePullSecrets }} imagePullSecrets: {{- toYaml . | nindent 8 }} diff --git a/helm/charts/nico-machine-a-tron/values.yaml b/helm/charts/nico-machine-a-tron/values.yaml index 9c939318f2..478121cbf4 100644 --- a/helm/charts/nico-machine-a-tron/values.yaml +++ b/helm/charts/nico-machine-a-tron/values.yaml @@ -28,6 +28,11 @@ namespaceOverride: "" replicas: 1 automountServiceAccountToken: true + +## Graceful shutdown timeout (seconds) +## Allows machine-a-tron to clean up resources when pod is deleted +terminationGracePeriodSeconds: 60 + annotations: configmap.reloader.stakater.com/reload: '{{ include "nico-machine-a-tron.name" . }}-config-files' From a79f6a43251f82289de2a7c8f16a8011377fa808 Mon Sep 17 00:00:00 2001 From: Alexander Korobkov Date: Mon, 22 Jun 2026 13:55:25 -0600 Subject: [PATCH 07/28] feat(helm): nico machine-a-tron readme --- helm/charts/nico-machine-a-tron/README.md | 36 +++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/helm/charts/nico-machine-a-tron/README.md b/helm/charts/nico-machine-a-tron/README.md index c9724110b8..ad2cc17b72 100644 --- a/helm/charts/nico-machine-a-tron/README.md +++ b/helm/charts/nico-machine-a-tron/README.md @@ -78,6 +78,42 @@ machines: adminDhcpRelayAddress: "192.168.176.1" ``` +#### DHCP Relay Addresses + +The `oobDhcpRelayAddress` and `adminDhcpRelayAddress` values tell NICo which network +segment to allocate IPs from. These must match gateway addresses of networks configured +in NICo's site config. + +**Note:** All machine groups typically share the same relay addresses since they connect +to the same OOB and admin networks. The relay address is used only for network segment +matching - NICo handles IP allocation from the configured prefix. + +#### NICo Network Configuration + +Machine-A-Tron requires corresponding networks to be configured in NICo's +`nico-api-site-config.toml`. The relay addresses must match the gateway of these networks: + +```toml +# OOB network for BMC management (matches oobDhcpRelayAddress) +[networks.oob-bmc] +type = "underlay" +prefix = "192.168.192.0/24" # Any prefix size works (/16, /23, /24, etc.) +gateway = "192.168.192.1" # This is the oobDhcpRelayAddress +mtu = 1500 +reserve_first = 2 + +# Admin network for host provisioning (matches adminDhcpRelayAddress) +[networks.admin] +type = "admin" +prefix = "192.168.176.0/24" +gateway = "192.168.176.1" # This is the adminDhcpRelayAddress +mtu = 9000 +reserve_first = 5 +``` + +NICo uses PostgreSQL's `<<=` operator to match relay addresses against configured +network prefixes, so the network size is fully configurable based on your environment. + ### Hardware Types Supported `hwType` values (from `HostHardwareType` enum in `crates/bmc-mock/src/lib.rs`): From 2ccb016ef4fd5cd4cc90ced6b4bf01550caf1dd7 Mon Sep 17 00:00:00 2001 From: Alexander Korobkov Date: Mon, 22 Jun 2026 14:10:11 -0600 Subject: [PATCH 08/28] fix(helm): nico machine-a-tron umbrella chart metadata --- helm/Chart.yaml | 3 +++ helm/values.yaml | 10 ++++++++++ 2 files changed, 13 insertions(+) diff --git a/helm/Chart.yaml b/helm/Chart.yaml index 12a69869c4..e5e07b5738 100644 --- a/helm/Chart.yaml +++ b/helm/Chart.yaml @@ -44,3 +44,6 @@ dependencies: - name: unbound version: "0.1.0" condition: unbound.enabled + - name: nico-machine-a-tron + version: "0.1.0" + condition: nico-machine-a-tron.enabled diff --git a/helm/values.yaml b/helm/values.yaml index 92e7a81bdc..1ea49ea43f 100644 --- a/helm/values.yaml +++ b/helm/values.yaml @@ -165,3 +165,13 @@ nico-ssh-console-rs: ## --------------------------------------------------------------------------- unbound: enabled: false + +## --------------------------------------------------------------------------- +## nico-machine-a-tron - Mock machine simulator (development/testing only) +## Simulates bare-metal machines, DPUs, switches and power shelves for +## testing NICo without physical hardware. NOT for production use. +## Deploy standalone for dev environments: +## helm install mat ./helm/charts/nico-machine-a-tron -n nico-system ... +## --------------------------------------------------------------------------- +nico-machine-a-tron: + enabled: false From 9e2c5478c55e1361e28c7f311c56a11ed532606f Mon Sep 17 00:00:00 2001 From: Alexander Korobkov Date: Mon, 22 Jun 2026 14:39:01 -0600 Subject: [PATCH 09/28] fix(helm): nico machine-a-tron chart review --- .../templates/_helpers.tpl | 2 +- .../templates/certificate.yaml | 27 +++---------------- .../templates/configmap.yaml | 2 +- .../templates/external-service.yaml | 1 + .../tests/configmap_test.yaml | 12 ++++----- helm/charts/nico-machine-a-tron/values.yaml | 6 ++--- 6 files changed, 15 insertions(+), 35 deletions(-) diff --git a/helm/charts/nico-machine-a-tron/templates/_helpers.tpl b/helm/charts/nico-machine-a-tron/templates/_helpers.tpl index f209b26618..b27a1fdf47 100644 --- a/helm/charts/nico-machine-a-tron/templates/_helpers.tpl +++ b/helm/charts/nico-machine-a-tron/templates/_helpers.tpl @@ -24,7 +24,7 @@ In other deployments, the main nico image contains all binaries. {{- define "nico-machine-a-tron.image" -}} {{- if not (eq (toString (.Values.image.repository | default "")) "") }} {{- .Values.image.repository }}:{{ .Values.image.tag | default "latest" }} -{{- else if .Values.global.image }} +{{- else if and .Values.global.image (not (eq (toString (.Values.global.image.repository | default "")) "")) (not (eq (toString (.Values.global.image.tag | default "")) "")) }} {{- .Values.global.image.repository }}:{{ .Values.global.image.tag }} {{- else }} {{- "nico:latest" }} diff --git a/helm/charts/nico-machine-a-tron/templates/certificate.yaml b/helm/charts/nico-machine-a-tron/templates/certificate.yaml index 093eb8176e..3cf32793f3 100644 --- a/helm/charts/nico-machine-a-tron/templates/certificate.yaml +++ b/helm/charts/nico-machine-a-tron/templates/certificate.yaml @@ -6,26 +6,7 @@ metadata: labels: {{- include "nico-machine-a-tron.labels" . | nindent 4 }} spec: - duration: {{ .Values.global.certificate.duration }} - renewBefore: {{ .Values.global.certificate.renewBefore }} - secretName: {{ include "nico-machine-a-tron.name" . }}-certificate - commonName: {{ printf "%s.%s.svc.cluster.local" (include "nico-machine-a-tron.name" .) (include "nico-machine-a-tron.namespace" .) }} - dnsNames: - - {{ printf "%s.%s.svc.cluster.local" (include "nico-machine-a-tron.name" .) (include "nico-machine-a-tron.namespace" .) }} - - {{ printf "%s.%s" (include "nico-machine-a-tron.name" .) (include "nico-machine-a-tron.namespace" .) }} - - {{ printf "%s-bmc-mock.%s.svc.cluster.local" (include "nico-machine-a-tron.name" .) (include "nico-machine-a-tron.namespace" .) }} - {{- range .Values.certificate.extraDnsNames | default list }} - - {{ . }} - {{- end }} - uris: - - {{ printf "spiffe://%s/%s/sa/%s" .Values.global.spiffe.trustDomain (include "nico-machine-a-tron.namespace" .) (include "nico-machine-a-tron.name" .) }} - {{- range .Values.certificate.extraUris | default list }} - - {{ . }} - {{- end }} - privateKey: - algorithm: {{ .Values.global.certificate.privateKey.algorithm }} - size: {{ .Values.global.certificate.privateKey.size }} - issuerRef: - kind: {{ .Values.global.certificate.issuerRef.kind }} - name: {{ .Values.global.certificate.issuerRef.name }} - group: {{ .Values.global.certificate.issuerRef.group }} + {{- $bmcMockDns := printf "%s-bmc-mock.%s.svc.cluster.local" (include "nico-machine-a-tron.name" .) (include "nico-machine-a-tron.namespace" .) }} + {{- $extraDns := prepend (.Values.certificate.extraDnsNames | default list) $bmcMockDns }} + {{- $certWithExtra := merge (dict "extraDnsNames" $extraDns) .Values.certificate }} + {{- include "nico-machine-a-tron.certificateSpec" (dict "name" (printf "%s-certificate" (include "nico-machine-a-tron.name" .)) "svcName" (include "nico-machine-a-tron.name" .) "namespace" (include "nico-machine-a-tron.namespace" .) "cert" $certWithExtra "global" .Values.global) | nindent 2 }} diff --git a/helm/charts/nico-machine-a-tron/templates/configmap.yaml b/helm/charts/nico-machine-a-tron/templates/configmap.yaml index 1ca86b6486..cf4fc9a6bb 100644 --- a/helm/charts/nico-machine-a-tron/templates/configmap.yaml +++ b/helm/charts/nico-machine-a-tron/templates/configmap.yaml @@ -17,7 +17,7 @@ data: {{- if .Values.machineATron.logFile }} log_file = {{ .Values.machineATron.logFile | quote }} {{- end }} - bmc_mock_port = {{ .Values.machineATron.bmcMockPort }} + bmc_mock_port = {{ .Values.service.bmcMock.port }} use_single_bmc_mock = {{ .Values.machineATron.useSingleBmcMock }} mock_bmc_ssh_server = {{ .Values.machineATron.mockBmcSshServer }} {{- if .Values.machineATron.mockBmcSshPort }} diff --git a/helm/charts/nico-machine-a-tron/templates/external-service.yaml b/helm/charts/nico-machine-a-tron/templates/external-service.yaml index 18c07eddfd..4bde72e1dd 100644 --- a/helm/charts/nico-machine-a-tron/templates/external-service.yaml +++ b/helm/charts/nico-machine-a-tron/templates/external-service.yaml @@ -18,5 +18,6 @@ spec: ports: - port: {{ .Values.service.bmcMock.port }} name: bmc-mock + targetPort: redfish protocol: TCP {{- end }} diff --git a/helm/charts/nico-machine-a-tron/tests/configmap_test.yaml b/helm/charts/nico-machine-a-tron/tests/configmap_test.yaml index cba00bc599..035aa06603 100644 --- a/helm/charts/nico-machine-a-tron/tests/configmap_test.yaml +++ b/helm/charts/nico-machine-a-tron/tests/configmap_test.yaml @@ -11,7 +11,7 @@ tests: value: nico-machine-a-tron-config-files - matchRegex: path: data["mat.toml"] - pattern: "host_count = 3" + pattern: "host_count = 10" - matchRegex: path: data["mat.toml"] pattern: "use_single_bmc_mock = true" @@ -19,16 +19,16 @@ tests: - it: should configure custom host count set: machines: - config: - hostCount: 10 - dpuPerHostCount: 2 + dell-hosts: + hostCount: 20 + dpuPerHostCount: 4 asserts: - matchRegex: path: data["mat.toml"] - pattern: "host_count = 10" + pattern: "host_count = 20" - matchRegex: path: data["mat.toml"] - pattern: "dpu_per_host_count = 2" + pattern: "dpu_per_host_count = 4" - it: should allow config override set: diff --git a/helm/charts/nico-machine-a-tron/values.yaml b/helm/charts/nico-machine-a-tron/values.yaml index 478121cbf4..d05330f6b7 100644 --- a/helm/charts/nico-machine-a-tron/values.yaml +++ b/helm/charts/nico-machine-a-tron/values.yaml @@ -119,10 +119,8 @@ machineATron: ## Log file path (empty = stdout) logFile: "" - ## BMC mock port (1266 is the devspace default) - bmcMockPort: 1266 - ## Use single BMC mock with header-based routing (required for k8s) + ## Port is configured via service.bmcMock.port useSingleBmcMock: true ## Enable mock BMC SSH server @@ -130,7 +128,7 @@ machineATron: mockBmcSshPort: 2222 ## If set, machine-a-tron will configure NICo to use this as bmc_proxy host - ## Combined with bmcMockPort to form host:port + ## Combined with service.bmcMock.port to form host:port configureBmcProxyHost: "" ## Directory for persisting machine state between restarts From cb70b8662ea6b7363cf98b6b07ec390fb1223f87 Mon Sep 17 00:00:00 2001 From: Alexander Korobkov Date: Tue, 23 Jun 2026 12:39:40 -0600 Subject: [PATCH 10/28] feat(helm): nico machine-a-tron large scale --- helm/charts/nico-machine-a-tron/README.md | 445 +++++++++++++----- .../templates/bmc-proxy-configmap.yaml | 50 ++ .../templates/bmc-proxy-deployment.yaml | 82 ++++ .../templates/metallb-pool.yaml | 71 +++ .../templates/metallb-services.yaml | 103 ++++ helm/charts/nico-machine-a-tron/values.yaml | 117 ++++- helm/templates/NOTES.txt | 3 + 7 files changed, 761 insertions(+), 110 deletions(-) create mode 100644 helm/charts/nico-machine-a-tron/templates/bmc-proxy-configmap.yaml create mode 100644 helm/charts/nico-machine-a-tron/templates/bmc-proxy-deployment.yaml create mode 100644 helm/charts/nico-machine-a-tron/templates/metallb-pool.yaml create mode 100644 helm/charts/nico-machine-a-tron/templates/metallb-services.yaml diff --git a/helm/charts/nico-machine-a-tron/README.md b/helm/charts/nico-machine-a-tron/README.md index ad2cc17b72..8f84d6166d 100644 --- a/helm/charts/nico-machine-a-tron/README.md +++ b/helm/charts/nico-machine-a-tron/README.md @@ -1,163 +1,362 @@ # Machine-A-Tron Helm Chart -Helm chart for deploying Machine-A-Tron - a mock machine simulator for NICo development and testing. +Helm chart for deploying Machine-A-Tron - a mock machine simulator for NICo testing. ## Overview -Machine-A-Tron creates simulated bare-metal machines that behave like real hosts, allowing developers to: +Machine-A-Tron creates simulated bare-metal machines that behave like real hosts, allowing you to: - Test NICo without physical hardware - Simulate multiple hosts, DPUs, switches and power shelves -- Develop and debug the full machine lifecycle +- Perform load testing at scale (up to 5000 machines) +- Run simulations alongside real hardware -## Prerequisites +## Deployment Modes -- Kubernetes 1.27+ -- Helm 3.12+ -- cert-manager for TLS certificate management -- NICo API server deployed and accessible +Machine-A-Tron supports **two mutually exclusive deployment modes**: -## Installation +| Mode | Use Case | Real HW Compatible | Network Setup | +|------|----------|-------------------|---------------| +| **Override Mode** | Development environments | No | Simple - single endpoint | +| **MetalLB Mode** | Load testing with real HW | Yes | Complex - per-BMC IPs | -```bash -# Install with default values -helm install machine-a-tron ./helm/charts/nico-machine-a-tron +### Choosing a Mode -# Install with custom values, eg. with 10 hosts, 2 DPUs each -helm install machine-a-tron ./helm/charts/nico-machine-a-tron \ - --set machines.config.hostCount=10 \ - --set machines.config.dpuPerHostCount=2 +```mermaid +flowchart TD + Q1{Need to run alongside
real hardware?} + Q1 -->|No| M1[Override Mode] + Q1 -->|Yes| M2[MetalLB Mode] -# Install with a values file -helm install machine-a-tron ./helm/charts/nico-machine-a-tron -f my-values.yaml + M1 --> D1[Dev/test environments
Simulated machines only] + M2 --> D2[Load testing
Mixed real + simulated] ``` -## Configuration +--- -### Key Configuration Options +## Mode 1: Override Mode (Development) -| Parameter | Description | Default | -|-----------|-------------|---------| -| `machineATron.nicoApiUrl` | URL of the NICo API server | `https://nico-api.nico-system.svc.cluster.local:1079` | -| `machineATron.bmcMockPort` | Port for BMC mock service | `1266` | -| `machineATron.useSingleBmcMock` | Use header-based BMC routing (required for k8s) | `true` | -| `machineATron.cleanupOnQuit` | Delete created machines from API on quit | `false` | -| `terminationGracePeriodSeconds` | Graceful shutdown timeout | `60` | -| `machines..hostCount` | Number of mock hosts to create | `10` | -| `machines..dpuPerHostCount` | DPUs per host | `2` | -| `machines..hwType` | Hardware type to simulate | `dell_poweredge_r750` | -| `persistence.enabled` | Enable persistent storage for machine state | `false` | +**Use this for development environments where only simulated machines are needed.** -### Machine Configuration +In this mode, NICo's Site-Explorer is configured to redirect ALL Redfish calls to +machine-a-tron via `override_target_host`. This is simple but **incompatible with +real hardware** since all BMC traffic goes to the mock. -The `machines` section supports **multiple named groups** with different hardware types: +### Architecture -```yaml -machines: - # Dell hosts with 2 DPUs each - dell-hosts: - hwType: dell_poweredge_r750 - hostCount: 10 - dpuPerHostCount: 2 - oobDhcpRelayAddress: "192.168.192.1" - adminDhcpRelayAddress: "192.168.176.1" +```mermaid +flowchart LR + subgraph NICo["NICo Cluster"] + SE[Site-Explorer] + API[NICo API] + end - # NVIDIA DGX H100 hosts - dgx-hosts: - hwType: nvidia_dgx_h100 - hostCount: 5 - dpuPerHostCount: 1 - oobDhcpRelayAddress: "192.168.192.1" - adminDhcpRelayAddress: "192.168.176.1" + subgraph MAT["Machine-A-Tron"] + BMC[BMC Mock
Single endpoint] + end - # Power shelves (no DPUs) - power-shelves: - hwType: liteon_power_shelf - hostCount: 2 - dpuPerHostCount: 0 - oobDhcpRelayAddress: "192.168.192.1" - adminDhcpRelayAddress: "192.168.176.1" + SE -->|"ALL Redfish calls
override_target_host"| BMC + BMC <-->|"DHCP, status"| API +``` + +### Setup + +**1. Deploy Machine-A-Tron:** + +```bash +helm upgrade --install nico ./helm \ + --namespace nico-system \ + --set nico-machine-a-tron.enabled=true \ + --set nico-machine-a-tron.machines.dell-hosts.hostCount=10 \ + --set nico-machine-a-tron.machines.dell-hosts.dpuPerHostCount=2 ``` -#### DHCP Relay Addresses +**2. Configure NICo Site Config:** -The `oobDhcpRelayAddress` and `adminDhcpRelayAddress` values tell NICo which network -segment to allocate IPs from. These must match gateway addresses of networks configured -in NICo's site config. +```toml +# nico-api-site-config.toml +[site_explorer] +enabled = true +create_machines = true -**Note:** All machine groups typically share the same relay addresses since they connect -to the same OOB and admin networks. The relay address is used only for network segment -matching - NICo handles IP allocation from the configured prefix. +# Redirect ALL Redfish calls to machine-a-tron +override_target_host = "nico-machine-a-tron-bmc-mock" +override_target_port = 1266 +``` -#### NICo Network Configuration +**3. Configure NICo Networks:** -Machine-A-Tron requires corresponding networks to be configured in NICo's -`nico-api-site-config.toml`. The relay addresses must match the gateway of these networks: +The DHCP relay addresses in machine-a-tron must match NICo network configuration: ```toml -# OOB network for BMC management (matches oobDhcpRelayAddress) +# OOB network for BMC management [networks.oob-bmc] type = "underlay" -prefix = "192.168.192.0/24" # Any prefix size works (/16, /23, /24, etc.) -gateway = "192.168.192.1" # This is the oobDhcpRelayAddress -mtu = 1500 -reserve_first = 2 +prefix = "192.168.192.0/24" +gateway = "192.168.192.1" # matches oobDhcpRelayAddress -# Admin network for host provisioning (matches adminDhcpRelayAddress) +# Admin network for host provisioning [networks.admin] type = "admin" prefix = "192.168.176.0/24" -gateway = "192.168.176.1" # This is the adminDhcpRelayAddress -mtu = 9000 -reserve_first = 5 +gateway = "192.168.176.1" # matches adminDhcpRelayAddress +``` + +### Pros and Cons (Override Mode) + +**Pros:** +- Simple setup - no MetalLB required +- Works in any Kubernetes cluster + +**Cons:** +- Cannot use real hardware - all Redfish calls go to mock + +--- + +## Mode 2: MetalLB Mode (Large-Scale Testing) + +**Use this for dev or load testing environments where simulated machines run alongside real hardware.** + +In this mode, each simulated BMC gets a dedicated external IP via MetalLB. NICo +Site-Explorer connects directly to these IPs, allowing simulated and real machines +to coexist on the same NICo instance. + +### Architecture (MetalLB Mode) + +```mermaid +flowchart LR + subgraph NICo["NICo Cluster"] + SE[Site-Explorer] + end + + subgraph Real["Real Hardware"] + R1[Real BMC
10.50.0.10] + R2[Real BMC
10.50.0.11] + end + + subgraph MetalLB["MetalLB (Simulated BMCs)"] + LB1["10.100.0.2"] + LB2["10.100.0.3"] + LBN["... 13500 IPs"] + end + + subgraph MAT["Machine-A-Tron Pod"] + NGINX["NGINX Proxy"] + BMC["BMC Mock Registry"] + end + + SE -->|"Redfish"| R1 + SE -->|"Redfish"| R2 + SE -->|"Redfish"| LB1 + SE -->|"Redfish"| LB2 + + LB1 --> NGINX + LB2 --> NGINX + LBN --> NGINX + NGINX -->|"Forwarded header"| BMC +``` + +### Prerequisites (MetalLB Mode) + +- MetalLB installed and configured +- BGP mode required for scale 2000+ services +- Dedicated IP range for simulated BMCs (separate from real hardware) + +### Setup (MetalLB Mode) + +**1. Deploy with MetalLB BMC Proxy enabled:** + +```bash +helm upgrade --install nico ./helm \ + --namespace nico-system \ + --set nico-machine-a-tron.enabled=true \ + --set nico-machine-a-tron.bmcProxy.enabled=true \ + --set nico-machine-a-tron.bmcProxy.ipRange="10.100.0.0-10.100.7.254" \ + --set nico-machine-a-tron.bmcProxy.bgp.enabled=true \ + --set nico-machine-a-tron.machines.dell-hosts.hostCount=100 \ + --set nico-machine-a-tron.machines.dell-hosts.dpuPerHostCount=2 \ + --set nico-machine-a-tron.machines.dell-hosts.oobDhcpRelayAddress="10.100.0.1" +``` + +**2. NICo Site Config - NO override_target:** + +```toml +# nico-api-site-config.toml +[site_explorer] +enabled = true +create_machines = true + +# DO NOT set override_target_host - let NICo connect to actual BMC IPs +# override_target_host = ... # NOT SET! +# override_target_port = ... # NOT SET! +``` + +**3. Configure Separate Networks:** + +```toml +# Real hardware OOB network +[networks.real-oob] +type = "underlay" +prefix = "10.50.0.0/24" +gateway = "10.50.0.1" + +# Simulated BMC network (MetalLB range) +[networks.simulated-oob] +type = "underlay" +prefix = "10.100.0.0/21" +gateway = "10.100.0.1" # matches oobDhcpRelayAddress for simulated machines +``` + +### IP Allocation - How It Works + +NICo allocates IPs **sequentially** via gRPC API: +- First machine gets `.2` (after gateway `.1`) +- Second machine gets `.3` +- And so on... + +This means MetalLB services with sequential IPs will match NICo allocations +**as long as the subnet is dedicated to machine-a-tron**. + +### Scale Limits + +| Scale | Subnet | Max BMCs | BGP Required | +|-------|--------|----------|--------------| +| Small | /21 | 2048 | No (L2 OK for testing) | +| Medium | /20 | 4096 | Recommended | +| Large | /19 | 8192 | Yes | +| Max | /18 | 16384 | Yes | + +The chart enforces a **hard limit of 16384 BMCs** (/18 subnet) to protect cluster stability. + +If you exceed the limit, the chart **fails**: ``` +SCALE LIMIT EXCEEDED: 16500 BMCs requested, maximum is 16384 (/18 subnet). +``` + +### BGP Configuration (Required for Large Scale) + +For deployments with 2000+ services, BGP mode with route aggregation is **required**: + +```yaml +bmcProxy: + enabled: true + ipRange: "10.100.0.2-10.100.15.254" # /20 range + + ipPool: + create: true + l2Advertisement: false # Disable L2 for BGP + + bgp: + enabled: true + aggregationLength: 20 # Announce single /20 instead of 4000+ /32s +``` + +**Why BGP?** Without `aggregationLength`, MetalLB advertises each service IP as an +individual /32 route. With thousands of services, this overwhelms upstream router control planes. + +### Pros and Cons (MetalLB Mode) + +**Pros:** +- Compatible with real hardware +- Enables realistic load testing + +**Cons:** +- Requires MetalLB with BGP for large scale +- More complex network setup +- Requires dedicated IP range + +--- + +## Configuration Reference + +### Key Parameters + +| Parameter | Description | Default | +|-----------|-------------|---------| +| `machineATron.nicoApiUrl` | NICo API server URL | `https://nico-api.nico-system.svc.cluster.local:1079` | +| `machineATron.cleanupOnQuit` | Delete machines from API on shutdown | `false` | +| `machines..hostCount` | Number of hosts to simulate | `10` | +| `machines..dpuPerHostCount` | DPUs per host | `2` | +| `machines..hwType` | Hardware type | `dell_poweredge_r750` | +| `machines..oobDhcpRelayAddress` | BMC network gateway | `10.100.0.1` | +| `machines..adminDhcpRelayAddress` | Admin network gateway | `10.101.0.1` | +| `bmcProxy.enabled` | Enable MetalLB mode | `false` | +| `bmcProxy.ipRange` | MetalLB IP pool range | `""` | +| `bmcProxy.bgp.enabled` | Enable BGP advertisement | `false` | +| `terminationGracePeriodSeconds` | Shutdown timeout | `60` | + +### Machine Groups + +Configure multiple machine groups with different hardware types: + +```yaml +machines: + gb300-hosts: + hwType: supermicro_gb300_nvl + hostCount: 100 + dpuPerHostCount: 2 + oobDhcpRelayAddress: "10.100.0.1" + adminDhcpRelayAddress: "10.101.0.1" -NICo uses PostgreSQL's `<<=` operator to match relay addresses against configured -network prefixes, so the network size is fully configurable based on your environment. + switches: + hwType: nvidia_switch_nd5200_ld + hostCount: 10 + dpuPerHostCount: 0 + oobDhcpRelayAddress: "10.100.0.1" + adminDhcpRelayAddress: "10.101.0.1" + + power-shelves: + hwType: liteon_power_shelf + hostCount: 5 + dpuPerHostCount: 0 + oobDhcpRelayAddress: "10.100.0.1" + adminDhcpRelayAddress: "10.101.0.1" +``` -### Hardware Types +### Supported Hardware Types -Supported `hwType` values (from `HostHardwareType` enum in `crates/bmc-mock/src/lib.rs`): +From `HostHardwareType` enum in `crates/bmc-mock/src/lib.rs`: - `dell_poweredge_r750` (default) +- `supermicro_gb300_nvl` +- `nvidia_dgx_gb300` +- `nvidia_dgx_h100` - `wiwynn_gb200_nvl` - `lenovo_gb300_nvl` -- `nvidia_dgx_gb300` -- `supermicro_gb300_nvl` - `liteon_power_shelf` - `nvidia_switch_nd5200_ld` -- `nvidia_dgx_h100` - `generic_ami` - `generic_supermicro` -### NICo Site Configuration +### Service Count Calculation -For Machine-A-Tron to work correctly, NICo must be configured to route Redfish calls through the mock: +The chart auto-calculates LoadBalancer services (MetalLB mode) from machine config: -```toml -[site_explorer] -override_target_port = 1266 -override_target_host = "nico-machine-a-tron-bmc-mock" # k8s service name -enabled = true -create_machines = true ``` +totalBMCs = Σ(hostCount + hostCount × dpuPerHostCount) for all groups +``` + +| Config | Calculation | Services | +|--------|-------------|----------| +| 10 hosts, 2 DPUs | 10 + 20 | 30 | +| 100 hosts, 2 DPUs | 100 + 200 | 300 | +| 4500 hosts, 2 DPUs | 4500 + 9000 | 13500 | -## Graceful Shutdown +--- -The chart supports graceful shutdown when pods are deleted. This gives machine-a-tron -time to clean up resources: +## Additional Features + +### Graceful Shutdown ```yaml -# Shutdown timeout -terminationGracePeriodSeconds: 60 +terminationGracePeriodSeconds: 60 # Seconds to wait before SIGKILL machineATron: - # Set to true to delete created machines from NICo API on shutdown - cleanupOnQuit: true + cleanupOnQuit: true # Delete machines from NICo API ``` -## Persistence +### Persistence -Enable persistence to preserve machine state across pod restarts: +Preserve machine state across restarts: ```yaml persistence: @@ -166,22 +365,50 @@ persistence: size: 1Gi ``` -## External Access +### Monitoring -To expose the BMC mock externally (eg. for local development): +Enable Prometheus metrics: ```yaml -externalService: +serviceMonitor: enabled: true - type: LoadBalancer + interval: 30s ``` -## Monitoring +--- -Enable Prometheus ServiceMonitor: +## Troubleshooting -```yaml -serviceMonitor: - enabled: true - interval: 30s +### Pod CrashLoopBackOff + +Check logs: +```bash +kubectl -n nico-system logs deployment/nico-machine-a-tron ``` + +Common causes: +- NICo API unreachable - check `machineATron.nicoApiUrl` +- Certificate issues - check cert-manager +- Missing network config in NICo + +### MetalLB Services Not Getting IPs + +```bash +# Check MetalLB speaker logs +kubectl -n metallb-system logs -l app=metallb,component=speaker + +# Verify IPAddressPool exists +kubectl -n metallb-system get ipaddresspools + +# Check BGPAdvertisement +kubectl -n metallb-system get bgpadvertisements +``` + +### Scale Limit Error + +If you see: +``` +SCALE LIMIT EXCEEDED: 16500 BMCs requested, maximum is 16384 (/18 subnet). +``` + +**Solution:** Reduce `hostCount` or `dpuPerHostCount` to stay within the /18 limit (16384 BMCs max). diff --git a/helm/charts/nico-machine-a-tron/templates/bmc-proxy-configmap.yaml b/helm/charts/nico-machine-a-tron/templates/bmc-proxy-configmap.yaml new file mode 100644 index 0000000000..18c7b16467 --- /dev/null +++ b/helm/charts/nico-machine-a-tron/templates/bmc-proxy-configmap.yaml @@ -0,0 +1,50 @@ +{{- if .Values.bmcProxy.enabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "nico-machine-a-tron.name" . }}-bmc-proxy-config + namespace: {{ include "nico-machine-a-tron.namespace" . }} + labels: + {{- include "nico-machine-a-tron.labels" . | nindent 4 }} + app.kubernetes.io/component: bmc-proxy +data: + nginx.conf: | + worker_processes auto; + + events { + worker_connections 65535; + use epoll; + multi_accept on; + } + + http { + upstream machine_a_tron_backend { + server {{ include "nico-machine-a-tron.name" . }}:{{ .Values.service.bmcMock.port }}; + keepalive 256; + } + + server { + listen 443 ssl; + + ssl_certificate /etc/ssl/certs/tls.crt; + ssl_certificate_key /etc/ssl/private/tls.key; + ssl_protocols TLSv1.2 TLSv1.3; + ssl_ciphers HIGH:!annul:!MD5; + ssl_session_cache shared:SSL:10m; + ssl_session_timeout 10m; + + location / { + proxy_pass https://machine_a_tron_backend; + proxy_http_version 1.1; + proxy_set_header Connection ""; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + # Key header: tells machine-a-tron which BMC IP was requested + proxy_set_header Forwarded "host=$server_addr"; + proxy_ssl_verify off; + } + } + } +{{- end }} diff --git a/helm/charts/nico-machine-a-tron/templates/bmc-proxy-deployment.yaml b/helm/charts/nico-machine-a-tron/templates/bmc-proxy-deployment.yaml new file mode 100644 index 0000000000..bdf14ec496 --- /dev/null +++ b/helm/charts/nico-machine-a-tron/templates/bmc-proxy-deployment.yaml @@ -0,0 +1,82 @@ +{{- if .Values.bmcProxy.enabled }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "nico-machine-a-tron.name" . }}-bmc-proxy + namespace: {{ include "nico-machine-a-tron.namespace" . }} + labels: + {{- include "nico-machine-a-tron.labels" . | nindent 4 }} + app.kubernetes.io/component: bmc-proxy +spec: + replicas: {{ .Values.bmcProxy.nginx.replicas | default 1 }} + selector: + matchLabels: + {{- include "nico-machine-a-tron.selectorLabels" . | nindent 6 }} + app.kubernetes.io/component: bmc-proxy + template: + metadata: + labels: + {{- include "nico-machine-a-tron.selectorLabels" . | nindent 8 }} + app.kubernetes.io/component: bmc-proxy + annotations: + checksum/nginx-config: {{ include (print $.Template.BasePath "/bmc-proxy-configmap.yaml") . | sha256sum }} + spec: + ## Pod affinity: schedule nginx on the same node as machine-a-tron + ## to minimize network hops between proxy and backend + affinity: + podAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchLabels: + {{- include "nico-machine-a-tron.selectorLabels" . | nindent 18 }} + topologyKey: kubernetes.io/hostname + containers: + - name: nginx + image: {{ .Values.bmcProxy.nginx.image | default "nginx:alpine" }} + imagePullPolicy: IfNotPresent + ports: + - name: https + containerPort: 443 + protocol: TCP + volumeMounts: + - name: nginx-config + mountPath: /etc/nginx/nginx.conf + subPath: nginx.conf + readOnly: true + - name: tls-certs + mountPath: /etc/ssl/certs + readOnly: true + - name: tls-key + mountPath: /etc/ssl/private + readOnly: true + {{- with .Values.bmcProxy.nginx.resources }} + resources: + {{- toYaml . | nindent 12 }} + {{- end }} + livenessProbe: + tcpSocket: + port: https + initialDelaySeconds: 10 + periodSeconds: 30 + readinessProbe: + tcpSocket: + port: https + initialDelaySeconds: 5 + periodSeconds: 10 + volumes: + - name: nginx-config + configMap: + name: {{ include "nico-machine-a-tron.name" . }}-bmc-proxy-config + - name: tls-certs + secret: + secretName: {{ .Values.bmcProxy.tls.secretName | default (printf "%s-certificate" (include "nico-machine-a-tron.name" .)) }} + items: + - key: tls.crt + path: tls.crt + - name: tls-key + secret: + secretName: {{ .Values.bmcProxy.tls.secretName | default (printf "%s-certificate" (include "nico-machine-a-tron.name" .)) }} + items: + - key: tls.key + path: tls.key +{{- end }} diff --git a/helm/charts/nico-machine-a-tron/templates/metallb-pool.yaml b/helm/charts/nico-machine-a-tron/templates/metallb-pool.yaml new file mode 100644 index 0000000000..4f739d4898 --- /dev/null +++ b/helm/charts/nico-machine-a-tron/templates/metallb-pool.yaml @@ -0,0 +1,71 @@ +{{- if .Values.bmcProxy.enabled }} +{{- if .Values.bmcProxy.ipPool.create }} +{{- if .Values.bmcProxy.ipRange }} +--- +apiVersion: metallb.io/v1beta1 +kind: IPAddressPool +metadata: + name: {{ include "nico-machine-a-tron.name" . }}-bmc-pool + namespace: {{ .Values.bmcProxy.ipPool.namespace | default "metallb-system" }} + labels: + {{- include "nico-machine-a-tron.labels" . | nindent 4 }} +spec: + addresses: + - {{ .Values.bmcProxy.ipRange }} + {{- if .Values.bmcProxy.ipPool.autoAssign }} + autoAssign: {{ .Values.bmcProxy.ipPool.autoAssign }} + {{- end }} +{{- if .Values.bmcProxy.bgp.enabled }} +--- +## BGP Advertisement with route aggregation +## CRITICAL: aggregationLength prevents advertising 2000+ individual /32 routes +## which would overwhelm upstream router control planes +apiVersion: metallb.io/v1beta1 +kind: BGPAdvertisement +metadata: + name: {{ include "nico-machine-a-tron.name" . }}-bmc-bgp + namespace: {{ .Values.bmcProxy.ipPool.namespace | default "metallb-system" }} + labels: + {{- include "nico-machine-a-tron.labels" . | nindent 4 }} +spec: + ipAddressPools: + - {{ include "nico-machine-a-tron.name" . }}-bmc-pool + ## Aggregate all service IPs into a single /18 announcement + ## This is MANDATORY for ultra-scale (2000+ services) + aggregationLength: {{ .Values.bmcProxy.bgp.aggregationLength | default 18 }} + {{- if .Values.bmcProxy.bgp.aggregationLengthV6 }} + aggregationLengthV6: {{ .Values.bmcProxy.bgp.aggregationLengthV6 }} + {{- end }} + {{- if .Values.bmcProxy.bgp.localPref }} + localPref: {{ .Values.bmcProxy.bgp.localPref }} + {{- end }} + {{- if .Values.bmcProxy.bgp.communities }} + communities: + {{- toYaml .Values.bmcProxy.bgp.communities | nindent 4 }} + {{- end }} + {{- if .Values.bmcProxy.bgp.peers }} + peers: + {{- toYaml .Values.bmcProxy.bgp.peers | nindent 4 }} + {{- end }} +{{- else if .Values.bmcProxy.ipPool.l2Advertisement }} +--- +## L2 Advertisement - only suitable for small scale or same-segment testing +## WARNING: Not recommended for production ultra-scale deployments +apiVersion: metallb.io/v1beta1 +kind: L2Advertisement +metadata: + name: {{ include "nico-machine-a-tron.name" . }}-bmc-l2 + namespace: {{ .Values.bmcProxy.ipPool.namespace | default "metallb-system" }} + labels: + {{- include "nico-machine-a-tron.labels" . | nindent 4 }} +spec: + ipAddressPools: + - {{ include "nico-machine-a-tron.name" . }}-bmc-pool + {{- if .Values.bmcProxy.ipPool.l2Interfaces }} + interfaces: + {{- toYaml .Values.bmcProxy.ipPool.l2Interfaces | nindent 4 }} + {{- end }} +{{- end }} +{{- end }} +{{- end }} +{{- end }} diff --git a/helm/charts/nico-machine-a-tron/templates/metallb-services.yaml b/helm/charts/nico-machine-a-tron/templates/metallb-services.yaml new file mode 100644 index 0000000000..273c4d6537 --- /dev/null +++ b/helm/charts/nico-machine-a-tron/templates/metallb-services.yaml @@ -0,0 +1,103 @@ +{{- if .Values.bmcProxy.enabled }} +{{- if .Values.bmcProxy.ipRange }} +{{- $root := . }} +{{- $namespace := include "nico-machine-a-tron.namespace" . }} +{{- $name := include "nico-machine-a-tron.name" . }} +{{- $port := .Values.bmcProxy.service.port | default 443 }} + +{{/* +Calculate total BMC count from all machine groups: + totalBMCs = sum(hostCount + hostCount * dpuPerHostCount) for each group +*/}} +{{- $totalBMCs := 0 }} +{{- range $groupName, $group := .Values.machines }} +{{- if $group }} +{{- $hostCount := $group.hostCount | default 0 }} +{{- $dpuPerHost := $group.dpuPerHostCount | default 0 }} +{{- $groupTotal := add $hostCount (mul $hostCount $dpuPerHost) }} +{{- $totalBMCs = add $totalBMCs $groupTotal }} +{{- end }} +{{- end }} + +{{/* +SCALE LIMIT: Maximum /18 subnet (16384 IPs) to protect cluster stability +- Creating too many services risks overwhelming kube-proxy and MetalLB +- Use BGP mode with aggregation for large scale deployments +*/}} +{{- $maxServices := 16384 }} +{{- if gt $totalBMCs $maxServices }} +{{- fail (printf "SCALE LIMIT EXCEEDED: %d BMCs requested, maximum is %d (/18 subnet). Reduce hostCount or dpuPerHostCount." $totalBMCs $maxServices) }} +{{- end }} + +{{/* +Parse IP range: "10.100.0.2-10.100.7.254" -> start IP parts +We generate services from startIP to startIP + totalBMCs - 1 +TODO: Need a more elegant solution +*/}} +{{- $ipRange := .Values.bmcProxy.ipRange }} +{{- $startIP := (split "-" $ipRange)._0 }} +{{- $ipParts := split "." $startIP }} +{{- $octet1 := $ipParts._0 | int }} +{{- $octet2 := $ipParts._1 | int }} +{{- $octet3 := $ipParts._2 | int }} +{{- $octet4 := $ipParts._3 | int }} + +{{/* Generate services for each BMC */}} +{{- range $i := until (int $totalBMCs) }} +{{/* +Calculate IP: baseIP + offset +Handle octet overflow: .4 → .3 → .2 → .1 +*/}} +{{- $offset := $i }} +{{- $o4 := add $octet4 $offset }} +{{- $o3 := $octet3 }} +{{- $o2 := $octet2 }} +{{- $o1 := $octet1 }} + +{{/* Handle octet 4 overflow */}} +{{- if gt $o4 255 }} +{{- $o3 = add $o3 (div $o4 256) }} +{{- $o4 = mod $o4 256 }} +{{- end }} + +{{/* Handle octet 3 overflow */}} +{{- if gt $o3 255 }} +{{- $o2 = add $o2 (div $o3 256) }} +{{- $o3 = mod $o3 256 }} +{{- end }} + +{{/* Handle octet 2 overflow */}} +{{- if gt $o2 255 }} +{{- $o1 = add $o1 (div $o2 256) }} +{{- $o2 = mod $o2 256 }} +{{- end }} + +{{- $currentIP := printf "%d.%d.%d.%d" (int $o1) (int $o2) (int $o3) (int $o4) }} +{{- $safeName := $currentIP | replace "." "-" }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ $name }}-bmc-{{ $safeName }} + namespace: {{ $namespace }} + labels: + {{- include "nico-machine-a-tron.labels" $root | nindent 4 }} + app.kubernetes.io/component: bmc-lb + bmc-ip: "{{ $currentIP }}" + bmc-index: "{{ $i }}" + annotations: + metallb.universe.tf/loadBalancerIPs: "{{ $currentIP }}" +spec: + type: LoadBalancer + externalTrafficPolicy: Local + selector: + {{- include "nico-machine-a-tron.selectorLabels" $root | nindent 4 }} + app.kubernetes.io/component: bmc-proxy + ports: + - name: https + port: {{ $port }} + targetPort: 443 + protocol: TCP +{{- end }} +{{- end }} +{{- end }} diff --git a/helm/charts/nico-machine-a-tron/values.yaml b/helm/charts/nico-machine-a-tron/values.yaml index d05330f6b7..f47b22669a 100644 --- a/helm/charts/nico-machine-a-tron/values.yaml +++ b/helm/charts/nico-machine-a-tron/values.yaml @@ -1,5 +1,23 @@ ## ============================================================================= -## nico-machine-a-tron - Mock machine simulator for NICo development/testing +## nico-machine-a-tron - Mock machine simulator for NICo testing +## ============================================================================= +## +## DEPLOYMENT MODES (mutually exclusive): +## +## 1. OVERRIDE MODE (Development) - bmcProxy.enabled: false +## - NICo Site-Explorer redirects ALL Redfish calls to machine-a-tron +## - Simple setup, no MetalLB required +## - INCOMPATIBLE with real hardware (all BMC traffic goes to mock) +## - Requires: NICo site config with override_target_host set +## +## 2. METALLB MODE (Large-Scale Testing) - bmcProxy.enabled: true +## - Each simulated BMC gets dedicated external IP via MetalLB +## - Compatible with real hardware (can run alongside real machines) +## - Requires: MetalLB with BGP for scale, dedicated IP range +## - Requires: NICo site config WITHOUT override_target_host +## +## See README.md for detailed setup instructions for each mode. +## ## ============================================================================= global: @@ -212,6 +230,103 @@ externalService: externalTrafficPolicy: Local annotations: {} +## ============================================================================= +## BMC Proxy with MetalLB - Provides dedicated IPs for each simulated BMC +## ============================================================================= +## This deploys an NGINX reverse proxy that adds the Forwarded header +## so machine-a-tron can route requests to the correct mock BMC based on IP. +## +## Architecture (NICo Site-Explorer connects DIRECTLY to MetalLB IPs): +## +## Site-Explorer ---> MetalLB LB Services ---> NGINX ---> Machine-A-Tron +## (Redfish calls) (1 per BMC IP) (adds Forwarded header) +## +## HOW IP ALLOCATION WORKS: +## NICo allocates IPs sequentially from the network prefix via gRPC API. +## If the subnet is empty, IPs are assigned in order: .2, .3, .4, ... +## MetalLB services are created with matching sequential IPs. +## +## Service count is AUTO-CALCULATED from machine configuration: +## totalBMCs = sum(hostCount + hostCount * dpuPerHostCount) for all groups +## +## SCALE RECOMMENDATIONS: +## - /21 (2048 IPs) - Default, suitable for most test scenarios +## - /20 (4096 IPs) - Medium scale testing +## - /19 (8192 IPs) - Large scale testing +## - /18 (16384 IPs) - Maximum supported, requires BGP mode +## +## Example: 500 hosts with 2 DPUs each = 500 + 1000 = 1500 services (fits in /21) +## +bmcProxy: + ## Enable the NGINX BMC proxy with MetalLB LoadBalancer services + enabled: false + + ## IP range start address for MetalLB pool + ## Services are created sequentially: startIP, startIP+1, startIP+2, ... + ## Number of services = total BMCs (auto-calculated from machine config) + ## + ## IMPORTANT: This IP range must match the NICo network configuration + ## so that IPs allocated by NICo match the MetalLB service IPs. + ## + ## Examples by scale: + ## /21 (2048 IPs): "10.100.0.2-10.100.7.254" + ## /20 (4096 IPs): "10.100.0.2-10.100.15.254" + ## /19 (8192 IPs): "10.100.0.2-10.100.31.254" + ## /18 (16384 IPs): "10.100.0.2-10.100.63.254" (max, requires BGP) + ipRange: "" + + ## MetalLB IPAddressPool configuration + ipPool: + ## Create the MetalLB IPAddressPool resource + create: true + ## Namespace for MetalLB resources (default: metallb-system) + namespace: metallb-system + ## Enable auto-assign from this pool + autoAssign: true + ## L2 Advertisement (only for small-scale testing, NOT for production) + l2Advertisement: false + ## Interfaces for L2 advertisement (optional) + # l2Interfaces: + # - eth0 + + ## ========================================================================== + ## BGP Configuration - REQUIRED for large scale deployments + ## ========================================================================== + ## BGP mode with route aggregation prevents overwhelming upstream routers + ## with thousands of individual /32 routes. Instead, we announce a single + ## /18 supernet summary route (16000+ services) + ## + bgp: + ## Enable BGP mode. It's REQUIRED for large scale machine-a-tron tests + enabled: false + ## Route aggregation length - announces single /18 instead of 16000+ /32s + ## This is MANDATORY for large scale deployment + aggregationLength: 18 + + ## NGINX reverse proxy settings + nginx: + image: nginx:alpine + replicas: 1 + resources: + limits: + cpu: 2 + memory: 1Gi + requests: + cpu: 500m + memory: 256Mi + + ## TLS settings for NGINX + tls: + ## Use the machine-a-tron certificate for NGINX + secretName: "" # defaults to {{ .Release.Name }}-certificate + + ## Service settings + service: + ## Port NGINX listens on (external-facing) + port: 443 + ## Protocol for health checks + protocol: TCP + configFiles: ## Override the entire mat.toml config (leave empty to use generated config) matConfig: "" diff --git a/helm/templates/NOTES.txt b/helm/templates/NOTES.txt index bfb2ae32c4..685e307a3b 100644 --- a/helm/templates/NOTES.txt +++ b/helm/templates/NOTES.txt @@ -30,6 +30,9 @@ Deployed sub-charts: {{- if index .Values "nico-ssh-console-rs" "enabled" | default true }} ✓ nico-ssh-console-rs {{- end }} +{{- if index .Values "nico-machine-a-tron" "enabled" | default false }} + ✓ nico-machine-a-tron (mock machines for dev/test) +{{- end }} {{- if index .Values "unbound" "enabled" | default false }} ✓ unbound {{- end }} From 9776be3e37b7737a741b66a4398fcec48ff3acac Mon Sep 17 00:00:00 2001 From: Alexander Korobkov Date: Tue, 23 Jun 2026 13:01:59 -0600 Subject: [PATCH 11/28] chore(helm): nico machine-a-tron nginx proxy rename --- helm/charts/nico-machine-a-tron/README.md | 14 ++-- .../templates/bmc-proxy-configmap.yaml | 4 +- .../templates/bmc-proxy-deployment.yaml | 18 ++-- .../templates/metallb-pool.yaml | 44 +++++----- .../templates/metallb-services.yaml | 10 +-- helm/charts/nico-machine-a-tron/values.yaml | 82 +++++++++---------- 6 files changed, 84 insertions(+), 88 deletions(-) diff --git a/helm/charts/nico-machine-a-tron/README.md b/helm/charts/nico-machine-a-tron/README.md index 8f84d6166d..3a9001d08b 100644 --- a/helm/charts/nico-machine-a-tron/README.md +++ b/helm/charts/nico-machine-a-tron/README.md @@ -169,9 +169,9 @@ flowchart LR helm upgrade --install nico ./helm \ --namespace nico-system \ --set nico-machine-a-tron.enabled=true \ - --set nico-machine-a-tron.bmcProxy.enabled=true \ - --set nico-machine-a-tron.bmcProxy.ipRange="10.100.0.0-10.100.7.254" \ - --set nico-machine-a-tron.bmcProxy.bgp.enabled=true \ + --set nico-machine-a-tron.nginxBmcProxy.enabled=true \ + --set nico-machine-a-tron.nginxBmcProxy.ipRange="10.100.0.0-10.100.7.254" \ + --set nico-machine-a-tron.nginxBmcProxy.bgp.enabled=true \ --set nico-machine-a-tron.machines.dell-hosts.hostCount=100 \ --set nico-machine-a-tron.machines.dell-hosts.dpuPerHostCount=2 \ --set nico-machine-a-tron.machines.dell-hosts.oobDhcpRelayAddress="10.100.0.1" @@ -237,7 +237,7 @@ SCALE LIMIT EXCEEDED: 16500 BMCs requested, maximum is 16384 (/18 subnet). For deployments with 2000+ services, BGP mode with route aggregation is **required**: ```yaml -bmcProxy: +nginxBmcProxy: enabled: true ipRange: "10.100.0.2-10.100.15.254" # /20 range @@ -279,9 +279,9 @@ individual /32 route. With thousands of services, this overwhelms upstream route | `machines..hwType` | Hardware type | `dell_poweredge_r750` | | `machines..oobDhcpRelayAddress` | BMC network gateway | `10.100.0.1` | | `machines..adminDhcpRelayAddress` | Admin network gateway | `10.101.0.1` | -| `bmcProxy.enabled` | Enable MetalLB mode | `false` | -| `bmcProxy.ipRange` | MetalLB IP pool range | `""` | -| `bmcProxy.bgp.enabled` | Enable BGP advertisement | `false` | +| `nginxBmcProxy.enabled` | Enable MetalLB mode | `false` | +| `nginxBmcProxy.ipRange` | MetalLB IP pool range | `""` | +| `nginxBmcProxy.bgp.enabled` | Enable BGP advertisement | `false` | | `terminationGracePeriodSeconds` | Shutdown timeout | `60` | ### Machine Groups diff --git a/helm/charts/nico-machine-a-tron/templates/bmc-proxy-configmap.yaml b/helm/charts/nico-machine-a-tron/templates/bmc-proxy-configmap.yaml index 18c7b16467..53d2ef039e 100644 --- a/helm/charts/nico-machine-a-tron/templates/bmc-proxy-configmap.yaml +++ b/helm/charts/nico-machine-a-tron/templates/bmc-proxy-configmap.yaml @@ -1,4 +1,4 @@ -{{- if .Values.bmcProxy.enabled }} +{{- if .Values.nginxBmcProxy.enabled }} apiVersion: v1 kind: ConfigMap metadata: @@ -6,7 +6,7 @@ metadata: namespace: {{ include "nico-machine-a-tron.namespace" . }} labels: {{- include "nico-machine-a-tron.labels" . | nindent 4 }} - app.kubernetes.io/component: bmc-proxy + app.kubernetes.io/component: nginx-bmc-proxy data: nginx.conf: | worker_processes auto; diff --git a/helm/charts/nico-machine-a-tron/templates/bmc-proxy-deployment.yaml b/helm/charts/nico-machine-a-tron/templates/bmc-proxy-deployment.yaml index bdf14ec496..a00980e4aa 100644 --- a/helm/charts/nico-machine-a-tron/templates/bmc-proxy-deployment.yaml +++ b/helm/charts/nico-machine-a-tron/templates/bmc-proxy-deployment.yaml @@ -1,4 +1,4 @@ -{{- if .Values.bmcProxy.enabled }} +{{- if .Values.nginxBmcProxy.enabled }} apiVersion: apps/v1 kind: Deployment metadata: @@ -6,18 +6,18 @@ metadata: namespace: {{ include "nico-machine-a-tron.namespace" . }} labels: {{- include "nico-machine-a-tron.labels" . | nindent 4 }} - app.kubernetes.io/component: bmc-proxy + app.kubernetes.io/component: nginx-bmc-proxy spec: - replicas: {{ .Values.bmcProxy.nginx.replicas | default 1 }} + replicas: {{ .Values.nginxBmcProxy.replicas | default 1 }} selector: matchLabels: {{- include "nico-machine-a-tron.selectorLabels" . | nindent 6 }} - app.kubernetes.io/component: bmc-proxy + app.kubernetes.io/component: nginx-bmc-proxy template: metadata: labels: {{- include "nico-machine-a-tron.selectorLabels" . | nindent 8 }} - app.kubernetes.io/component: bmc-proxy + app.kubernetes.io/component: nginx-bmc-proxy annotations: checksum/nginx-config: {{ include (print $.Template.BasePath "/bmc-proxy-configmap.yaml") . | sha256sum }} spec: @@ -32,7 +32,7 @@ spec: topologyKey: kubernetes.io/hostname containers: - name: nginx - image: {{ .Values.bmcProxy.nginx.image | default "nginx:alpine" }} + image: {{ .Values.nginxBmcProxy.image | default "nginx:alpine" }} imagePullPolicy: IfNotPresent ports: - name: https @@ -49,7 +49,7 @@ spec: - name: tls-key mountPath: /etc/ssl/private readOnly: true - {{- with .Values.bmcProxy.nginx.resources }} + {{- with .Values.nginxBmcProxy.resources }} resources: {{- toYaml . | nindent 12 }} {{- end }} @@ -69,13 +69,13 @@ spec: name: {{ include "nico-machine-a-tron.name" . }}-bmc-proxy-config - name: tls-certs secret: - secretName: {{ .Values.bmcProxy.tls.secretName | default (printf "%s-certificate" (include "nico-machine-a-tron.name" .)) }} + secretName: {{ .Values.nginxBmcProxy.tlsSecretName | default (printf "%s-certificate" (include "nico-machine-a-tron.name" .)) }} items: - key: tls.crt path: tls.crt - name: tls-key secret: - secretName: {{ .Values.bmcProxy.tls.secretName | default (printf "%s-certificate" (include "nico-machine-a-tron.name" .)) }} + secretName: {{ .Values.nginxBmcProxy.tlsSecretName | default (printf "%s-certificate" (include "nico-machine-a-tron.name" .)) }} items: - key: tls.key path: tls.key diff --git a/helm/charts/nico-machine-a-tron/templates/metallb-pool.yaml b/helm/charts/nico-machine-a-tron/templates/metallb-pool.yaml index 4f739d4898..e690fe65e4 100644 --- a/helm/charts/nico-machine-a-tron/templates/metallb-pool.yaml +++ b/helm/charts/nico-machine-a-tron/templates/metallb-pool.yaml @@ -1,21 +1,21 @@ -{{- if .Values.bmcProxy.enabled }} -{{- if .Values.bmcProxy.ipPool.create }} -{{- if .Values.bmcProxy.ipRange }} +{{- if .Values.nginxBmcProxy.enabled }} +{{- if .Values.nginxBmcProxy.ipPool.create }} +{{- if .Values.nginxBmcProxy.ipRange }} --- apiVersion: metallb.io/v1beta1 kind: IPAddressPool metadata: name: {{ include "nico-machine-a-tron.name" . }}-bmc-pool - namespace: {{ .Values.bmcProxy.ipPool.namespace | default "metallb-system" }} + namespace: {{ .Values.nginxBmcProxy.ipPool.namespace | default "metallb-system" }} labels: {{- include "nico-machine-a-tron.labels" . | nindent 4 }} spec: addresses: - - {{ .Values.bmcProxy.ipRange }} - {{- if .Values.bmcProxy.ipPool.autoAssign }} - autoAssign: {{ .Values.bmcProxy.ipPool.autoAssign }} + - {{ .Values.nginxBmcProxy.ipRange }} + {{- if .Values.nginxBmcProxy.ipPool.autoAssign }} + autoAssign: {{ .Values.nginxBmcProxy.ipPool.autoAssign }} {{- end }} -{{- if .Values.bmcProxy.bgp.enabled }} +{{- if .Values.nginxBmcProxy.bgp.enabled }} --- ## BGP Advertisement with route aggregation ## CRITICAL: aggregationLength prevents advertising 2000+ individual /32 routes @@ -24,7 +24,7 @@ apiVersion: metallb.io/v1beta1 kind: BGPAdvertisement metadata: name: {{ include "nico-machine-a-tron.name" . }}-bmc-bgp - namespace: {{ .Values.bmcProxy.ipPool.namespace | default "metallb-system" }} + namespace: {{ .Values.nginxBmcProxy.ipPool.namespace | default "metallb-system" }} labels: {{- include "nico-machine-a-tron.labels" . | nindent 4 }} spec: @@ -32,22 +32,22 @@ spec: - {{ include "nico-machine-a-tron.name" . }}-bmc-pool ## Aggregate all service IPs into a single /18 announcement ## This is MANDATORY for ultra-scale (2000+ services) - aggregationLength: {{ .Values.bmcProxy.bgp.aggregationLength | default 18 }} - {{- if .Values.bmcProxy.bgp.aggregationLengthV6 }} - aggregationLengthV6: {{ .Values.bmcProxy.bgp.aggregationLengthV6 }} + aggregationLength: {{ .Values.nginxBmcProxy.bgp.aggregationLength | default 18 }} + {{- if .Values.nginxBmcProxy.bgp.aggregationLengthV6 }} + aggregationLengthV6: {{ .Values.nginxBmcProxy.bgp.aggregationLengthV6 }} {{- end }} - {{- if .Values.bmcProxy.bgp.localPref }} - localPref: {{ .Values.bmcProxy.bgp.localPref }} + {{- if .Values.nginxBmcProxy.bgp.localPref }} + localPref: {{ .Values.nginxBmcProxy.bgp.localPref }} {{- end }} - {{- if .Values.bmcProxy.bgp.communities }} + {{- if .Values.nginxBmcProxy.bgp.communities }} communities: - {{- toYaml .Values.bmcProxy.bgp.communities | nindent 4 }} + {{- toYaml .Values.nginxBmcProxy.bgp.communities | nindent 4 }} {{- end }} - {{- if .Values.bmcProxy.bgp.peers }} + {{- if .Values.nginxBmcProxy.bgp.peers }} peers: - {{- toYaml .Values.bmcProxy.bgp.peers | nindent 4 }} + {{- toYaml .Values.nginxBmcProxy.bgp.peers | nindent 4 }} {{- end }} -{{- else if .Values.bmcProxy.ipPool.l2Advertisement }} +{{- else if .Values.nginxBmcProxy.ipPool.l2Advertisement }} --- ## L2 Advertisement - only suitable for small scale or same-segment testing ## WARNING: Not recommended for production ultra-scale deployments @@ -55,15 +55,15 @@ apiVersion: metallb.io/v1beta1 kind: L2Advertisement metadata: name: {{ include "nico-machine-a-tron.name" . }}-bmc-l2 - namespace: {{ .Values.bmcProxy.ipPool.namespace | default "metallb-system" }} + namespace: {{ .Values.nginxBmcProxy.ipPool.namespace | default "metallb-system" }} labels: {{- include "nico-machine-a-tron.labels" . | nindent 4 }} spec: ipAddressPools: - {{ include "nico-machine-a-tron.name" . }}-bmc-pool - {{- if .Values.bmcProxy.ipPool.l2Interfaces }} + {{- if .Values.nginxBmcProxy.ipPool.l2Interfaces }} interfaces: - {{- toYaml .Values.bmcProxy.ipPool.l2Interfaces | nindent 4 }} + {{- toYaml .Values.nginxBmcProxy.ipPool.l2Interfaces | nindent 4 }} {{- end }} {{- end }} {{- end }} diff --git a/helm/charts/nico-machine-a-tron/templates/metallb-services.yaml b/helm/charts/nico-machine-a-tron/templates/metallb-services.yaml index 273c4d6537..17cb22f00e 100644 --- a/helm/charts/nico-machine-a-tron/templates/metallb-services.yaml +++ b/helm/charts/nico-machine-a-tron/templates/metallb-services.yaml @@ -1,9 +1,9 @@ -{{- if .Values.bmcProxy.enabled }} -{{- if .Values.bmcProxy.ipRange }} +{{- if .Values.nginxBmcProxy.enabled }} +{{- if .Values.nginxBmcProxy.ipRange }} {{- $root := . }} {{- $namespace := include "nico-machine-a-tron.namespace" . }} {{- $name := include "nico-machine-a-tron.name" . }} -{{- $port := .Values.bmcProxy.service.port | default 443 }} +{{- $port := .Values.nginxBmcProxy.servicePort | default 443 }} {{/* Calculate total BMC count from all machine groups: @@ -34,7 +34,7 @@ Parse IP range: "10.100.0.2-10.100.7.254" -> start IP parts We generate services from startIP to startIP + totalBMCs - 1 TODO: Need a more elegant solution */}} -{{- $ipRange := .Values.bmcProxy.ipRange }} +{{- $ipRange := .Values.nginxBmcProxy.ipRange }} {{- $startIP := (split "-" $ipRange)._0 }} {{- $ipParts := split "." $startIP }} {{- $octet1 := $ipParts._0 | int }} @@ -92,7 +92,7 @@ spec: externalTrafficPolicy: Local selector: {{- include "nico-machine-a-tron.selectorLabels" $root | nindent 4 }} - app.kubernetes.io/component: bmc-proxy + app.kubernetes.io/component: nginx-bmc-proxy ports: - name: https port: {{ $port }} diff --git a/helm/charts/nico-machine-a-tron/values.yaml b/helm/charts/nico-machine-a-tron/values.yaml index f47b22669a..fa76c8520b 100644 --- a/helm/charts/nico-machine-a-tron/values.yaml +++ b/helm/charts/nico-machine-a-tron/values.yaml @@ -4,13 +4,13 @@ ## ## DEPLOYMENT MODES (mutually exclusive): ## -## 1. OVERRIDE MODE (Development) - bmcProxy.enabled: false +## 1. OVERRIDE MODE (Development) - nginxBmcProxy.enabled: false ## - NICo Site-Explorer redirects ALL Redfish calls to machine-a-tron ## - Simple setup, no MetalLB required ## - INCOMPATIBLE with real hardware (all BMC traffic goes to mock) ## - Requires: NICo site config with override_target_host set ## -## 2. METALLB MODE (Large-Scale Testing) - bmcProxy.enabled: true +## 2. METALLB MODE (Large-Scale Testing) - nginxBmcProxy.enabled: true ## - Each simulated BMC gets dedicated external IP via MetalLB ## - Compatible with real hardware (can run alongside real machines) ## - Requires: MetalLB with BGP for scale, dedicated IP range @@ -257,76 +257,72 @@ externalService: ## ## Example: 500 hosts with 2 DPUs each = 500 + 1000 = 1500 services (fits in /21) ## -bmcProxy: +nginxBmcProxy: ## Enable the NGINX BMC proxy with MetalLB LoadBalancer services enabled: false - ## IP range start address for MetalLB pool - ## Services are created sequentially: startIP, startIP+1, startIP+2, ... + ## ========================================================================== + ## NGINX Reverse Proxy Settings + ## ========================================================================== + image: nginx:alpine + replicas: 1 + resources: + limits: + cpu: 2 + memory: 1Gi + requests: + cpu: 500m + memory: 256Mi + + ## TLS certificate secret name (defaults to machine-a-tron certificate) + tlsSecretName: "" + + ## Service port (external-facing) + servicePort: 443 + + ## ========================================================================== + ## MetalLB IP Pool Configuration + ## ========================================================================== + ## IP range for MetalLB pool. Services are created sequentially. ## Number of services = total BMCs (auto-calculated from machine config) ## ## IMPORTANT: This IP range must match the NICo network configuration ## so that IPs allocated by NICo match the MetalLB service IPs. ## + ## NICo network config (nico-api-site-config.toml): + ## [networks.simulated-oob] + ## type = "underlay" + ## prefix = "10.100.0.0/21" + ## gateway = "10.100.0.1" + ## + ## Machine-a-tron config (values.yaml): + ## machines.dell-hosts.oobDhcpRelayAddress: "10.100.0.1" + ## ## Examples by scale: ## /21 (2048 IPs): "10.100.0.2-10.100.7.254" - ## /20 (4096 IPs): "10.100.0.2-10.100.15.254" - ## /19 (8192 IPs): "10.100.0.2-10.100.31.254" - ## /18 (16384 IPs): "10.100.0.2-10.100.63.254" (max, requires BGP) + ## /18 (16384 IPs): "10.100.0.2-10.100.63.254" max supported ipRange: "" - ## MetalLB IPAddressPool configuration ipPool: ## Create the MetalLB IPAddressPool resource create: true - ## Namespace for MetalLB resources (default: metallb-system) + ## Namespace for MetalLB resources namespace: metallb-system ## Enable auto-assign from this pool autoAssign: true - ## L2 Advertisement (only for small-scale testing, NOT for production) + ## L2 Advertisement (only for small-scale testing) l2Advertisement: false - ## Interfaces for L2 advertisement (optional) - # l2Interfaces: - # - eth0 ## ========================================================================== ## BGP Configuration - REQUIRED for large scale deployments ## ========================================================================== ## BGP mode with route aggregation prevents overwhelming upstream routers - ## with thousands of individual /32 routes. Instead, we announce a single - ## /18 supernet summary route (16000+ services) - ## + ## with thousands of individual /32 routes. bgp: - ## Enable BGP mode. It's REQUIRED for large scale machine-a-tron tests enabled: false ## Route aggregation length - announces single /18 instead of 16000+ /32s - ## This is MANDATORY for large scale deployment aggregationLength: 18 - ## NGINX reverse proxy settings - nginx: - image: nginx:alpine - replicas: 1 - resources: - limits: - cpu: 2 - memory: 1Gi - requests: - cpu: 500m - memory: 256Mi - - ## TLS settings for NGINX - tls: - ## Use the machine-a-tron certificate for NGINX - secretName: "" # defaults to {{ .Release.Name }}-certificate - - ## Service settings - service: - ## Port NGINX listens on (external-facing) - port: 443 - ## Protocol for health checks - protocol: TCP - configFiles: ## Override the entire mat.toml config (leave empty to use generated config) matConfig: "" From 7a0b6e5313b5f22a4a96789953357714a7f9f615 Mon Sep 17 00:00:00 2001 From: Alexander Korobkov Date: Tue, 23 Jun 2026 13:09:59 -0600 Subject: [PATCH 12/28] chore(helm): nico machine-a-tron readme --- helm/charts/nico-machine-a-tron/README.md | 39 +++++++++++++---------- 1 file changed, 22 insertions(+), 17 deletions(-) diff --git a/helm/charts/nico-machine-a-tron/README.md b/helm/charts/nico-machine-a-tron/README.md index 3a9001d08b..f03e4ad2ad 100644 --- a/helm/charts/nico-machine-a-tron/README.md +++ b/helm/charts/nico-machine-a-tron/README.md @@ -45,17 +45,17 @@ real hardware** since all BMC traffic goes to the mock. ```mermaid flowchart LR - subgraph NICo["NICo Cluster"] + subgraph NICo["NICo Cluster (nico-system)"] SE[Site-Explorer] API[NICo API] end - subgraph MAT["Machine-A-Tron"] - BMC[BMC Mock
Single endpoint] + subgraph MAT["nico-mat namespace"] + BMC[Machine-A-Tron Pod
BMC Mock] end SE -->|"ALL Redfish calls
override_target_host"| BMC - BMC <-->|"DHCP, status"| API + BMC <-->|"gRPC: DHCP, status"| API ``` ### Setup @@ -64,7 +64,7 @@ flowchart LR ```bash helm upgrade --install nico ./helm \ - --namespace nico-system \ + --namespace nico-mat \ --set nico-machine-a-tron.enabled=true \ --set nico-machine-a-tron.machines.dell-hosts.hostCount=10 \ --set nico-machine-a-tron.machines.dell-hosts.dpuPerHostCount=2 @@ -124,7 +124,7 @@ to coexist on the same NICo instance. ```mermaid flowchart LR - subgraph NICo["NICo Cluster"] + subgraph NICo["NICo Cluster (nico-system)"] SE[Site-Explorer] end @@ -133,15 +133,20 @@ flowchart LR R2[Real BMC
10.50.0.11] end - subgraph MetalLB["MetalLB (Simulated BMCs)"] - LB1["10.100.0.2"] - LB2["10.100.0.3"] - LBN["... 13500 IPs"] - end + subgraph MAT["nico-mat namespace"] + subgraph MetalLB["MetalLB Services"] + LB1["10.100.0.2"] + LB2["10.100.0.3"] + LBN["..."] + end + + subgraph NGX["NGINX Pod"] + NGINX["NGINX Proxy"] + end - subgraph MAT["Machine-A-Tron Pod"] - NGINX["NGINX Proxy"] - BMC["BMC Mock Registry"] + subgraph MATpod["Machine-A-Tron Pod"] + BMC["BMC Mock Registry"] + end end SE -->|"Redfish"| R1 @@ -152,7 +157,7 @@ flowchart LR LB1 --> NGINX LB2 --> NGINX LBN --> NGINX - NGINX -->|"Forwarded header"| BMC + NGINX -->|"Forwarded: host=IP"| BMC ``` ### Prerequisites (MetalLB Mode) @@ -167,7 +172,7 @@ flowchart LR ```bash helm upgrade --install nico ./helm \ - --namespace nico-system \ + --namespace nico-mat \ --set nico-machine-a-tron.enabled=true \ --set nico-machine-a-tron.nginxBmcProxy.enabled=true \ --set nico-machine-a-tron.nginxBmcProxy.ipRange="10.100.0.0-10.100.7.254" \ @@ -383,7 +388,7 @@ serviceMonitor: Check logs: ```bash -kubectl -n nico-system logs deployment/nico-machine-a-tron +kubectl -n nico-mat logs deployment/nico-machine-a-tron ``` Common causes: From d1637b785ef3df4e32e5559a5fc3b40fcaeb9f6c Mon Sep 17 00:00:00 2001 From: Alexander Korobkov Date: Tue, 23 Jun 2026 13:28:17 -0600 Subject: [PATCH 13/28] fix(helm): nico machine-a-tron values --- helm/charts/nico-machine-a-tron/README.md | 2 +- helm/charts/nico-machine-a-tron/values.yaml | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/helm/charts/nico-machine-a-tron/README.md b/helm/charts/nico-machine-a-tron/README.md index f03e4ad2ad..a90e337347 100644 --- a/helm/charts/nico-machine-a-tron/README.md +++ b/helm/charts/nico-machine-a-tron/README.md @@ -175,7 +175,7 @@ helm upgrade --install nico ./helm \ --namespace nico-mat \ --set nico-machine-a-tron.enabled=true \ --set nico-machine-a-tron.nginxBmcProxy.enabled=true \ - --set nico-machine-a-tron.nginxBmcProxy.ipRange="10.100.0.0-10.100.7.254" \ + --set nico-machine-a-tron.nginxBmcProxy.ipRange="10.100.0.2-10.100.7.254" \ --set nico-machine-a-tron.nginxBmcProxy.bgp.enabled=true \ --set nico-machine-a-tron.machines.dell-hosts.hostCount=100 \ --set nico-machine-a-tron.machines.dell-hosts.dpuPerHostCount=2 \ diff --git a/helm/charts/nico-machine-a-tron/values.yaml b/helm/charts/nico-machine-a-tron/values.yaml index fa76c8520b..c1f58e1975 100644 --- a/helm/charts/nico-machine-a-tron/values.yaml +++ b/helm/charts/nico-machine-a-tron/values.yaml @@ -320,8 +320,9 @@ nginxBmcProxy: ## with thousands of individual /32 routes. bgp: enabled: false - ## Route aggregation length - announces single /18 instead of 16000+ /32s - aggregationLength: 18 + ## Route aggregation length - announces summary route instead of /32s + ## /21 = 2048 IPs (default), /18 = 16384 IPs (max) + aggregationLength: 21 configFiles: ## Override the entire mat.toml config (leave empty to use generated config) From b5731f91224ce7e98494322ac78276f837e8d812 Mon Sep 17 00:00:00 2001 From: Alexander Korobkov Date: Tue, 7 Jul 2026 16:06:05 -0600 Subject: [PATCH 14/28] feat(helm): nico machine-a-tron migrate to clusterip --- helm/charts/nico-machine-a-tron/README.md | 517 +++++++----------- .../templates/bmc-proxy-configmap.yaml | 50 -- .../templates/bmc-proxy-deployment.yaml | 82 --- .../templates/bmc-service-cidr.yaml | 55 ++ .../templates/bmc-services.yaml | 131 +++++ .../templates/certificate.yaml | 57 +- .../templates/configmap.yaml | 131 +++-- .../templates/deployment.yaml | 136 +++-- .../templates/metallb-pool.yaml | 71 --- .../templates/metallb-services.yaml | 103 ---- .../templates/service.yaml | 62 ++- helm/charts/nico-machine-a-tron/values.yaml | 343 ++++++------ 12 files changed, 841 insertions(+), 897 deletions(-) delete mode 100644 helm/charts/nico-machine-a-tron/templates/bmc-proxy-configmap.yaml delete mode 100644 helm/charts/nico-machine-a-tron/templates/bmc-proxy-deployment.yaml create mode 100644 helm/charts/nico-machine-a-tron/templates/bmc-service-cidr.yaml create mode 100644 helm/charts/nico-machine-a-tron/templates/bmc-services.yaml delete mode 100644 helm/charts/nico-machine-a-tron/templates/metallb-pool.yaml delete mode 100644 helm/charts/nico-machine-a-tron/templates/metallb-services.yaml diff --git a/helm/charts/nico-machine-a-tron/README.md b/helm/charts/nico-machine-a-tron/README.md index a90e337347..cd9397f2e0 100644 --- a/helm/charts/nico-machine-a-tron/README.md +++ b/helm/charts/nico-machine-a-tron/README.md @@ -7,413 +7,308 @@ Helm chart for deploying Machine-A-Tron - a mock machine simulator for NICo test Machine-A-Tron creates simulated bare-metal machines that behave like real hosts, allowing you to: - Test NICo without physical hardware - Simulate multiple hosts, DPUs, switches and power shelves -- Perform load testing at scale (up to 5000 machines) +- Perform load testing at scale (multiple pods, thousands of BMCs) - Run simulations alongside real hardware ## Deployment Modes -Machine-A-Tron supports **two mutually exclusive deployment modes**: - | Mode | Use Case | Real HW Compatible | Network Setup | |------|----------|-------------------|---------------| -| **Override Mode** | Development environments | No | Simple - single endpoint | -| **MetalLB Mode** | Load testing with real HW | Yes | Complex - per-BMC IPs | - -### Choosing a Mode - -```mermaid -flowchart TD - Q1{Need to run alongside
real hardware?} - Q1 -->|No| M1[Override Mode] - Q1 -->|Yes| M2[MetalLB Mode] - - M1 --> D1[Dev/test environments
Simulated machines only] - M2 --> D2[Load testing
Mixed real + simulated] -``` +| **Override Mode** | Development | No | Simple - single endpoint | +| **ClusterIP Mode** | Scale testing | Yes | Per-BMC ClusterIP services | --- ## Mode 1: Override Mode (Development) -**Use this for development environments where only simulated machines are needed.** - -In this mode, NICo's Site-Explorer is configured to redirect ALL Redfish calls to -machine-a-tron via `override_target_host`. This is simple but **incompatible with -real hardware** since all BMC traffic goes to the mock. - -### Architecture - -```mermaid -flowchart LR - subgraph NICo["NICo Cluster (nico-system)"] - SE[Site-Explorer] - API[NICo API] - end - - subgraph MAT["nico-mat namespace"] - BMC[Machine-A-Tron Pod
BMC Mock] - end +**Use for development environments where only simulated machines are needed.** - SE -->|"ALL Redfish calls
override_target_host"| BMC - BMC <-->|"gRPC: DHCP, status"| API -``` +NICo's Site-Explorer is configured to redirect ALL Redfish calls to machine-a-tron. +Simple but **incompatible with real hardware**. ### Setup -**1. Deploy Machine-A-Tron:** - ```bash helm upgrade --install nico ./helm \ --namespace nico-mat \ --set nico-machine-a-tron.enabled=true \ - --set nico-machine-a-tron.machines.dell-hosts.hostCount=10 \ - --set nico-machine-a-tron.machines.dell-hosts.dpuPerHostCount=2 + --set nico-machine-a-tron.pods.default.machines.rack-machines.hostCount=10 \ + --set nico-machine-a-tron.pods.default.machines.rack-machines.dpuPerHostCount=2 ``` -**2. Configure NICo Site Config:** - +**NICo Site Config:** ```toml -# nico-api-site-config.toml [site_explorer] -enabled = true -create_machines = true - -# Redirect ALL Redfish calls to machine-a-tron override_target_host = "nico-machine-a-tron-bmc-mock" override_target_port = 1266 ``` -**3. Configure NICo Networks:** - -The DHCP relay addresses in machine-a-tron must match NICo network configuration: - -```toml -# OOB network for BMC management -[networks.oob-bmc] -type = "underlay" -prefix = "192.168.192.0/24" -gateway = "192.168.192.1" # matches oobDhcpRelayAddress - -# Admin network for host provisioning -[networks.admin] -type = "admin" -prefix = "192.168.176.0/24" -gateway = "192.168.176.1" # matches adminDhcpRelayAddress -``` - -### Pros and Cons (Override Mode) - -**Pros:** -- Simple setup - no MetalLB required -- Works in any Kubernetes cluster - -**Cons:** -- Cannot use real hardware - all Redfish calls go to mock - --- -## Mode 2: MetalLB Mode (Large-Scale Testing) +## Mode 2: ClusterIP Mode (Scale Testing) -**Use this for dev or load testing environments where simulated machines run alongside real hardware.** +**Use for load testing environments where simulated machines run alongside real hardware.** -In this mode, each simulated BMC gets a dedicated external IP via MetalLB. NICo -Site-Explorer connects directly to these IPs, allowing simulated and real machines -to coexist on the same NICo instance. +Each simulated BMC gets a dedicated ClusterIP service. Supports multi-pod deployments. -### Architecture (MetalLB Mode) +### Architecture ```mermaid -flowchart LR - subgraph NICo["NICo Cluster (nico-system)"] - SE[Site-Explorer] - end - - subgraph Real["Real Hardware"] - R1[Real BMC
10.50.0.10] - R2[Real BMC
10.50.0.11] - end - - subgraph MAT["nico-mat namespace"] - subgraph MetalLB["MetalLB Services"] - LB1["10.100.0.2"] - LB2["10.100.0.3"] - LBN["..."] +flowchart TB + subgraph CIDR["ServiceCIDR: 10.100.0.0/20 (reserved)"] + direction TB + subgraph pod0["pod-0 (~14 racks)"] + cidr0["10.100.0.0/22
1021 IPs"] end - - subgraph NGX["NGINX Pod"] - NGINX["NGINX Proxy"] + subgraph pod1["pod-1 (~14 racks)"] + cidr1["10.100.4.0/22
1021 IPs"] end - - subgraph MATpod["Machine-A-Tron Pod"] - BMC["BMC Mock Registry"] + subgraph pod2["pod-2 (~14 racks)"] + cidr2["10.100.8.0/22
1021 IPs"] end end - SE -->|"Redfish"| R1 - SE -->|"Redfish"| R2 - SE -->|"Redfish"| LB1 - SE -->|"Redfish"| LB2 - - LB1 --> NGINX - LB2 --> NGINX - LBN --> NGINX - NGINX -->|"Forwarded: host=IP"| BMC -``` - -### Prerequisites (MetalLB Mode) - -- MetalLB installed and configured -- BGP mode required for scale 2000+ services -- Dedicated IP range for simulated BMCs (separate from real hardware) - -### Setup (MetalLB Mode) - -**1. Deploy with MetalLB BMC Proxy enabled:** - -```bash -helm upgrade --install nico ./helm \ - --namespace nico-mat \ - --set nico-machine-a-tron.enabled=true \ - --set nico-machine-a-tron.nginxBmcProxy.enabled=true \ - --set nico-machine-a-tron.nginxBmcProxy.ipRange="10.100.0.2-10.100.7.254" \ - --set nico-machine-a-tron.nginxBmcProxy.bgp.enabled=true \ - --set nico-machine-a-tron.machines.dell-hosts.hostCount=100 \ - --set nico-machine-a-tron.machines.dell-hosts.dpuPerHostCount=2 \ - --set nico-machine-a-tron.machines.dell-hosts.oobDhcpRelayAddress="10.100.0.1" + SE[Site-Explorer] --> cidr0 + SE --> cidr1 + SE --> cidr2 ``` -**2. NICo Site Config - NO override_target:** - -```toml -# nico-api-site-config.toml -[site_explorer] -enabled = true -create_machines = true +### Prerequisites -# DO NOT set override_target_host - let NICo connect to actual BMC IPs -# override_target_host = ... # NOT SET! -# override_target_port = ... # NOT SET! -``` +- **Kubernetes 1.29+** (ServiceCIDR API support) +- IP range within cluster's default service CIDR (e.g., `10.96.0.0/12`) -**3. Configure Separate Networks:** +### Single Pod Setup -```toml -# Real hardware OOB network -[networks.real-oob] -type = "underlay" -prefix = "10.50.0.0/24" -gateway = "10.50.0.1" +```yaml +# values.yaml +bmcServices: + enabled: true + serviceCIDR: + create: true -# Simulated BMC network (MetalLB range) -[networks.simulated-oob] -type = "underlay" -prefix = "10.100.0.0/21" -gateway = "10.100.0.1" # matches oobDhcpRelayAddress for simulated machines +pods: + default: + cidr: "10.100.0.0/22" # 1021 IPs for ~14 racks + machines: + compute: + hwType: wiwynn_gb200_nvl + hostCount: 252 # 18 trays × 14 racks + dpuPerHostCount: 2 # 2 BF3 per tray → 504 DPU BMCs + oobDhcpRelayAddress: "10.100.0.1" + switches: + hwType: nvidia_switch_nd5200_ld + hostCount: 126 # 9 switches × 14 racks + dpuPerHostCount: 0 + oobDhcpRelayAddress: "10.100.0.1" + power: + hwType: liteon_power_shelf + hostCount: 112 # 8 shelves × 14 racks + dpuPerHostCount: 0 + oobDhcpRelayAddress: "10.100.0.1" ``` -### IP Allocation - How It Works - -NICo allocates IPs **sequentially** via gRPC API: -- First machine gets `.2` (after gateway `.1`) -- Second machine gets `.3` -- And so on... - -This means MetalLB services with sequential IPs will match NICo allocations -**as long as the subnet is dedicated to machine-a-tron**. - -### Scale Limits - -| Scale | Subnet | Max BMCs | BGP Required | -|-------|--------|----------|--------------| -| Small | /21 | 2048 | No (L2 OK for testing) | -| Medium | /20 | 4096 | Recommended | -| Large | /19 | 8192 | Yes | -| Max | /18 | 16384 | Yes | - -The chart enforces a **hard limit of 16384 BMCs** (/18 subnet) to protect cluster stability. - -If you exceed the limit, the chart **fails**: -``` -SCALE LIMIT EXCEEDED: 16500 BMCs requested, maximum is 16384 (/18 subnet). -``` +**Total BMCs:** 252 + 504 + 126 + 112 = **994 BMCs** (fits in /22) -### BGP Configuration (Required for Large Scale) +### Multi-Pod Setup (Large Scale) -For deployments with 2000+ services, BGP mode with route aggregation is **required**: +For deployments exceeding 1021 BMCs, use multiple pods with separate CIDRs: ```yaml -nginxBmcProxy: +# values.yaml +bmcServices: enabled: true - ipRange: "10.100.0.2-10.100.15.254" # /20 range - - ipPool: + serviceCIDR: create: true - l2Advertisement: false # Disable L2 for BGP - - bgp: - enabled: true - aggregationLength: 20 # Announce single /20 instead of 4000+ /32s + cidr: "10.100.0.0/20" # Covers all pods + +pods: + pod-0: + cidr: "10.100.0.0/22" + machines: + compute: + hwType: wiwynn_gb200_nvl + hostCount: 252 + dpuPerHostCount: 2 + oobDhcpRelayAddress: "10.100.0.1" + switches: + hwType: nvidia_switch_nd5200_ld + hostCount: 126 + oobDhcpRelayAddress: "10.100.0.1" + power: + hwType: liteon_power_shelf + hostCount: 112 + oobDhcpRelayAddress: "10.100.0.1" + + pod-1: + cidr: "10.100.4.0/22" + machines: + compute: + hwType: wiwynn_gb200_nvl + hostCount: 252 + dpuPerHostCount: 2 + oobDhcpRelayAddress: "10.100.4.1" + switches: + hwType: nvidia_switch_nd5200_ld + hostCount: 126 + oobDhcpRelayAddress: "10.100.4.1" + power: + hwType: liteon_power_shelf + hostCount: 112 + oobDhcpRelayAddress: "10.100.4.1" + + pod-2: + cidr: "10.100.8.0/22" + machines: + # ... same pattern ``` -**Why BGP?** Without `aggregationLength`, MetalLB advertises each service IP as an -individual /32 route. With thousands of services, this overwhelms upstream router control planes. +### What Gets Created -### Pros and Cons (MetalLB Mode) +**Per pod:** +| Resource | Name Pattern | +|----------|--------------| +| Deployment | `nico-machine-a-tron-pod-0` | +| ConfigMap | `nico-machine-a-tron-pod-0-config-files` | +| Certificate | `nico-machine-a-tron-pod-0-certificate` | +| Service | `nico-machine-a-tron-pod-0-bmc-mock` | -**Pros:** -- Compatible with real hardware -- Enables realistic load testing +**Per BMC:** +| Resource | Name Pattern | +|----------|--------------| +| ClusterIP Service | `nico-machine-a-tron-bmc-10-100-0-2` | -**Cons:** -- Requires MetalLB with BGP for large scale -- More complex network setup -- Requires dedicated IP range +**Cluster-scoped:** +| Resource | Name | +|----------|------| +| ServiceCIDR | `nico-machine-a-tron-bmc-cidr` | ---- - -## Configuration Reference +### Scale Guidelines -### Key Parameters +**BMC count per GB200 NVL72 rack:** +| Component | Count | BMCs per Unit | Total | +|-----------|-------|---------------|-------| +| Compute trays | 18 | 3 (1 tray + 2 BF3) | 54 | +| NVLink switches | 9 | 1 | 9 | +| Power shelves | 8 | 1 | 8 | +| **Total per rack** | | | **71** | -| Parameter | Description | Default | -|-----------|-------------|---------| -| `machineATron.nicoApiUrl` | NICo API server URL | `https://nico-api.nico-system.svc.cluster.local:1079` | -| `machineATron.cleanupOnQuit` | Delete machines from API on shutdown | `false` | -| `machines..hostCount` | Number of hosts to simulate | `10` | -| `machines..dpuPerHostCount` | DPUs per host | `2` | -| `machines..hwType` | Hardware type | `dell_poweredge_r750` | -| `machines..oobDhcpRelayAddress` | BMC network gateway | `10.100.0.1` | -| `machines..adminDhcpRelayAddress` | Admin network gateway | `10.101.0.1` | -| `nginxBmcProxy.enabled` | Enable MetalLB mode | `false` | -| `nginxBmcProxy.ipRange` | MetalLB IP pool range | `""` | -| `nginxBmcProxy.bgp.enabled` | Enable BGP advertisement | `false` | -| `terminationGracePeriodSeconds` | Shutdown timeout | `60` | +**CIDR sizing:** +| CIDR | Usable IPs | Racks per Pod | +|------|------------|---------------| +| /24 | 253 | ~3 | +| /23 | 509 | ~7 | +| /22 | 1021 | ~14 | +| /21 | 2045 | ~28 | -### Machine Groups +### NICo Configuration -Configure multiple machine groups with different hardware types: +**DO NOT set `override_target_host`** - let NICo connect to actual BMC IPs: -```yaml -machines: - gb300-hosts: - hwType: supermicro_gb300_nvl - hostCount: 100 - dpuPerHostCount: 2 - oobDhcpRelayAddress: "10.100.0.1" - adminDhcpRelayAddress: "10.101.0.1" - - switches: - hwType: nvidia_switch_nd5200_ld - hostCount: 10 - dpuPerHostCount: 0 - oobDhcpRelayAddress: "10.100.0.1" - adminDhcpRelayAddress: "10.101.0.1" - - power-shelves: - hwType: liteon_power_shelf - hostCount: 5 - dpuPerHostCount: 0 - oobDhcpRelayAddress: "10.100.0.1" - adminDhcpRelayAddress: "10.101.0.1" +```toml +[site_explorer] +enabled = true +create_machines = true +# override_target_host = ... # DO NOT SET ``` -### Supported Hardware Types - -From `HostHardwareType` enum in `crates/bmc-mock/src/lib.rs`: - -- `dell_poweredge_r750` (default) -- `supermicro_gb300_nvl` -- `nvidia_dgx_gb300` -- `nvidia_dgx_h100` -- `wiwynn_gb200_nvl` -- `lenovo_gb300_nvl` -- `liteon_power_shelf` -- `nvidia_switch_nd5200_ld` -- `generic_ami` -- `generic_supermicro` - -### Service Count Calculation - -The chart auto-calculates LoadBalancer services (MetalLB mode) from machine config: +**Network config per pod:** +```toml +[networks.pod-0-oob] +type = "underlay" +prefix = "10.100.0.0/22" +gateway = "10.100.0.1" +[networks.pod-1-oob] +type = "underlay" +prefix = "10.100.4.0/22" +gateway = "10.100.4.1" ``` -totalBMCs = Σ(hostCount + hostCount × dpuPerHostCount) for all groups -``` - -| Config | Calculation | Services | -|--------|-------------|----------| -| 10 hosts, 2 DPUs | 10 + 20 | 30 | -| 100 hosts, 2 DPUs | 100 + 200 | 300 | -| 4500 hosts, 2 DPUs | 4500 + 9000 | 13500 | --- -## Additional Features +## Configuration Reference -### Graceful Shutdown +### Pod Configuration ```yaml -terminationGracePeriodSeconds: 60 # Seconds to wait before SIGKILL - -machineATron: - cleanupOnQuit: true # Delete machines from NICo API +pods: + : + cidr: "" # Required for bmcServices mode + machines: + : + hwType: wiwynn_gb200_nvl + hostCount: 10 + dpuPerHostCount: 2 + oobDhcpRelayAddress: "10.100.0.1" + adminDhcpRelayAddress: "192.168.176.1" + # ... other machine settings ``` -### Persistence - -Preserve machine state across restarts: +### BMC Services Configuration ```yaml -persistence: - enabled: true - storageClass: "standard" - size: 1Gi +bmcServices: + enabled: false + servicePort: 443 # External HTTPS port + serviceCIDR: + create: true + name: "" # Defaults to -bmc-cidr + cidr: "" # Auto-detected for single pod, required for multi-pod ``` -### Monitoring +### Supported Hardware Types -Enable Prometheus metrics: - -```yaml -serviceMonitor: - enabled: true - interval: 30s -``` +| Type | Description | +|------|-------------| +| `supermicro_gb300_nvl` | Supermicro GB300 NVL | +| `nvidia_dgx_gb300` | NVIDIA DGX GB300 | +| `nvidia_dgx_h100` | NVIDIA DGX H100 | +| `wiwynn_gb200_nvl` | Wiwynn GB200 NVL | +| `lenovo_gb300_nvl` | Lenovo GB300 NVL | +| `dell_poweredge_r750` | Dell PowerEdge R750 | +| `liteon_power_shelf` | Liteon Power Shelf | +| `nvidia_switch_nd5200_ld` | NVIDIA ND5200 Switch | +| `generic_ami` | Generic AMI BMC | +| `generic_supermicro` | Generic Supermicro BMC | --- ## Troubleshooting -### Pod CrashLoopBackOff +### ServiceCIDR Not Ready -Check logs: ```bash -kubectl -n nico-mat logs deployment/nico-machine-a-tron +kubectl get servicecidr -o wide ``` -Common causes: -- NICo API unreachable - check `machineATron.nicoApiUrl` -- Certificate issues - check cert-manager -- Missing network config in NICo +Causes: +- CIDR outside cluster's default service CIDR +- Kubernetes version < 1.29 -### MetalLB Services Not Getting IPs +### Service IP Allocation Failed ```bash -# Check MetalLB speaker logs -kubectl -n metallb-system logs -l app=metallb,component=speaker +kubectl -n nico-mat describe svc nico-machine-a-tron-bmc-10-100-0-2 +``` -# Verify IPAddressPool exists -kubectl -n metallb-system get ipaddresspools +Causes: +- IP already in use +- ServiceCIDR not ready -# Check BGPAdvertisement -kubectl -n metallb-system get bgpadvertisements -``` +### Pod Not Receiving Traffic -### Scale Limit Error +Verify selector labels match: +```bash +# Check service selector +kubectl -n nico-mat get svc nico-machine-a-tron-bmc-10-100-0-2 -o jsonpath='{.spec.selector}' -If you see: -``` -SCALE LIMIT EXCEEDED: 16500 BMCs requested, maximum is 16384 (/18 subnet). +# Check pod labels +kubectl -n nico-mat get pods -l nvidia-infra-controller/pod-name=pod-0 --show-labels ``` -**Solution:** Reduce `hostCount` or `dpuPerHostCount` to stay within the /18 limit (16384 BMCs max). +### View Generated Config + +```bash +kubectl -n nico-mat get cm nico-machine-a-tron-pod-0-config-files -o yaml +``` diff --git a/helm/charts/nico-machine-a-tron/templates/bmc-proxy-configmap.yaml b/helm/charts/nico-machine-a-tron/templates/bmc-proxy-configmap.yaml deleted file mode 100644 index 53d2ef039e..0000000000 --- a/helm/charts/nico-machine-a-tron/templates/bmc-proxy-configmap.yaml +++ /dev/null @@ -1,50 +0,0 @@ -{{- if .Values.nginxBmcProxy.enabled }} -apiVersion: v1 -kind: ConfigMap -metadata: - name: {{ include "nico-machine-a-tron.name" . }}-bmc-proxy-config - namespace: {{ include "nico-machine-a-tron.namespace" . }} - labels: - {{- include "nico-machine-a-tron.labels" . | nindent 4 }} - app.kubernetes.io/component: nginx-bmc-proxy -data: - nginx.conf: | - worker_processes auto; - - events { - worker_connections 65535; - use epoll; - multi_accept on; - } - - http { - upstream machine_a_tron_backend { - server {{ include "nico-machine-a-tron.name" . }}:{{ .Values.service.bmcMock.port }}; - keepalive 256; - } - - server { - listen 443 ssl; - - ssl_certificate /etc/ssl/certs/tls.crt; - ssl_certificate_key /etc/ssl/private/tls.key; - ssl_protocols TLSv1.2 TLSv1.3; - ssl_ciphers HIGH:!annul:!MD5; - ssl_session_cache shared:SSL:10m; - ssl_session_timeout 10m; - - location / { - proxy_pass https://machine_a_tron_backend; - proxy_http_version 1.1; - proxy_set_header Connection ""; - proxy_set_header Host $host; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header X-Forwarded-Proto $scheme; - # Key header: tells machine-a-tron which BMC IP was requested - proxy_set_header Forwarded "host=$server_addr"; - proxy_ssl_verify off; - } - } - } -{{- end }} diff --git a/helm/charts/nico-machine-a-tron/templates/bmc-proxy-deployment.yaml b/helm/charts/nico-machine-a-tron/templates/bmc-proxy-deployment.yaml deleted file mode 100644 index a00980e4aa..0000000000 --- a/helm/charts/nico-machine-a-tron/templates/bmc-proxy-deployment.yaml +++ /dev/null @@ -1,82 +0,0 @@ -{{- if .Values.nginxBmcProxy.enabled }} -apiVersion: apps/v1 -kind: Deployment -metadata: - name: {{ include "nico-machine-a-tron.name" . }}-bmc-proxy - namespace: {{ include "nico-machine-a-tron.namespace" . }} - labels: - {{- include "nico-machine-a-tron.labels" . | nindent 4 }} - app.kubernetes.io/component: nginx-bmc-proxy -spec: - replicas: {{ .Values.nginxBmcProxy.replicas | default 1 }} - selector: - matchLabels: - {{- include "nico-machine-a-tron.selectorLabels" . | nindent 6 }} - app.kubernetes.io/component: nginx-bmc-proxy - template: - metadata: - labels: - {{- include "nico-machine-a-tron.selectorLabels" . | nindent 8 }} - app.kubernetes.io/component: nginx-bmc-proxy - annotations: - checksum/nginx-config: {{ include (print $.Template.BasePath "/bmc-proxy-configmap.yaml") . | sha256sum }} - spec: - ## Pod affinity: schedule nginx on the same node as machine-a-tron - ## to minimize network hops between proxy and backend - affinity: - podAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - - labelSelector: - matchLabels: - {{- include "nico-machine-a-tron.selectorLabels" . | nindent 18 }} - topologyKey: kubernetes.io/hostname - containers: - - name: nginx - image: {{ .Values.nginxBmcProxy.image | default "nginx:alpine" }} - imagePullPolicy: IfNotPresent - ports: - - name: https - containerPort: 443 - protocol: TCP - volumeMounts: - - name: nginx-config - mountPath: /etc/nginx/nginx.conf - subPath: nginx.conf - readOnly: true - - name: tls-certs - mountPath: /etc/ssl/certs - readOnly: true - - name: tls-key - mountPath: /etc/ssl/private - readOnly: true - {{- with .Values.nginxBmcProxy.resources }} - resources: - {{- toYaml . | nindent 12 }} - {{- end }} - livenessProbe: - tcpSocket: - port: https - initialDelaySeconds: 10 - periodSeconds: 30 - readinessProbe: - tcpSocket: - port: https - initialDelaySeconds: 5 - periodSeconds: 10 - volumes: - - name: nginx-config - configMap: - name: {{ include "nico-machine-a-tron.name" . }}-bmc-proxy-config - - name: tls-certs - secret: - secretName: {{ .Values.nginxBmcProxy.tlsSecretName | default (printf "%s-certificate" (include "nico-machine-a-tron.name" .)) }} - items: - - key: tls.crt - path: tls.crt - - name: tls-key - secret: - secretName: {{ .Values.nginxBmcProxy.tlsSecretName | default (printf "%s-certificate" (include "nico-machine-a-tron.name" .)) }} - items: - - key: tls.key - path: tls.key -{{- end }} diff --git a/helm/charts/nico-machine-a-tron/templates/bmc-service-cidr.yaml b/helm/charts/nico-machine-a-tron/templates/bmc-service-cidr.yaml new file mode 100644 index 0000000000..ea24422efd --- /dev/null +++ b/helm/charts/nico-machine-a-tron/templates/bmc-service-cidr.yaml @@ -0,0 +1,55 @@ +{{- if .Values.bmcServices.enabled }} +{{- if .Values.bmcServices.serviceCIDR.create }} +{{- $root := . }} +{{- $baseName := include "nico-machine-a-tron.name" . }} + +{{/* +Determine the ServiceCIDR to use: +1. If explicitly set in bmcServices.serviceCIDR.cidr, use that +2. Otherwise, use the CIDR from the first pod (single-pod case) +3. For multi-pod with different CIDRs, user must set serviceCIDR.cidr explicitly +*/}} +{{- $serviceCIDR := "" }} +{{- if .Values.bmcServices.serviceCIDR.cidr }} +{{- $serviceCIDR = .Values.bmcServices.serviceCIDR.cidr }} +{{- else }} +{{/* Collect all pod CIDRs */}} +{{- $cidrs := list }} +{{- range $podName, $podConfig := .Values.pods }} +{{- if $podConfig.cidr }} +{{- $cidrs = append $cidrs $podConfig.cidr }} +{{- end }} +{{- end }} +{{- if eq (len $cidrs) 1 }} +{{- $serviceCIDR = index $cidrs 0 }} +{{- else if gt (len $cidrs) 1 }} +{{- fail "Multiple pods with different CIDRs detected. Set bmcServices.serviceCIDR.cidr to a CIDR that covers all pod CIDRs." }} +{{- end }} +{{- end }} + +{{- if $serviceCIDR }} +--- +## ServiceCIDR reserves a dedicated IP range for BMC services. +## This prevents Kubernetes from allocating these IPs to other services. +## Requires Kubernetes 1.29+ (beta in 1.31+). +## +## The ServiceCIDR is cluster-scoped and persists independently of this release. +## Deleting the helm release will NOT delete the ServiceCIDR to prevent +## disruption to other releases using the same CIDR. +apiVersion: networking.k8s.io/v1beta1 +kind: ServiceCIDR +metadata: + name: {{ .Values.bmcServices.serviceCIDR.name | default (printf "%s-bmc-cidr" $baseName) }} + labels: + {{- include "nico-machine-a-tron.labels" . | nindent 4 }} + app.kubernetes.io/component: bmc-service-cidr + annotations: + helm.sh/resource-policy: keep + nvidia-infra-controller/purpose: "Reserved IP range for machine-a-tron BMC services" + nvidia-infra-controller/cidr: "{{ $serviceCIDR }}" +spec: + cidrs: + - {{ $serviceCIDR }} +{{- end }} +{{- end }} +{{- end }} diff --git a/helm/charts/nico-machine-a-tron/templates/bmc-services.yaml b/helm/charts/nico-machine-a-tron/templates/bmc-services.yaml new file mode 100644 index 0000000000..54f2353f2a --- /dev/null +++ b/helm/charts/nico-machine-a-tron/templates/bmc-services.yaml @@ -0,0 +1,131 @@ +{{- if .Values.bmcServices.enabled }} +{{- $root := . }} +{{- $namespace := include "nico-machine-a-tron.namespace" . }} +{{- $baseName := include "nico-machine-a-tron.name" . }} +{{- $servicePort := .Values.bmcServices.servicePort | default 443 }} +{{- $targetPort := .Values.service.bmcMock.port | default 1266 }} + +{{/* +Count pods that have CIDR configured (for naming and selector logic) +*/}} +{{- $podsWithCIDR := 0 }} +{{- range $podName, $podConfig := .Values.pods }} +{{- if $podConfig.cidr }} +{{- $podsWithCIDR = add $podsWithCIDR 1 }} +{{- end }} +{{- end }} + +{{- range $podName, $podConfig := .Values.pods }} +{{/* +Skip pods without CIDR (e.g., the default pod when custom pods are defined) +*/}} +{{- if not $podConfig.cidr }} +{{- continue }} +{{- end }} + +{{- $name := $baseName }} +{{- if gt $podsWithCIDR 1 }} +{{- $name = printf "%s-%s" $baseName $podName }} +{{- end }} + +{{/* +Parse CIDR and calculate available IPs +*/}} +{{- $cidrParts := split "/" $podConfig.cidr }} +{{- $networkIP := $cidrParts._0 }} +{{- $prefixLen := $cidrParts._1 | int }} +{{- $ipParts := split "." $networkIP }} +{{- $octet1 := $ipParts._0 | int }} +{{- $octet2 := $ipParts._1 | int }} +{{- $octet3 := $ipParts._2 | int }} +{{- $octet4 := $ipParts._3 | int }} + +{{/* +Calculate total IPs and usable IPs: 2^(32-prefix) - 3 (network, gateway, broadcast) +*/}} +{{- $hostBits := sub 32 $prefixLen }} +{{- $totalIPs := 1 }} +{{- range $i := until (int $hostBits) }} +{{- $totalIPs = mul $totalIPs 2 }} +{{- end }} +{{- $usableIPs := sub $totalIPs 3 }} + +{{/* +Calculate total BMCs needed for this pod +*/}} +{{- $totalBMCs := 0 }} +{{- range $sectionName, $section := $podConfig.machines }} +{{- if $section }} +{{- $hostCount := $section.hostCount | default 1 }} +{{- $dpuCount := $section.dpuPerHostCount | default 0 }} +{{- $sectionBMCs := add $hostCount (mul $hostCount $dpuCount) }} +{{- $totalBMCs = add $totalBMCs $sectionBMCs }} +{{- end }} +{{- end }} + +{{/* +Validate we have enough IPs +*/}} +{{- if gt (int $totalBMCs) (int $usableIPs) }} +{{- fail (printf "INSUFFICIENT IP CAPACITY for pod '%s': %d BMCs requested but CIDR %s only has %d usable IPs" $podName $totalBMCs $podConfig.cidr $usableIPs) }} +{{- end }} + +{{/* +Generate ClusterIP services for each BMC +Services start at .2 (skipping .0 network and .1 gateway) +*/}} +{{- $startOctet4 := add $octet4 2 }} +{{- range $bmcIndex := until (int $totalBMCs) }} +{{- $offset := $bmcIndex }} +{{- $o4 := add $startOctet4 $offset }} +{{- $o3 := $octet3 }} +{{- $o2 := $octet2 }} +{{- $o1 := $octet1 }} + +{{/* Handle octet overflows */}} +{{- if gt $o4 255 }} +{{- $o3 = add $o3 (div $o4 256) }} +{{- $o4 = mod $o4 256 }} +{{- end }} +{{- if gt $o3 255 }} +{{- $o2 = add $o2 (div $o3 256) }} +{{- $o3 = mod $o3 256 }} +{{- end }} +{{- if gt $o2 255 }} +{{- $o1 = add $o1 (div $o2 256) }} +{{- $o2 = mod $o2 256 }} +{{- end }} + +{{- $currentIP := printf "%d.%d.%d.%d" (int $o1) (int $o2) (int $o3) (int $o4) }} +{{- $safeName := $currentIP | replace "." "-" }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ $baseName }}-bmc-{{ $safeName }} + namespace: {{ $namespace }} + labels: + {{- include "nico-machine-a-tron.labels" $root | nindent 4 }} + app.kubernetes.io/component: bmc-service + nvidia-infra-controller/bmc-ip: "{{ $currentIP }}" + nvidia-infra-controller/bmc-index: "{{ $bmcIndex }}" + nvidia-infra-controller/pod-name: {{ $podName | quote }} + annotations: + nvidia-infra-controller/purpose: "ClusterIP service for simulated BMC" + nvidia-infra-controller/pod-cidr: "{{ $podConfig.cidr }}" +spec: + type: ClusterIP + clusterIP: {{ $currentIP }} + selector: + {{- include "nico-machine-a-tron.selectorLabels" $root | nindent 4 }} + {{- if gt $podsWithCIDR 1 }} + nvidia-infra-controller/pod-name: {{ $podName | quote }} + {{- end }} + ports: + - name: https + port: {{ $servicePort }} + targetPort: {{ $targetPort }} + protocol: TCP +{{- end }} +{{- end }} +{{- end }} diff --git a/helm/charts/nico-machine-a-tron/templates/certificate.yaml b/helm/charts/nico-machine-a-tron/templates/certificate.yaml index 3cf32793f3..17d9401239 100644 --- a/helm/charts/nico-machine-a-tron/templates/certificate.yaml +++ b/helm/charts/nico-machine-a-tron/templates/certificate.yaml @@ -1,12 +1,55 @@ +{{- $root := . }} +{{- $namespace := include "nico-machine-a-tron.namespace" . }} +{{- $baseName := include "nico-machine-a-tron.name" . }} + +{{/* +Count active pods (same logic as deployment.yaml) +*/}} +{{- $activePods := 0 }} +{{- range $podName, $podConfig := .Values.pods }} +{{- if $root.Values.bmcServices.enabled }} +{{- if $podConfig.cidr }} +{{- $activePods = add $activePods 1 }} +{{- end }} +{{- else }} +{{- if $podConfig.machines }} +{{- $activePods = add $activePods 1 }} +{{- end }} +{{- end }} +{{- end }} + +{{- range $podName, $podConfig := .Values.pods }} +{{/* +Skip conditions (same as deployment.yaml) +*/}} +{{- if $root.Values.bmcServices.enabled }} +{{- if not $podConfig.cidr }} +{{- continue }} +{{- end }} +{{- else }} +{{- if not $podConfig.machines }} +{{- continue }} +{{- end }} +{{- end }} + +{{- $name := $baseName }} +{{- if gt $activePods 1 }} +{{- $name = printf "%s-%s" $baseName $podName }} +{{- end }} +--- apiVersion: cert-manager.io/v1 kind: Certificate metadata: - name: {{ include "nico-machine-a-tron.name" . }}-certificate - namespace: {{ include "nico-machine-a-tron.namespace" . }} + name: {{ $name }}-certificate + namespace: {{ $namespace }} labels: - {{- include "nico-machine-a-tron.labels" . | nindent 4 }} + {{- include "nico-machine-a-tron.labels" $root | nindent 4 }} + {{- if gt $activePods 1 }} + nvidia-infra-controller/pod-name: {{ $podName | quote }} + {{- end }} spec: - {{- $bmcMockDns := printf "%s-bmc-mock.%s.svc.cluster.local" (include "nico-machine-a-tron.name" .) (include "nico-machine-a-tron.namespace" .) }} - {{- $extraDns := prepend (.Values.certificate.extraDnsNames | default list) $bmcMockDns }} - {{- $certWithExtra := merge (dict "extraDnsNames" $extraDns) .Values.certificate }} - {{- include "nico-machine-a-tron.certificateSpec" (dict "name" (printf "%s-certificate" (include "nico-machine-a-tron.name" .)) "svcName" (include "nico-machine-a-tron.name" .) "namespace" (include "nico-machine-a-tron.namespace" .) "cert" $certWithExtra "global" .Values.global) | nindent 2 }} + {{- $bmcMockDns := printf "%s-bmc-mock.%s.svc.cluster.local" $name $namespace }} + {{- $extraDns := prepend ($root.Values.certificate.extraDnsNames | default list) $bmcMockDns }} + {{- $certWithExtra := merge (dict "extraDnsNames" $extraDns) $root.Values.certificate }} + {{- include "nico-machine-a-tron.certificateSpec" (dict "name" (printf "%s-certificate" $name) "svcName" $name "namespace" $namespace "cert" $certWithExtra "global" $root.Values.global) | nindent 2 }} +{{- end }} diff --git a/helm/charts/nico-machine-a-tron/templates/configmap.yaml b/helm/charts/nico-machine-a-tron/templates/configmap.yaml index cf4fc9a6bb..79e4f48ae8 100644 --- a/helm/charts/nico-machine-a-tron/templates/configmap.yaml +++ b/helm/charts/nico-machine-a-tron/templates/configmap.yaml @@ -1,45 +1,94 @@ +{{- $root := . }} +{{- $namespace := include "nico-machine-a-tron.namespace" . }} +{{- $baseName := include "nico-machine-a-tron.name" . }} + +{{/* +Count active pods (same logic as deployment.yaml) +*/}} +{{- $activePods := 0 }} +{{- range $podName, $podConfig := .Values.pods }} +{{- if $root.Values.bmcServices.enabled }} +{{- if $podConfig.cidr }} +{{- $activePods = add $activePods 1 }} +{{- end }} +{{- else }} +{{- if $podConfig.machines }} +{{- $activePods = add $activePods 1 }} +{{- end }} +{{- end }} +{{- end }} + +{{/* Machine defaults */}} +{{- $defaults := .Values.machineDefaults | default dict }} + +{{- range $podName, $podConfig := .Values.pods }} +{{/* +Skip conditions (same as deployment.yaml) +*/}} +{{- if $root.Values.bmcServices.enabled }} +{{- if not $podConfig.cidr }} +{{- continue }} +{{- end }} +{{- else }} +{{- if not $podConfig.machines }} +{{- continue }} +{{- end }} +{{- end }} + +{{- $name := $baseName }} +{{- if gt $activePods 1 }} +{{- $name = printf "%s-%s" $baseName $podName }} +{{- end }} +--- apiVersion: v1 kind: ConfigMap metadata: - name: {{ include "nico-machine-a-tron.name" . }}-config-files - namespace: {{ include "nico-machine-a-tron.namespace" . }} + name: {{ $name }}-config-files + namespace: {{ $namespace }} labels: - {{- include "nico-machine-a-tron.labels" . | nindent 4 }} + {{- include "nico-machine-a-tron.labels" $root | nindent 4 }} + {{- if gt $activePods 1 }} + nvidia-infra-controller/pod-name: {{ $podName | quote }} + {{- end }} data: mat.toml: | - {{- if .Values.configFiles.matConfig }} - {{- .Values.configFiles.matConfig | nindent 4 }} + {{- if and $root.Values.configFiles.matConfigs (index $root.Values.configFiles.matConfigs $podName) }} + {{- index $root.Values.configFiles.matConfigs $podName | nindent 4 }} {{- else }} # Machine-A-Tron configuration - generated by Helm - carbide_api_url = {{ .Values.machineATron.nicoApiUrl | quote }} - interface = {{ .Values.machineATron.interface | quote }} - tui_enabled = {{ .Values.machineATron.tuiEnabled }} - {{- if .Values.machineATron.logFile }} - log_file = {{ .Values.machineATron.logFile | quote }} + # Pod: {{ $podName }} + {{- if $podConfig.cidr }} + # CIDR: {{ $podConfig.cidr }} + {{- end }} + carbide_api_url = {{ $root.Values.machineATron.nicoApiUrl | quote }} + interface = {{ $root.Values.machineATron.interface | quote }} + tui_enabled = {{ $root.Values.machineATron.tuiEnabled }} + {{- if $root.Values.machineATron.logFile }} + log_file = {{ $root.Values.machineATron.logFile | quote }} {{- end }} - bmc_mock_port = {{ .Values.service.bmcMock.port }} - use_single_bmc_mock = {{ .Values.machineATron.useSingleBmcMock }} - mock_bmc_ssh_server = {{ .Values.machineATron.mockBmcSshServer }} - {{- if .Values.machineATron.mockBmcSshPort }} - mock_bmc_ssh_port = {{ .Values.machineATron.mockBmcSshPort }} + bmc_mock_port = {{ $root.Values.service.bmcMock.port }} + use_single_bmc_mock = {{ $root.Values.machineATron.useSingleBmcMock }} + mock_bmc_ssh_server = {{ $root.Values.machineATron.mockBmcSshServer }} + {{- if $root.Values.machineATron.mockBmcSshPort }} + mock_bmc_ssh_port = {{ $root.Values.machineATron.mockBmcSshPort }} {{- end }} - {{- if .Values.machineATron.configureBmcProxyHost }} - configure_carbide_bmc_proxy_host = {{ .Values.machineATron.configureBmcProxyHost | quote }} + {{- if $root.Values.machineATron.configureBmcProxyHost }} + configure_carbide_bmc_proxy_host = {{ $root.Values.machineATron.configureBmcProxyHost | quote }} {{- end }} - {{- if .Values.persistence.enabled }} - persist_dir = {{ .Values.machineATron.persistDir | quote }} + {{- if $root.Values.persistence.enabled }} + persist_dir = {{ $root.Values.machineATron.persistDir | quote }} {{- end }} - cleanup_on_quit = {{ .Values.machineATron.cleanupOnQuit }} - register_expected_machines = {{ .Values.machineATron.registerExpectedMachines }} - {{- if .Values.machineATron.hostBmcPassword }} - host_bmc_password = {{ .Values.machineATron.hostBmcPassword | quote }} + cleanup_on_quit = {{ $root.Values.machineATron.cleanupOnQuit }} + register_expected_machines = {{ $root.Values.machineATron.registerExpectedMachines }} + {{- if $root.Values.machineATron.hostBmcPassword }} + host_bmc_password = {{ $root.Values.machineATron.hostBmcPassword | quote }} {{- end }} - {{- if .Values.machineATron.dpuBmcPassword }} - dpu_bmc_password = {{ .Values.machineATron.dpuBmcPassword | quote }} + {{- if $root.Values.machineATron.dpuBmcPassword }} + dpu_bmc_password = {{ $root.Values.machineATron.dpuBmcPassword | quote }} {{- end }} - api_refresh_interval = {{ .Values.machineATron.apiRefreshInterval | quote }} + api_refresh_interval = {{ $root.Values.machineATron.apiRefreshInterval | quote }} - {{- range $sectionName, $section := .Values.machines }} + {{- range $sectionName, $section := $podConfig.machines }} {{- if $section }} [machines.{{ $sectionName }}] @@ -47,22 +96,24 @@ data: hw_type = {{ $section.hwType | quote }} {{- end }} host_count = {{ $section.hostCount | default 1 }} - vpc_count = {{ $section.vpcCount | default 0 }} - subnets_per_vpc = {{ $section.subnetsPerVpc | default 0 }} - dpu_per_host_count = {{ $section.dpuPerHostCount | default 1 }} - dpu_reboot_delay = {{ $section.dpuRebootDelay | default 1 }} - host_reboot_delay = {{ $section.hostRebootDelay | default 1 }} - scout_run_interval = {{ $section.scoutRunInterval | default "60s" | quote }} - run_interval_working = {{ $section.runIntervalWorking | default "1s" | quote }} - run_interval_idle = {{ $section.runIntervalIdle | default "10s" | quote }} - network_status_run_interval = {{ $section.networkStatusRunInterval | default "20s" | quote }} - template_dir = {{ $section.templateDir | default "/opt/machine-a-tron/templates" | quote }} + vpc_count = {{ $section.vpcCount | default $defaults.vpcCount | default 0 }} + subnets_per_vpc = {{ $section.subnetsPerVpc | default $defaults.subnetsPerVpc | default 0 }} + dpu_per_host_count = {{ $section.dpuPerHostCount | default 0 }} + dpu_reboot_delay = {{ $section.dpuRebootDelay | default $defaults.dpuRebootDelay | default 1 }} + host_reboot_delay = {{ $section.hostRebootDelay | default $defaults.hostRebootDelay | default 1 }} + scout_run_interval = {{ $section.scoutRunInterval | default $defaults.scoutRunInterval | default "60s" | quote }} + run_interval_working = {{ $section.runIntervalWorking | default $defaults.runIntervalWorking | default "1s" | quote }} + run_interval_idle = {{ $section.runIntervalIdle | default $defaults.runIntervalIdle | default "10s" | quote }} + network_status_run_interval = {{ $section.networkStatusRunInterval | default $defaults.networkStatusRunInterval | default "20s" | quote }} + template_dir = {{ $section.templateDir | default $defaults.templateDir | default "/opt/machine-a-tron/templates" | quote }} oob_dhcp_relay_address = {{ $section.oobDhcpRelayAddress | default "192.168.192.1" | quote }} admin_dhcp_relay_address = {{ $section.adminDhcpRelayAddress | default "192.168.176.1" | quote }} - {{- if $section.networkVirtualizationType }} - network_virtualization_type = {{ $section.networkVirtualizationType | quote }} + {{- $nvType := $section.networkVirtualizationType | default $defaults.networkVirtualizationType }} + {{- if $nvType }} + network_virtualization_type = {{ $nvType | quote }} {{- end }} - dpus_in_nic_mode = {{ $section.dpusInNicMode | default false }} + dpus_in_nic_mode = {{ $section.dpusInNicMode | default $defaults.dpusInNicMode | default false }} {{- end }} {{- end }} {{- end }} +{{- end }} diff --git a/helm/charts/nico-machine-a-tron/templates/deployment.yaml b/helm/charts/nico-machine-a-tron/templates/deployment.yaml index 86758584b9..05074aa06f 100644 --- a/helm/charts/nico-machine-a-tron/templates/deployment.yaml +++ b/helm/charts/nico-machine-a-tron/templates/deployment.yaml @@ -1,66 +1,118 @@ +{{- $root := . }} +{{- $namespace := include "nico-machine-a-tron.namespace" . }} +{{- $baseName := include "nico-machine-a-tron.name" . }} + +{{/* +Count pods that should be deployed: +- When bmcServices.enabled: only pods with CIDR +- Otherwise: all pods with machines +*/}} +{{- $activePods := 0 }} +{{- range $podName, $podConfig := .Values.pods }} +{{- if $root.Values.bmcServices.enabled }} +{{- if $podConfig.cidr }} +{{- $activePods = add $activePods 1 }} +{{- end }} +{{- else }} +{{- if $podConfig.machines }} +{{- $activePods = add $activePods 1 }} +{{- end }} +{{- end }} +{{- end }} + +{{- range $podName, $podConfig := .Values.pods }} +{{/* +Skip conditions: +- When bmcServices.enabled: skip pods without CIDR +- Otherwise: skip pods without machines +*/}} +{{- if $root.Values.bmcServices.enabled }} +{{- if not $podConfig.cidr }} +{{- continue }} +{{- end }} +{{- else }} +{{- if not $podConfig.machines }} +{{- continue }} +{{- end }} +{{- end }} + +{{- $name := $baseName }} +{{- if gt $activePods 1 }} +{{- $name = printf "%s-%s" $baseName $podName }} +{{- end }} +--- apiVersion: apps/v1 kind: Deployment metadata: - name: {{ include "nico-machine-a-tron.name" . }} - namespace: {{ include "nico-machine-a-tron.namespace" . }} + name: {{ $name }} + namespace: {{ $namespace }} labels: - {{- include "nico-machine-a-tron.labels" . | nindent 4 }} + {{- include "nico-machine-a-tron.labels" $root | nindent 4 }} + {{- if gt $activePods 1 }} + nvidia-infra-controller/pod-name: {{ $podName | quote }} + {{- end }} annotations: - {{- tpl (toYaml .Values.annotations) . | nindent 4 }} + {{- tpl (toYaml $root.Values.annotations) $root | nindent 4 }} spec: - replicas: {{ .Values.replicas }} + replicas: 1 strategy: type: Recreate selector: matchLabels: - {{- include "nico-machine-a-tron.selectorLabels" . | nindent 6 }} + {{- include "nico-machine-a-tron.selectorLabels" $root | nindent 6 }} + {{- if gt $activePods 1 }} + nvidia-infra-controller/pod-name: {{ $podName | quote }} + {{- end }} template: metadata: labels: - {{- include "nico-machine-a-tron.labels" . | nindent 8 }} + {{- include "nico-machine-a-tron.labels" $root | nindent 8 }} + {{- if gt $activePods 1 }} + nvidia-infra-controller/pod-name: {{ $podName | quote }} + {{- end }} annotations: - kubectl.kubernetes.io/default-container: {{ include "nico-machine-a-tron.name" . }} + kubectl.kubernetes.io/default-container: {{ $baseName }} spec: - automountServiceAccountToken: {{ .Values.automountServiceAccountToken }} - serviceAccountName: {{ include "nico-machine-a-tron.name" . }} - terminationGracePeriodSeconds: {{ .Values.terminationGracePeriodSeconds }} - {{- with .Values.global.imagePullSecrets }} + automountServiceAccountToken: {{ $root.Values.automountServiceAccountToken }} + serviceAccountName: {{ $baseName }} + terminationGracePeriodSeconds: {{ $root.Values.terminationGracePeriodSeconds }} + {{- with $root.Values.global.imagePullSecrets }} imagePullSecrets: {{- toYaml . | nindent 8 }} {{- end }} containers: - - name: {{ include "nico-machine-a-tron.name" . }} - image: "{{ include "nico-machine-a-tron.image" . }}" - imagePullPolicy: {{ (.Values.global.image).pullPolicy | default .Values.image.pullPolicy | default "IfNotPresent" }} + - name: {{ $baseName }} + image: "{{ include "nico-machine-a-tron.image" $root }}" + imagePullPolicy: {{ ($root.Values.global.image).pullPolicy | default $root.Values.image.pullPolicy | default "IfNotPresent" }} command: - {{- toYaml .Values.command | nindent 12 }} + {{- toYaml $root.Values.command | nindent 12 }} env: - name: VAULT_ROLE_ID valueFrom: secretKeyRef: - name: {{ .Values.envFrom.vaultApproleTokens.secretName }} + name: {{ $root.Values.envFrom.vaultApproleTokens.secretName }} key: VAULT_ROLE_ID optional: true - name: VAULT_SECRET_ID valueFrom: secretKeyRef: - name: {{ .Values.envFrom.vaultApproleTokens.secretName }} + name: {{ $root.Values.envFrom.vaultApproleTokens.secretName }} key: VAULT_SECRET_ID optional: true - name: VAULT_TOKEN valueFrom: secretKeyRef: - name: {{ .Values.envFrom.vaultToken.secretName }} + name: {{ $root.Values.envFrom.vaultToken.secretName }} key: token optional: true - name: VAULT_ADDR valueFrom: configMapKeyRef: - name: {{ .Values.envFrom.vaultClusterInfo.configMapName }} + name: {{ $root.Values.envFrom.vaultClusterInfo.configMapName }} key: VAULT_SERVICE optional: true - name: MACHINE_A_TRON_CONFIG_PATH - value: "/etc/nico/machine-a-tron/mat.toml" + value: "/etc/nvidia-infra-controller/machine-a-tron/mat.toml" - name: FORGE_ROOT_CA_PATH value: "/var/run/secrets/spiffe.io/ca.crt" - name: CLIENT_CERT_PATH @@ -68,28 +120,28 @@ spec: - name: CLIENT_KEY_PATH value: "/var/run/secrets/spiffe.io/tls.key" - name: RUST_BACKTRACE - value: {{ .Values.env.RUST_BACKTRACE | quote }} + value: {{ $root.Values.env.RUST_BACKTRACE | quote }} - name: RUST_LIB_BACKTRACE - value: {{ .Values.env.RUST_LIB_BACKTRACE | quote }} - {{- with .Values.extraEnv }} + value: {{ $root.Values.env.RUST_LIB_BACKTRACE | quote }} + {{- with $root.Values.extraEnv }} {{- toYaml . | nindent 12 }} {{- end }} ports: - name: redfish - containerPort: {{ .Values.service.bmcMock.port }} + containerPort: {{ $root.Values.service.bmcMock.port }} - name: ssh - containerPort: {{ .Values.machineATron.mockBmcSshPort }} + containerPort: {{ $root.Values.machineATron.mockBmcSshPort }} - name: metrics - containerPort: {{ .Values.service.metrics.port }} + containerPort: {{ $root.Values.service.metrics.port }} livenessProbe: - {{- toYaml .Values.livenessProbe | nindent 12 }} + {{- toYaml $root.Values.livenessProbe | nindent 12 }} readinessProbe: - {{- toYaml .Values.readinessProbe | nindent 12 }} + {{- toYaml $root.Values.readinessProbe | nindent 12 }} volumeMounts: - - name: {{ include "nico-machine-a-tron.name" . }}-config-files + - name: config-files mountPath: /etc/forge/machine-a-tron - - name: {{ include "nico-machine-a-tron.name" . }}-config-files - mountPath: /etc/nico/machine-a-tron + - name: config-files + mountPath: /etc/nvidia-infra-controller/machine-a-tron - name: spiffe mountPath: "/var/run/secrets/spiffe.io" readOnly: true @@ -99,24 +151,28 @@ spec: - name: nico-roots mountPath: "/var/run/secrets/nico-roots" readOnly: true - {{- if .Values.persistence.enabled }} + {{- if $root.Values.persistence.enabled }} - name: data - mountPath: {{ .Values.machineATron.persistDir }} + mountPath: {{ $root.Values.machineATron.persistDir }} + {{- if gt $activePods 1 }} + subPath: {{ $podName }} + {{- end }} {{- end }} resources: - {{- toYaml .Values.resources | nindent 12 }} + {{- toYaml $root.Values.resources | nindent 12 }} volumes: - - name: {{ include "nico-machine-a-tron.name" . }}-config-files + - name: config-files configMap: - name: {{ include "nico-machine-a-tron.name" . }}-config-files + name: {{ $name }}-config-files - name: spiffe secret: - secretName: {{ include "nico-machine-a-tron.name" . }}-certificate + secretName: {{ $name }}-certificate - name: nico-roots secret: secretName: nico-roots - {{- if .Values.persistence.enabled }} + {{- if $root.Values.persistence.enabled }} - name: data persistentVolumeClaim: - claimName: {{ include "nico-machine-a-tron.name" . }}-data + claimName: {{ $baseName }}-data {{- end }} +{{- end }} diff --git a/helm/charts/nico-machine-a-tron/templates/metallb-pool.yaml b/helm/charts/nico-machine-a-tron/templates/metallb-pool.yaml deleted file mode 100644 index e690fe65e4..0000000000 --- a/helm/charts/nico-machine-a-tron/templates/metallb-pool.yaml +++ /dev/null @@ -1,71 +0,0 @@ -{{- if .Values.nginxBmcProxy.enabled }} -{{- if .Values.nginxBmcProxy.ipPool.create }} -{{- if .Values.nginxBmcProxy.ipRange }} ---- -apiVersion: metallb.io/v1beta1 -kind: IPAddressPool -metadata: - name: {{ include "nico-machine-a-tron.name" . }}-bmc-pool - namespace: {{ .Values.nginxBmcProxy.ipPool.namespace | default "metallb-system" }} - labels: - {{- include "nico-machine-a-tron.labels" . | nindent 4 }} -spec: - addresses: - - {{ .Values.nginxBmcProxy.ipRange }} - {{- if .Values.nginxBmcProxy.ipPool.autoAssign }} - autoAssign: {{ .Values.nginxBmcProxy.ipPool.autoAssign }} - {{- end }} -{{- if .Values.nginxBmcProxy.bgp.enabled }} ---- -## BGP Advertisement with route aggregation -## CRITICAL: aggregationLength prevents advertising 2000+ individual /32 routes -## which would overwhelm upstream router control planes -apiVersion: metallb.io/v1beta1 -kind: BGPAdvertisement -metadata: - name: {{ include "nico-machine-a-tron.name" . }}-bmc-bgp - namespace: {{ .Values.nginxBmcProxy.ipPool.namespace | default "metallb-system" }} - labels: - {{- include "nico-machine-a-tron.labels" . | nindent 4 }} -spec: - ipAddressPools: - - {{ include "nico-machine-a-tron.name" . }}-bmc-pool - ## Aggregate all service IPs into a single /18 announcement - ## This is MANDATORY for ultra-scale (2000+ services) - aggregationLength: {{ .Values.nginxBmcProxy.bgp.aggregationLength | default 18 }} - {{- if .Values.nginxBmcProxy.bgp.aggregationLengthV6 }} - aggregationLengthV6: {{ .Values.nginxBmcProxy.bgp.aggregationLengthV6 }} - {{- end }} - {{- if .Values.nginxBmcProxy.bgp.localPref }} - localPref: {{ .Values.nginxBmcProxy.bgp.localPref }} - {{- end }} - {{- if .Values.nginxBmcProxy.bgp.communities }} - communities: - {{- toYaml .Values.nginxBmcProxy.bgp.communities | nindent 4 }} - {{- end }} - {{- if .Values.nginxBmcProxy.bgp.peers }} - peers: - {{- toYaml .Values.nginxBmcProxy.bgp.peers | nindent 4 }} - {{- end }} -{{- else if .Values.nginxBmcProxy.ipPool.l2Advertisement }} ---- -## L2 Advertisement - only suitable for small scale or same-segment testing -## WARNING: Not recommended for production ultra-scale deployments -apiVersion: metallb.io/v1beta1 -kind: L2Advertisement -metadata: - name: {{ include "nico-machine-a-tron.name" . }}-bmc-l2 - namespace: {{ .Values.nginxBmcProxy.ipPool.namespace | default "metallb-system" }} - labels: - {{- include "nico-machine-a-tron.labels" . | nindent 4 }} -spec: - ipAddressPools: - - {{ include "nico-machine-a-tron.name" . }}-bmc-pool - {{- if .Values.nginxBmcProxy.ipPool.l2Interfaces }} - interfaces: - {{- toYaml .Values.nginxBmcProxy.ipPool.l2Interfaces | nindent 4 }} - {{- end }} -{{- end }} -{{- end }} -{{- end }} -{{- end }} diff --git a/helm/charts/nico-machine-a-tron/templates/metallb-services.yaml b/helm/charts/nico-machine-a-tron/templates/metallb-services.yaml deleted file mode 100644 index 17cb22f00e..0000000000 --- a/helm/charts/nico-machine-a-tron/templates/metallb-services.yaml +++ /dev/null @@ -1,103 +0,0 @@ -{{- if .Values.nginxBmcProxy.enabled }} -{{- if .Values.nginxBmcProxy.ipRange }} -{{- $root := . }} -{{- $namespace := include "nico-machine-a-tron.namespace" . }} -{{- $name := include "nico-machine-a-tron.name" . }} -{{- $port := .Values.nginxBmcProxy.servicePort | default 443 }} - -{{/* -Calculate total BMC count from all machine groups: - totalBMCs = sum(hostCount + hostCount * dpuPerHostCount) for each group -*/}} -{{- $totalBMCs := 0 }} -{{- range $groupName, $group := .Values.machines }} -{{- if $group }} -{{- $hostCount := $group.hostCount | default 0 }} -{{- $dpuPerHost := $group.dpuPerHostCount | default 0 }} -{{- $groupTotal := add $hostCount (mul $hostCount $dpuPerHost) }} -{{- $totalBMCs = add $totalBMCs $groupTotal }} -{{- end }} -{{- end }} - -{{/* -SCALE LIMIT: Maximum /18 subnet (16384 IPs) to protect cluster stability -- Creating too many services risks overwhelming kube-proxy and MetalLB -- Use BGP mode with aggregation for large scale deployments -*/}} -{{- $maxServices := 16384 }} -{{- if gt $totalBMCs $maxServices }} -{{- fail (printf "SCALE LIMIT EXCEEDED: %d BMCs requested, maximum is %d (/18 subnet). Reduce hostCount or dpuPerHostCount." $totalBMCs $maxServices) }} -{{- end }} - -{{/* -Parse IP range: "10.100.0.2-10.100.7.254" -> start IP parts -We generate services from startIP to startIP + totalBMCs - 1 -TODO: Need a more elegant solution -*/}} -{{- $ipRange := .Values.nginxBmcProxy.ipRange }} -{{- $startIP := (split "-" $ipRange)._0 }} -{{- $ipParts := split "." $startIP }} -{{- $octet1 := $ipParts._0 | int }} -{{- $octet2 := $ipParts._1 | int }} -{{- $octet3 := $ipParts._2 | int }} -{{- $octet4 := $ipParts._3 | int }} - -{{/* Generate services for each BMC */}} -{{- range $i := until (int $totalBMCs) }} -{{/* -Calculate IP: baseIP + offset -Handle octet overflow: .4 → .3 → .2 → .1 -*/}} -{{- $offset := $i }} -{{- $o4 := add $octet4 $offset }} -{{- $o3 := $octet3 }} -{{- $o2 := $octet2 }} -{{- $o1 := $octet1 }} - -{{/* Handle octet 4 overflow */}} -{{- if gt $o4 255 }} -{{- $o3 = add $o3 (div $o4 256) }} -{{- $o4 = mod $o4 256 }} -{{- end }} - -{{/* Handle octet 3 overflow */}} -{{- if gt $o3 255 }} -{{- $o2 = add $o2 (div $o3 256) }} -{{- $o3 = mod $o3 256 }} -{{- end }} - -{{/* Handle octet 2 overflow */}} -{{- if gt $o2 255 }} -{{- $o1 = add $o1 (div $o2 256) }} -{{- $o2 = mod $o2 256 }} -{{- end }} - -{{- $currentIP := printf "%d.%d.%d.%d" (int $o1) (int $o2) (int $o3) (int $o4) }} -{{- $safeName := $currentIP | replace "." "-" }} ---- -apiVersion: v1 -kind: Service -metadata: - name: {{ $name }}-bmc-{{ $safeName }} - namespace: {{ $namespace }} - labels: - {{- include "nico-machine-a-tron.labels" $root | nindent 4 }} - app.kubernetes.io/component: bmc-lb - bmc-ip: "{{ $currentIP }}" - bmc-index: "{{ $i }}" - annotations: - metallb.universe.tf/loadBalancerIPs: "{{ $currentIP }}" -spec: - type: LoadBalancer - externalTrafficPolicy: Local - selector: - {{- include "nico-machine-a-tron.selectorLabels" $root | nindent 4 }} - app.kubernetes.io/component: nginx-bmc-proxy - ports: - - name: https - port: {{ $port }} - targetPort: 443 - protocol: TCP -{{- end }} -{{- end }} -{{- end }} diff --git a/helm/charts/nico-machine-a-tron/templates/service.yaml b/helm/charts/nico-machine-a-tron/templates/service.yaml index 808e60db53..a8faab7fc6 100644 --- a/helm/charts/nico-machine-a-tron/templates/service.yaml +++ b/helm/charts/nico-machine-a-tron/templates/service.yaml @@ -1,22 +1,68 @@ +{{- $root := . }} +{{- $namespace := include "nico-machine-a-tron.namespace" . }} +{{- $baseName := include "nico-machine-a-tron.name" . }} + +{{/* +Count active pods (same logic as deployment.yaml) +*/}} +{{- $activePods := 0 }} +{{- range $podName, $podConfig := .Values.pods }} +{{- if $root.Values.bmcServices.enabled }} +{{- if $podConfig.cidr }} +{{- $activePods = add $activePods 1 }} +{{- end }} +{{- else }} +{{- if $podConfig.machines }} +{{- $activePods = add $activePods 1 }} +{{- end }} +{{- end }} +{{- end }} + +{{- range $podName, $podConfig := .Values.pods }} +{{/* +Skip conditions (same as deployment.yaml) +*/}} +{{- if $root.Values.bmcServices.enabled }} +{{- if not $podConfig.cidr }} +{{- continue }} +{{- end }} +{{- else }} +{{- if not $podConfig.machines }} +{{- continue }} +{{- end }} +{{- end }} + +{{- $name := $baseName }} +{{- if gt $activePods 1 }} +{{- $name = printf "%s-%s" $baseName $podName }} +{{- end }} +--- apiVersion: v1 kind: Service metadata: - name: {{ include "nico-machine-a-tron.name" . }}-bmc-mock - namespace: {{ include "nico-machine-a-tron.namespace" . }} + name: {{ $name }}-bmc-mock + namespace: {{ $namespace }} labels: - {{- include "nico-machine-a-tron.labels" . | nindent 4 }} - app: {{ include "nico-machine-a-tron.name" . }} + {{- include "nico-machine-a-tron.labels" $root | nindent 4 }} + app: {{ $baseName }} + {{- if gt $activePods 1 }} + nvidia-infra-controller/pod-name: {{ $podName | quote }} + {{- end }} spec: selector: - {{- include "nico-machine-a-tron.selectorLabels" . | nindent 4 }} + {{- include "nico-machine-a-tron.selectorLabels" $root | nindent 4 }} + {{- if gt $activePods 1 }} + nvidia-infra-controller/pod-name: {{ $podName | quote }} + {{- end }} ports: - - port: {{ .Values.service.bmcMock.port }} + - port: {{ $root.Values.service.bmcMock.port }} name: redfish targetPort: redfish protocol: TCP - {{- if .Values.machineATron.mockBmcSshServer }} - - port: {{ .Values.service.ssh.port }} + {{- if $root.Values.machineATron.mockBmcSshServer }} + - port: {{ $root.Values.service.ssh.port }} name: ssh targetPort: ssh protocol: TCP {{- end }} +{{- end }} diff --git a/helm/charts/nico-machine-a-tron/values.yaml b/helm/charts/nico-machine-a-tron/values.yaml index c1f58e1975..0970c74f27 100644 --- a/helm/charts/nico-machine-a-tron/values.yaml +++ b/helm/charts/nico-machine-a-tron/values.yaml @@ -2,21 +2,19 @@ ## nico-machine-a-tron - Mock machine simulator for NICo testing ## ============================================================================= ## -## DEPLOYMENT MODES (mutually exclusive): +## DEPLOYMENT MODES: ## -## 1. OVERRIDE MODE (Development) - nginxBmcProxy.enabled: false +## 1. OVERRIDE MODE (Development) - bmcServices.enabled: false ## - NICo Site-Explorer redirects ALL Redfish calls to machine-a-tron -## - Simple setup, no MetalLB required -## - INCOMPATIBLE with real hardware (all BMC traffic goes to mock) -## - Requires: NICo site config with override_target_host set +## - Simple setup, single pod +## - INCOMPATIBLE with real hardware ## -## 2. METALLB MODE (Large-Scale Testing) - nginxBmcProxy.enabled: true -## - Each simulated BMC gets dedicated external IP via MetalLB -## - Compatible with real hardware (can run alongside real machines) -## - Requires: MetalLB with BGP for scale, dedicated IP range -## - Requires: NICo site config WITHOUT override_target_host +## 2. CLUSTERIP MODE (Large-Scale Testing) - bmcServices.enabled: true +## - Each simulated BMC gets dedicated ClusterIP service +## - Supports multi-pod deployments via `pods` configuration +## - Compatible with real hardware ## -## See README.md for detailed setup instructions for each mode. +## See README.md for detailed setup instructions. ## ## ============================================================================= @@ -44,11 +42,9 @@ image: namespaceOverride: "" -replicas: 1 automountServiceAccountToken: true ## Graceful shutdown timeout (seconds) -## Allows machine-a-tron to clean up resources when pod is deleted terminationGracePeriodSeconds: 60 annotations: @@ -77,7 +73,6 @@ certificate: identityNamespace: "" dnsNames: [] uris: [] - ## Extra DNS name for the bmc-mock service is added automatically in _helpers.tpl extraDnsNames: [] extraUris: [] @@ -123,207 +118,185 @@ serviceMonitor: interval: 30s scrapeTimeout: 25s -## Machine-a-tron specific configuration +## ============================================================================= +## Machine-a-tron Core Settings (shared across all pods) +## ============================================================================= machineATron: - ## URL of the NICo API server nicoApiUrl: "https://nico-api.nico-system.svc.cluster.local:1079" - - ## Network interface for BMC mock IP binding (not used in k8s single-mock mode) interface: "NOTUSED" - - ## TUI is disabled in kubernetes - logs go to stdout tuiEnabled: false - - ## Log file path (empty = stdout) logFile: "" - - ## Use single BMC mock with header-based routing (required for k8s) - ## Port is configured via service.bmcMock.port useSingleBmcMock: true - - ## Enable mock BMC SSH server mockBmcSshServer: true mockBmcSshPort: 2222 - - ## If set, machine-a-tron will configure NICo to use this as bmc_proxy host - ## Combined with service.bmcMock.port to form host:port configureBmcProxyHost: "" - - ## Directory for persisting machine state between restarts persistDir: "/tmp/machine-a-tron-data" - - ## Delete created machines from API on quit cleanupOnQuit: false - - ## Auto-register mock hosts as expected machines registerExpectedMachines: true - - ## Override BMC passwords (leave empty for factory defaults) hostBmcPassword: "" dpuBmcPassword: "" - - ## API refresh interval apiRefreshInterval: "2s" -## Machine configuration sections -## Each named section defines a group of mock machines with specific settings -## You can define multiple sections with different hardware types -## Example: -## machines: -## dell-hosts: -## hwType: dell_poweredge_r750 -## hostCount: 10 -## dpuPerHostCount: 2 -## ... -## dgx-hosts: -## hwType: nvidia_dgx_h100 -## hostCount: 5 -## dpuPerHostCount: 1 -## ... -machines: - ## Machine group section. The name "dell-hosts" is arbitrary - you can rename it - ## or add more sections (e.g., "dgx-hosts", "power-shelves") to simulate different - ## hardware types. Each section creates a separate group of mock machines. - dell-hosts: - ## Hardware type (serde names from HostHardwareType enum): - ## dell_poweredge_r750 (default), wiwynn_gb200_nvl, lenovo_gb300_nvl, - ## nvidia_dgx_gb300, supermicro_gb300_nvl, liteon_power_shelf, - ## nvidia_switch_nd5200_ld, nvidia_dgx_h100, generic_ami, generic_supermicro - hwType: dell_poweredge_r750 - ## Number of host machines to create - hostCount: 10 - ## Number of VPCs to create - vpcCount: 0 - ## Number of subnets per VPC - subnetsPerVpc: 0 - ## Number of DPUs per host - dpuPerHostCount: 2 - ## Reboot delay in seconds (short for dev/test) - dpuRebootDelay: 1 - hostRebootDelay: 1 - ## State machine intervals (faster for dev) - scoutRunInterval: "60s" - runIntervalWorking: "1s" - runIntervalIdle: "10s" - networkStatusRunInterval: "20s" - ## Template directory for BMC mock data - templateDir: "/opt/machine-a-tron/templates" - ## DHCP relay addresses (must match site config) - oobDhcpRelayAddress: "192.168.192.1" - adminDhcpRelayAddress: "192.168.176.1" - ## Network virtualization type: etv, etv_nvue, or fnn - networkVirtualizationType: "" - ## DPUs run in NIC mode (no PXE boot) - dpusInNicMode: false - -## Persistence volume for machine state -persistence: - enabled: false - storageClass: "" - size: 1Gi - accessModes: - - ReadWriteOnce - -externalService: - enabled: false - type: LoadBalancer - externalTrafficPolicy: Local - annotations: {} - ## ============================================================================= -## BMC Proxy with MetalLB - Provides dedicated IPs for each simulated BMC +## Machine Defaults - Shared settings for all machine groups ## ============================================================================= -## This deploys an NGINX reverse proxy that adds the Forwarded header -## so machine-a-tron can route requests to the correct mock BMC based on IP. -## -## Architecture (NICo Site-Explorer connects DIRECTLY to MetalLB IPs): +## These defaults apply to all machine groups in all pods. +## Override per-group by specifying the value in pods.*.machines.*. ## -## Site-Explorer ---> MetalLB LB Services ---> NGINX ---> Machine-A-Tron -## (Redfish calls) (1 per BMC IP) (adds Forwarded header) +machineDefaults: + vpcCount: 0 + subnetsPerVpc: 0 + dpuRebootDelay: 1 + hostRebootDelay: 1 + scoutRunInterval: "60s" + runIntervalWorking: "1s" + runIntervalIdle: "10s" + networkStatusRunInterval: "20s" + templateDir: "/opt/machine-a-tron/templates" + networkVirtualizationType: "" + dpusInNicMode: false + +## ============================================================================= +## Pod Configurations - One pod per config entry +## ============================================================================= +## Each entry creates: +## - 1 Deployment (pod) +## - 1 ConfigMap with mat.toml +## - 1 Certificate +## - 1 internal Service +## - N ClusterIP BMC services (when bmcServices.enabled) ## -## HOW IP ALLOCATION WORKS: -## NICo allocates IPs sequentially from the network prefix via gRPC API. -## If the subnet is empty, IPs are assigned in order: .2, .3, .4, ... -## MetalLB services are created with matching sequential IPs. +## SCALE: Use /22 CIDR per pod = 1021 usable IPs = ~14 racks worth of BMCs ## -## Service count is AUTO-CALCULATED from machine configuration: -## totalBMCs = sum(hostCount + hostCount * dpuPerHostCount) for all groups +## BMC count per GB200 NVL72 rack: 71 +## - Compute trays: 18 × 3 = 54 +## - NVLink switches: 9 × 1 = 9 +## - Power shelves: 8 × 1 = 8 ## -## SCALE RECOMMENDATIONS: -## - /21 (2048 IPs) - Default, suitable for most test scenarios -## - /20 (4096 IPs) - Medium scale testing -## - /19 (8192 IPs) - Large scale testing -## - /18 (16384 IPs) - Maximum supported, requires BGP mode +## Per-machine group, only these fields are typically needed: +## - hwType: Hardware type (required) +## - hostCount: Number of hosts (required) +## - dpuPerHostCount: DPUs per host (default: 0) +## - oobDhcpRelayAddress: OOB network gateway +## - adminDhcpRelayAddress: Admin network gateway ## -## Example: 500 hosts with 2 DPUs each = 500 + 1000 = 1500 services (fits in /21) +## All other settings inherit from machineDefaults above. ## -nginxBmcProxy: - ## Enable the NGINX BMC proxy with MetalLB LoadBalancer services - enabled: false +pods: + ## Single-pod configuration is used when bmcServices.enabled=false (by default) + ## For multi-pod deployments, add more entries below and enable bmcServices + default: + cidr: "" + + ## Machine groups for this pod + machines: + rack-machines: + hwType: wiwynn_gb200_nvl + hostCount: 10 + dpuPerHostCount: 2 + oobDhcpRelayAddress: "192.168.192.1" + adminDhcpRelayAddress: "192.168.176.1" - ## ========================================================================== - ## NGINX Reverse Proxy Settings - ## ========================================================================== - image: nginx:alpine - replicas: 1 - resources: - limits: - cpu: 2 - memory: 1Gi - requests: - cpu: 500m - memory: 256Mi +## ============================================================================= +## Multi-Pod Example (bmcServices.enabled is required) +## ============================================================================= +## Uncomment and customize for large-scale deployments: +## +# mat-0: +# cidr: "10.100.0.0/22" # 1021 IPs for ~14 racks +# machines: +# compute: +# hwType: wiwynn_gb200_nvl +# hostCount: 252 # 18 trays × 14 racks +# dpuPerHostCount: 2 # 2 BF3 per tray +# oobDhcpRelayAddress: "10.100.0.1" +# adminDhcpRelayAddress: "192.168.176.1" +# switches: +# hwType: nvidia_switch_nd5200_ld +# hostCount: 126 # 9 switches × 14 racks +# oobDhcpRelayAddress: "10.100.0.1" +# adminDhcpRelayAddress: "192.168.176.1" +# power: +# hwType: liteon_power_shelf +# hostCount: 112 # 8 shelves × 14 racks +# oobDhcpRelayAddress: "10.100.0.1" +# adminDhcpRelayAddress: "192.168.176.1" +# +# mat-1: +# cidr: "10.100.4.0/22" # Next /22 block +# machines: +# compute: +# hwType: wiwynn_gb200_nvl +# hostCount: 252 +# dpuPerHostCount: 2 +# oobDhcpRelayAddress: "10.100.4.1" +# adminDhcpRelayAddress: "192.168.176.1" +# # ... same pattern - ## TLS certificate secret name (defaults to machine-a-tron certificate) - tlsSecretName: "" +## ============================================================================= +## BMC Services Configuration +## ============================================================================= +bmcServices: + enabled: false - ## Service port (external-facing) + ## External port for BMC services (matches real BMC HTTPS port) servicePort: 443 - ## ========================================================================== - ## MetalLB IP Pool Configuration - ## ========================================================================== - ## IP range for MetalLB pool. Services are created sequentially. - ## Number of services = total BMCs (auto-calculated from machine config) - ## - ## IMPORTANT: This IP range must match the NICo network configuration - ## so that IPs allocated by NICo match the MetalLB service IPs. - ## - ## NICo network config (nico-api-site-config.toml): - ## [networks.simulated-oob] - ## type = "underlay" - ## prefix = "10.100.0.0/21" - ## gateway = "10.100.0.1" - ## - ## Machine-a-tron config (values.yaml): - ## machines.dell-hosts.oobDhcpRelayAddress: "10.100.0.1" - ## - ## Examples by scale: - ## /21 (2048 IPs): "10.100.0.2-10.100.7.254" - ## /18 (16384 IPs): "10.100.0.2-10.100.63.254" max supported - ipRange: "" - - ipPool: - ## Create the MetalLB IPAddressPool resource + ## ServiceCIDR reserves the IP range (Kubernetes 1.29+) + serviceCIDR: create: true - ## Namespace for MetalLB resources - namespace: metallb-system - ## Enable auto-assign from this pool - autoAssign: true - ## L2 Advertisement (only for small-scale testing) - l2Advertisement: false + ## Name for the ServiceCIDR (cluster-scoped) + name: "" + ## Combined CIDR covering all pods (auto-calculated if empty for single pod) + ## Set manually if pods have non-contiguous CIDRs + cidr: "" - ## ========================================================================== - ## BGP Configuration - REQUIRED for large scale deployments - ## ========================================================================== - ## BGP mode with route aggregation prevents overwhelming upstream routers - ## with thousands of individual /32 routes. - bgp: - enabled: false - ## Route aggregation length - announces summary route instead of /32s - ## /21 = 2048 IPs (default), /18 = 16384 IPs (max) - aggregationLength: 21 +## Persistence volume for machine state +persistence: + enabled: false + storageClass: "" + size: 1Gi + accessModes: + - ReadWriteOnce + +externalService: + enabled: false + type: LoadBalancer + externalTrafficPolicy: Local + annotations: {} configFiles: - ## Override the entire mat.toml config (leave empty to use generated config) - matConfig: "" + ## Override mat.toml per pod (optional) + ## Key = pod name, value = full config content + ## When set, this completely replaces the auto-generated config for that pod. + ## + ## Example - simple single-pod config: + # matConfigs: + # default: | + # carbide_api_url = "https://nico-api.nico-system.svc.cluster.local:1079" + # interface = "NOTUSED" + # tui_enabled = false + # bmc_mock_port = 1266 + # use_single_bmc_mock = true + # mock_bmc_ssh_server = true + # mock_bmc_ssh_port = 2222 + # cleanup_on_quit = false + # register_expected_machines = true + # api_refresh_interval = "2s" + # + # [machines.compute] + # hw_type = "supermicro_gb300_nvl" + # host_count = 10 + # dpu_per_host_count = 2 + # oob_dhcp_relay_address = "10.100.0.1" + # admin_dhcp_relay_address = "192.168.176.1" + # template_dir = "/opt/machine-a-tron/templates" + # + # [machines.switches] + # hw_type = "nvidia_switch_nd5200_ld" + # host_count = 5 + # dpu_per_host_count = 0 + # oob_dhcp_relay_address = "10.100.0.1" + # admin_dhcp_relay_address = "192.168.176.1" + # template_dir = "/opt/machine-a-tron/templates" + matConfigs: {} From a5349550fab6d8c7daa14e328479a92c3dad8cb1 Mon Sep 17 00:00:00 2001 From: Alexander Korobkov Date: Tue, 7 Jul 2026 16:24:43 -0600 Subject: [PATCH 15/28] test(helm): nico machine-a-tron --- .../templates/external-service.yaml | 7 +- .../tests/certificate_test.yaml | 21 ++++++ .../tests/configmap_test.yaml | 69 ++++++++++++++++--- 3 files changed, 85 insertions(+), 12 deletions(-) diff --git a/helm/charts/nico-machine-a-tron/templates/external-service.yaml b/helm/charts/nico-machine-a-tron/templates/external-service.yaml index 4bde72e1dd..33f262c97f 100644 --- a/helm/charts/nico-machine-a-tron/templates/external-service.yaml +++ b/helm/charts/nico-machine-a-tron/templates/external-service.yaml @@ -1,4 +1,9 @@ -{{- if .Values.externalService.enabled }} +{{/* +External LoadBalancer service for Override Mode (development). +Only used when bmcServices is disabled - provides a single external endpoint. +When bmcServices.enabled, use ClusterIP services instead for per-BMC routing. +*/}} +{{- if and .Values.externalService.enabled (not .Values.bmcServices.enabled) }} apiVersion: v1 kind: Service metadata: diff --git a/helm/charts/nico-machine-a-tron/tests/certificate_test.yaml b/helm/charts/nico-machine-a-tron/tests/certificate_test.yaml index 7b67933546..20d7dd901c 100644 --- a/helm/charts/nico-machine-a-tron/tests/certificate_test.yaml +++ b/helm/charts/nico-machine-a-tron/tests/certificate_test.yaml @@ -19,3 +19,24 @@ tests: asserts: - isKind: of: Certificate + + - it: should create per-pod certificates in multi-pod mode + set: + bmcServices: + enabled: true + pods: + mat-0: + cidr: "10.100.0.0/22" + machines: + compute: + hwType: supermicro_gb300_nvl + hostCount: 5 + mat-1: + cidr: "10.100.4.0/22" + machines: + compute: + hwType: supermicro_gb300_nvl + hostCount: 5 + asserts: + - hasDocuments: + count: 2 diff --git a/helm/charts/nico-machine-a-tron/tests/configmap_test.yaml b/helm/charts/nico-machine-a-tron/tests/configmap_test.yaml index 035aa06603..41f46ac80c 100644 --- a/helm/charts/nico-machine-a-tron/tests/configmap_test.yaml +++ b/helm/charts/nico-machine-a-tron/tests/configmap_test.yaml @@ -16,28 +16,75 @@ tests: path: data["mat.toml"] pattern: "use_single_bmc_mock = true" - - it: should configure custom host count + - it: should configure custom host count via pods structure set: - machines: - dell-hosts: - hostCount: 20 - dpuPerHostCount: 4 + pods: + default: + machines: + dell-hosts: + hwType: dell_poweredge_r750 + hostCount: 20 + dpuPerHostCount: 2 asserts: - matchRegex: path: data["mat.toml"] pattern: "host_count = 20" - matchRegex: path: data["mat.toml"] - pattern: "dpu_per_host_count = 4" + pattern: "dpu_per_host_count = 2" + - matchRegex: + path: data["mat.toml"] + pattern: 'hw_type = "dell_poweredge_r750"' - - it: should allow config override + - it: should allow full config override via matConfigs set: configFiles: - matConfig: | - carbide_api_url = "https://custom-api:443" - [machines.custom] - host_count = 5 + matConfigs: + default: | + carbide_api_url = "https://custom-api:443" + [machines.custom] + host_count = 5 asserts: - matchRegex: path: data["mat.toml"] pattern: "custom-api" + + - it: should create per-pod configmaps in multi-pod mode + set: + bmcServices: + enabled: true + pods: + mat-0: + cidr: "10.100.0.0/22" + machines: + compute: + hwType: supermicro_gb300_nvl + hostCount: 10 + mat-1: + cidr: "10.100.4.0/22" + machines: + compute: + hwType: supermicro_gb300_nvl + hostCount: 20 + asserts: + - hasDocuments: + count: 2 + + - it: should use machineDefaults for all machine groups + set: + machineDefaults: + runIntervalIdle: "30s" + scoutRunInterval: "120s" + pods: + default: + machines: + compute: + hwType: supermicro_gb300_nvl + hostCount: 5 + asserts: + - matchRegex: + path: data["mat.toml"] + pattern: 'run_interval_idle = "30s"' + - matchRegex: + path: data["mat.toml"] + pattern: 'scout_run_interval = "120s"' From c815668980ab1644b1ceb08051110706d164d6b6 Mon Sep 17 00:00:00 2001 From: Alexander Korobkov Date: Wed, 8 Jul 2026 12:00:10 -0600 Subject: [PATCH 16/28] feat(dev): devspace use machine-a-tron chart --- dev/deployment/devspace/README.md | 18 ++- dev/deployment/devspace/machine-a-tron.yaml | 137 -------------------- dev/deployment/devspace/values.base.yaml | 14 ++ devspace.yaml | 38 ++---- 4 files changed, 35 insertions(+), 172 deletions(-) delete mode 100644 dev/deployment/devspace/machine-a-tron.yaml diff --git a/dev/deployment/devspace/README.md b/dev/deployment/devspace/README.md index 46c184233b..626d1a014a 100644 --- a/dev/deployment/devspace/README.md +++ b/dev/deployment/devspace/README.md @@ -99,7 +99,7 @@ DevSpace will: - build the local runtime images from [`Dockerfile.api`](Dockerfile.api), [`Dockerfile.bmc-proxy`](Dockerfile.bmc-proxy), and [`Dockerfile.machine-a-tron`](Dockerfile.machine-a-tron) - build the REST API, workflow, site-manager, site-agent, database migration, certificate-manager, and MCP images from [`rest-api/docker/local`](../../../rest-api/docker/local) -- deploy the Helm chart in [`helm/`](../../../helm) +- deploy the Helm chart in [`helm/`](../../../helm) (including `nico-machine-a-tron`) - deploy the REST umbrella, site-agent, and MCP charts in [`helm/rest`](../../../helm/rest) - apply the local-only `machine-a-tron` Kubernetes objects from [`machine-a-tron.yaml`](machine-a-tron.yaml) with `kubectl` - inject the built image names and DevSpace-generated tags into both deployments at runtime @@ -109,11 +109,9 @@ The image builds are configured in [`devspace.yaml`](../../../devspace.yaml). Th The DevSpace images also use Dockerfile-specific ignore files: [`Dockerfile.api.dockerignore`](Dockerfile.api.dockerignore), [`Dockerfile.bmc-proxy.dockerignore`](Dockerfile.bmc-proxy.dockerignore), and [`Dockerfile.machine-a-tron.dockerignore`](Dockerfile.machine-a-tron.dockerignore). This keeps the top-level [`.dockerignore`](../../../.dockerignore) aligned with the main branch for CI and release builds, while still giving the local DevSpace builds a small Docker context. -DevSpace watches the Rust workspace, toolchain metadata, and the runtime Dockerfiles to decide when images need rebuilding. +DevSpace watches the Rust workspace, toolchain metadata, and the runtime Dockerfiles to decide when images need rebuilding. On kind clusters, the pre-deploy hooks load all Core and REST images into the cluster selected by the current kube context. -The REST images use the existing local Dockerfiles and one shared per-run tag because the REST Helm charts intentionally consume a common repository and tag. On kind clusters, the pre-deploy hooks load all Core and REST images into the cluster selected by the current kube context. - -The production Helm chart is still only responsible for the product services. `machine-a-tron` is deployed separately as plain local-only Kubernetes objects in [`machine-a-tron.yaml`](machine-a-tron.yaml), with DevSpace wiring in the local image tag and certificate issuer from [`devspace.yaml`](../../../devspace.yaml). The local API and BMC proxy configs in [`values.base.yaml`](values.base.yaml) point BMC traffic at `machine-a-tron-bmc-mock.nico-system.svc.cluster.local:1266`. +The `nico-machine-a-tron` Helm subchart configuration is in [`values.base.yaml`](values.base.yaml). The local API and BMC proxy configs point BMC traffic at `nico-machine-a-tron-bmc-mock.nico-system.svc.cluster.local:1266`. Common usage: @@ -163,7 +161,13 @@ docker build -t "nico-bmc-proxy:" -f dev/deployment/devs docker build -t "machine-a-tron:" -f dev/deployment/devspace/Dockerfile.machine-a-tron . ``` -DevSpace then deploys the Helm chart with the built `nico-api` image wired into `global.image.repository` and `global.image.tag`, the built `nico-bmc-proxy` image wired into the `nico-bmc-proxy` chart values, and applies the local-only `machine-a-tron` manifest with its image wired into the `Deployment` spec. The REST images are built from the existing `rest-api/docker/local` Dockerfiles and are passed to the three existing REST Helm charts with the same generated tag. +DevSpace then deploys the Helm chart with: +- the built `nico-api` image wired into `global.image.repository` and `global.image.tag` +- the built `nico-bmc-proxy` image wired into the `nico-bmc-proxy` chart values +- the built `machine-a-tron` image wired into the `nico-machine-a-tron` chart values +- certificate issuer settings from the DevSpace environment variables + +The REST images are built from the existing `rest-api/docker/local` Dockerfiles and are passed to the three existing REST Helm charts with the same generated tag. ## Resetting the local environment @@ -175,7 +179,7 @@ Reset the complete local environment by running: devspace purge -n nico-system ``` -When the current context is `kind-`, the purge pipeline deletes and recreates that kind cluster with the same node image, then bootstraps clean prerequisites. This removes all Kubernetes state, including the Core and REST databases, Temporal namespaces and history, Vault data, Keycloak data, certificates, site registration, Helm releases, CRDs, and persistent volumes. +When the current context is `kind-`, the purge pipeline deletes and recreates that kind cluster with the same node image, then bootstraps clean prerequisites. This removes all Kubernetes state, including the Core and REST databases, Temporal namespaces and history, Vault data, Keycloak data, certificates, site registration, Helm releases (including machine-a-tron), CRDs, and persistent volumes. On any other Kubernetes context, the pipeline delegates to DevSpace's default purge behavior. It removes the deployments managed by this project without replacing the cluster or reinstalling separately managed prerequisites. diff --git a/dev/deployment/devspace/machine-a-tron.yaml b/dev/deployment/devspace/machine-a-tron.yaml deleted file mode 100644 index 135a4b860e..0000000000 --- a/dev/deployment/devspace/machine-a-tron.yaml +++ /dev/null @@ -1,137 +0,0 @@ -apiVersion: v1 -kind: ServiceAccount -metadata: - name: machine-a-tron ---- -apiVersion: v1 -kind: ConfigMap -metadata: - name: machine-a-tron-config -data: - mat.toml: | - carbide_api_url = "https://nico-api.nico-system.svc.cluster.local:1079" - interface = "NOTUSED" - tui_enabled = false - bmc_mock_port = 1266 - use_single_bmc_mock = true - mock_bmc_ssh_server = true - mock_bmc_ssh_port = 2222 - persist_dir = "/tmp/machine-a-tron-data" - register_expected_machines = true - - # machine-a-tron config generated from "just setup-k3s-env-ips" - [machines.config] - host_count = 10 - dpu_per_host_count = 2 - dpu_reboot_delay = 1 # in units of seconds - host_reboot_delay = 1 # in units of seconds - vpc_count = 0 - admin_dhcp_relay_address = "192.168.64.1" - oob_dhcp_relay_address = "192.168.192.1" - subnets_per_vpc = 0 - run_interval_working = "1s" - run_interval_idle = "10s" - template_dir = "/opt/machine-a-tron/templates" ---- -apiVersion: cert-manager.io/v1 -kind: Certificate -metadata: - name: machine-a-tron-certificate -spec: - duration: 720h0m0s - renewBefore: 360h0m0s - secretName: machine-a-tron-certificate - privateKey: - algorithm: ECDSA - size: 384 - issuerRef: - kind: Issuer - name: local-ca-issuer - group: cert-manager.io - dnsNames: - - machine-a-tron.nico-system.svc.cluster.local - - machine-a-tron-bmc-mock.nico-system.svc.cluster.local - uris: - - spiffe://nico.local/nico-system/sa/machine-a-tron ---- -apiVersion: v1 -kind: Service -metadata: - name: machine-a-tron-bmc-mock - labels: - app: machine-a-tron -spec: - selector: - app: machine-a-tron - ports: - - name: redfish - port: 1266 - targetPort: redfish - protocol: TCP - - name: ssh - port: 22 - targetPort: ssh - protocol: TCP ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - name: machine-a-tron - labels: - app: machine-a-tron -spec: - replicas: 1 - strategy: - type: Recreate - selector: - matchLabels: - app: machine-a-tron - template: - metadata: - labels: - app: machine-a-tron - spec: - serviceAccountName: machine-a-tron - containers: - - name: machine-a-tron - image: machine-a-tron - imagePullPolicy: IfNotPresent - command: - - /opt/machine-a-tron/bin/machine-a-tron - env: - - name: FORGE_ROOT_CA_PATH - value: /var/run/secrets/spiffe.io/ca.crt - - name: CLIENT_CERT_PATH - value: /var/run/secrets/spiffe.io/tls.crt - - name: CLIENT_KEY_PATH - value: /var/run/secrets/spiffe.io/tls.key - - name: MACHINE_A_TRON_CONFIG_PATH - value: /opt/machine-a-tron/config/mat.toml - ports: - - name: redfish - containerPort: 1266 - - name: ssh - containerPort: 2222 - readinessProbe: - tcpSocket: - port: redfish - initialDelaySeconds: 5 - periodSeconds: 10 - volumeMounts: - - name: spiffe - mountPath: /var/run/secrets/spiffe.io - readOnly: true - - name: config - mountPath: /opt/machine-a-tron/config - readOnly: true - - name: persist - mountPath: /tmp/machine-a-tron-data - volumes: - - name: spiffe - secret: - secretName: machine-a-tron-certificate - - name: config - configMap: - name: machine-a-tron-config - - name: persist - emptyDir: {} diff --git a/dev/deployment/devspace/values.base.yaml b/dev/deployment/devspace/values.base.yaml index cba862e532..e4e70fae3f 100644 --- a/dev/deployment/devspace/values.base.yaml +++ b/dev/deployment/devspace/values.base.yaml @@ -139,6 +139,20 @@ nico-bmc-proxy: client_cert = "/var/run/secrets/spiffe.io/tls.crt" client_key = "/var/run/secrets/spiffe.io/tls.key" +nico-machine-a-tron: + enabled: true + machineATron: + nicoApiUrl: "https://nico-api.nico-system.svc.cluster.local:1079" + pods: + default: + machines: + config: + hwType: wiwynn_gb200_nvl + hostCount: 10 + dpuPerHostCount: 2 + oobDhcpRelayAddress: "192.168.192.1" + adminDhcpRelayAddress: "192.168.176.1" + nico-dhcp: enabled: false nico-dns: diff --git a/devspace.yaml b/devspace.yaml index c0261daa28..aae7e68790 100644 --- a/devspace.yaml +++ b/devspace.yaml @@ -162,42 +162,24 @@ deployments: repository: ${runtime.images.nico-api.image} tag: ${runtime.images.nico-api.tag} pullPolicy: IfNotPresent + certificate: + issuerRef: + kind: ${LOCAL_DEV_CERT_ISSUER_KIND} + name: ${LOCAL_DEV_CERT_ISSUER_NAME} + group: ${LOCAL_DEV_CERT_ISSUER_GROUP} nico-bmc-proxy: image: repository: ${runtime.images.nico-bmc-proxy.image} tag: ${runtime.images.nico-bmc-proxy.tag} pullPolicy: IfNotPresent + nico-machine-a-tron: + image: + repository: ${runtime.images.machine-a-tron.image} + tag: ${runtime.images.machine-a-tron.tag} + pullPolicy: IfNotPresent valuesFiles: - dev/deployment/devspace/values.base.yaml - dev/deployment/devspace/values.generated.yaml - machine-a-tron-local: - namespace: ${LOCAL_DEV_NAMESPACE} - updateImageTags: true - kubectl: - manifests: - - dev/deployment/devspace/machine-a-tron.yaml - patches: - - target: - apiVersion: cert-manager.io/v1 - kind: Certificate - name: machine-a-tron-certificate - op: replace - path: spec.issuerRef.kind - value: ${LOCAL_DEV_CERT_ISSUER_KIND} - - target: - apiVersion: cert-manager.io/v1 - kind: Certificate - name: machine-a-tron-certificate - op: replace - path: spec.issuerRef.name - value: ${LOCAL_DEV_CERT_ISSUER_NAME} - - target: - apiVersion: cert-manager.io/v1 - kind: Certificate - name: machine-a-tron-certificate - op: replace - path: spec.issuerRef.group - value: ${LOCAL_DEV_CERT_ISSUER_GROUP} nico-rest: namespace: nico-rest updateImageTags: false From 0817144de72d407fc2ba5c1e17a4217de6292e75 Mon Sep 17 00:00:00 2001 From: Shayan Namaghi Date: Wed, 8 Jul 2026 11:43:39 -0700 Subject: [PATCH 17/28] feat(machine-a-tron): end-to-end deployment + scale testing to 4500 hosts MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Single consolidated commit on top of #2764 (ClusterIP migration). Adds everything needed to take a running NICo site from nothing to a simulated fleet with machines created and managed, validated live on dev6 at 1 -> 100 -> 1000 -> 4500 hosts (x2 DPUs = 13,500 BMC endpoints): - helm-prereqs/setup-machine-a-tron.sh: single idempotent script — namespace, pull secret, CA/Vault secret refresh, full BMC/UEFI credential chain, nico-core site config (bmc_proxy, simulated networks, site_explorer throughput knobs, lo-ip pool), DHCP dual-pool sizing with auto-fit, DB safety checks, cert reissue, throttled helm deploy, and a verification loop that shepherds ingestion (latch clearing + endpoint unparking). - helm-prereqs/cleanup-machine-a-tron.sh: full inverse for reproducible from-scratch runs (per-MAC Vault cred purge batched server-side). - values/machine-a-tron.yaml + machine-a-tron-scale.yaml on the #2764 pods..machines schema; scale mode uses PROXY-DIRECT transport (site_explorer.bmc_proxy + the Redfish client's own RFC 7239 "Forwarded: host=" header, routed by the mock's shared registry) — one ClusterIP service for the whole fleet, complementary to #2764's per-BMC ClusterIP Services for real-hardware coexistence. - fix(api): grant Machineatron the AddExpectedMachine permission — its register_expected_machines call was 403'd; the principal already holds the sibling grants (DiscoverDhcp, CreateNetworkSegment, GetExpectedSwitch). - Deployment guide + scale-testing writeup with the complete 19-issue log, measured ingestion rates, and open questions for review. Results: stage 2 = 3000/3000 machines END TO END OK in one unattended run (~25 min); stage 3 = 13,500/13,500 endpoints explored, 10k+ machines created and progressing through machine-controller lifecycle states on dev-sized postgres, with ingestion running autonomously through extended client connectivity outages. --- .../api-core/src/auth/internal_rbac_rules.rs | 2 +- crates/machine-a-tron/Dockerfile | 59 ++ docs/development/machine-a-tron-deployment.md | 323 +++++++ .../machine-a-tron-scale-testing.md | 168 ++++ helm-prereqs/cleanup-machine-a-tron.sh | 296 ++++++ helm-prereqs/setup-machine-a-tron.sh | 869 ++++++++++++++++++ helm-prereqs/values/machine-a-tron-scale.yaml | 98 ++ helm-prereqs/values/machine-a-tron.yaml | 105 +++ 8 files changed, 1919 insertions(+), 1 deletion(-) create mode 100644 crates/machine-a-tron/Dockerfile create mode 100644 docs/development/machine-a-tron-deployment.md create mode 100644 docs/development/machine-a-tron-scale-testing.md create mode 100755 helm-prereqs/cleanup-machine-a-tron.sh create mode 100755 helm-prereqs/setup-machine-a-tron.sh create mode 100644 helm-prereqs/values/machine-a-tron-scale.yaml create mode 100644 helm-prereqs/values/machine-a-tron.yaml diff --git a/crates/api-core/src/auth/internal_rbac_rules.rs b/crates/api-core/src/auth/internal_rbac_rules.rs index f331e5d4ed..0b04966f00 100644 --- a/crates/api-core/src/auth/internal_rbac_rules.rs +++ b/crates/api-core/src/auth/internal_rbac_rules.rs @@ -322,7 +322,7 @@ impl InternalRBACRules { x.perm("UpdateInstancePhoneHomeLastContact", vec![Agent]); x.perm("SetHostUefiPassword", vec![ForgeAdminCLI]); x.perm("ClearHostUefiPassword", vec![ForgeAdminCLI]); - x.perm("AddExpectedMachine", vec![ForgeAdminCLI, SiteAgent, Flow]); + x.perm("AddExpectedMachine", vec![ForgeAdminCLI, SiteAgent, Flow, Machineatron]); x.perm("DeleteExpectedMachine", vec![ForgeAdminCLI, SiteAgent]); x.perm("UpdateExpectedMachine", vec![ForgeAdminCLI, SiteAgent]); x.perm("CreateExpectedMachines", vec![ForgeAdminCLI, SiteAgent]); diff --git a/crates/machine-a-tron/Dockerfile b/crates/machine-a-tron/Dockerfile new file mode 100644 index 0000000000..041e1f10c2 --- /dev/null +++ b/crates/machine-a-tron/Dockerfile @@ -0,0 +1,59 @@ +# syntax=docker/dockerfile:1.7 +# +# Cross-compilation: native arm64 Rust compiler targeting x86_64-unknown-linux-gnu. +# Avoids QEMU emulation so that aws-lc-sys (s2n-bignum assembly) compiles correctly — +# QEMU SEGFAULTs on the .S files even though the binary runs fine on real x86_64. +# +# Build from the repo root: +# +# docker buildx build \ +# --platform linux/amd64 \ +# -f crates/machine-a-tron/Dockerfile \ +# --push \ +# -t //machine-a-tron:-amd64 \ +# . +# +FROM --platform=linux/arm64 rust:1.96.0-slim-bookworm AS builder + +RUN apt-get update && apt-get install -y --no-install-recommends \ + git \ + pkg-config \ + protobuf-compiler \ + libprotobuf-dev \ + gcc-x86-64-linux-gnu \ + libc6-dev-amd64-cross \ + && rm -rf /var/lib/apt/lists/* + +RUN rustup target add x86_64-unknown-linux-gnu + +ENV CARGO_HOME=/cargo-home +ENV CARGO_NET_GIT_FETCH_WITH_CLI=true +ENV CARGO_TARGET_DIR=/cargo-target +ENV CARGO_TARGET_X86_64_UNKNOWN_LINUX_GNU_LINKER=x86_64-linux-gnu-gcc +ENV CC_x86_64_unknown_linux_gnu=x86_64-linux-gnu-gcc +ENV RUST_BACKTRACE=1 + +WORKDIR /workspace +COPY . . + +RUN --mount=type=cache,id=nico-mat-cross-cargo-home,target=/cargo-home,sharing=locked \ + --mount=type=cache,id=nico-mat-cross-cargo-target,target=/cargo-target,sharing=locked \ + cargo build -p carbide-machine-a-tron --bin machine-a-tron --locked \ + --target x86_64-unknown-linux-gnu && \ + mkdir -p /artifacts && \ + cp /cargo-target/x86_64-unknown-linux-gnu/debug/machine-a-tron /artifacts/machine-a-tron + +FROM --platform=linux/amd64 debian:bookworm-slim + +RUN apt-get update && apt-get install -y --no-install-recommends \ + ca-certificates \ + iproute2 \ + iputils-ping \ + libssl3 \ + libudev1 \ + && rm -rf /var/lib/apt/lists/* + +RUN mkdir -p /opt/machine-a-tron/bin /opt/machine-a-tron/templates /tmp/machine-a-tron-data + +COPY --from=builder /artifacts/machine-a-tron /opt/machine-a-tron/bin/machine-a-tron +COPY crates/machine-a-tron/templates /opt/machine-a-tron/templates diff --git a/docs/development/machine-a-tron-deployment.md b/docs/development/machine-a-tron-deployment.md new file mode 100644 index 0000000000..eba185e6db --- /dev/null +++ b/docs/development/machine-a-tron-deployment.md @@ -0,0 +1,323 @@ +# machine-a-tron — Build & Deployment Guide + +machine-a-tron is a bare-metal simulator for NICo testing. It hosts mock DPUs and +servers via Redfish BMC, allowing end-to-end NICo flows without real hardware. This +guide documents everything needed to build the container image and deploy it on a +cluster. + +## Overview + +machine-a-tron runs in **Override Mode**: site-explorer redirects all Redfish traffic +to the mock BMC server running inside the pod. It is only suitable for +simulation-only clusters (no real hardware). + +## Quick path: `setup-machine-a-tron.sh` + +For a running NICo Core site, the fastest and most reliable way to deploy is the +end-to-end script, which performs every step in this guide (namespace, pull +secret, CA/Vault secret refresh, BMC credential seeding, `bmc_proxy` +configuration, DHCP-pool sizing, cert reissue, deploy, and verification) and is +idempotent: + +```bash +export KUBECONFIG=/path/to/kubeconfig +export REGISTRY_PULL_SECRET= # only if the pull secret is absent +helm-prereqs/setup-machine-a-tron.sh # add -y for non-interactive +``` + +The rest of this document explains what that script does and why, and is the +reference for manual deployment or debugging. The script's header comments +enumerate the non-obvious failure modes it guards against. + +## 1. Prerequisites + +- Docker with `buildx` and a `linux/arm64` builder available (see below) +- A container registry you can push to and the cluster can pull from + (referenced via `NICO_IMAGE_REGISTRY`, same convention as `setup.sh`) +- A cluster where machine-a-tron can reach `nico-api.nico-system.svc.cluster.local:1079` +- The `nico-machine-a-tron` Helm chart (`helm/charts/nico-machine-a-tron`) +- The image pull secret `machine-a-tron-pull` in the `nico-mat` namespace + (created automatically by the setup script from `REGISTRY_PULL_SECRET`) + +## 2. Building the Container Image + +### Why cross-compilation is required + +machine-a-tron must run on **x86_64** cluster nodes. The Rust dependency `aws-lc-sys` +contains hand-written x86_64 assembly (`s2n-bignum`). Compiling under QEMU emulation +causes a SIGSEGV in the assembler (`bignum_madd_n25519.S`). We therefore use true +cross-compilation: a native `linux/arm64` Rust compiler targeting +`x86_64-unknown-linux-gnu`. + +The `carbide-rpc` crate runs a protobuf build script that requires both `protoc` +and the protobuf well-known types (`libprotobuf-dev` on Debian). `libredfish` is a +Git dependency so `git` must also be present in the build stage. + +### Build command + +Run from the **repository root**: + +```bash +# Same convention as setup.sh: your registry/repository prefix, no scheme. +REGISTRY=${NICO_IMAGE_REGISTRY:?export NICO_IMAGE_REGISTRY=/} +COMMIT=$(git rev-parse --short HEAD) +TAG="${COMMIT}-amd64" + +docker buildx build \ + --platform linux/amd64 \ + -f crates/machine-a-tron/Dockerfile \ + --push \ + -t "${REGISTRY}/machine-a-tron:${TAG}" \ + . +``` + +> **Note**: `--load` does not work for cross-platform builds — use `--push` directly. +> The build takes ~4–5 minutes on a cold cache; subsequent builds with warm cache are +> under 30 seconds. + +### Registry authentication + +```bash +docker login "${NICO_IMAGE_REGISTRY%%/*}" \ + -u "${REGISTRY_PULL_USERNAME:-\$oauthtoken}" \ + -p "${REGISTRY_PULL_SECRET}" +``` + +For `nvcr.io` the username is the literal string `$oauthtoken`. + +## 3. Cluster Prerequisites + +### Namespaces and secrets + +```bash +# Create the deployment namespace +kubectl create namespace nico-mat --dry-run=client -o yaml | kubectl apply -f - + +# Image pull secret (replace with your NVIDIA API key) +kubectl create secret docker-registry machine-a-tron-pull \ + -n nico-mat \ + --docker-server="${NICO_IMAGE_REGISTRY%%/*}" \ + --docker-username="${REGISTRY_PULL_USERNAME:-\$oauthtoken}" \ + --docker-password="${REGISTRY_PULL_SECRET}" \ + --dry-run=client -o yaml | kubectl apply -f - + +# Copy nico-roots CA secret from nico-system +kubectl get secret nico-roots -n nico-system -o json \ + | jq 'del(.metadata.namespace,.metadata.resourceVersion,.metadata.uid,.metadata.creationTimestamp)' \ + | kubectl apply -n nico-mat -f - +``` + +### Vault secrets + +machine-a-tron's service account needs the Vault secrets present in its namespace +(even though they are optional — their absence produces confusing log noise): + +```bash +for secret in nico-vault-approle-tokens nico-vault-token vault-cluster-info; do + kubectl get secret "$secret" -n nico-system -o json \ + | jq 'del(.metadata.namespace,.metadata.resourceVersion,.metadata.uid,.metadata.creationTimestamp)' \ + | kubectl apply -n nico-mat -f - +done +``` + +> **Critical after a site reprovision:** re-copy `nico-roots` from `nico-system` +> as well. A reprovision recreates the nico-system CA; a machine-a-tron carried +> over from before will trust the *old* CA (stale `nico-roots`) and present a +> client cert signed by the old CA, so **every** mTLS call to nico-api fails +> with `client error (Connect)`. Re-copy the CA and delete the old cert secret +> (`nico-machine-a-tron-certificate`) so cert-manager reissues from the current +> CA: +> ```bash +> kubectl get secret nico-roots -n nico-system -o json \ +> | jq 'del(.metadata.namespace,.metadata.resourceVersion,.metadata.uid,.metadata.creationTimestamp,.metadata.ownerReferences)' \ +> | kubectl apply -n nico-mat -f - +> kubectl delete secret nico-machine-a-tron-certificate -n nico-mat --ignore-not-found +> ``` + +### BMC credentials in Vault (required for site-explorer) + +site-explorer's `check_preconditions` requires three site-default Vault +credentials before it will explore any endpoint: +`machines/bmc/site/root`, `machines/all_dpus/site_default/uefi-metadata-items/auth`, +and `machines/all_hosts/site_default/uefi-metadata-items/auth`. The two UEFI +paths are created by the nico-prereqs `kvSeeds` but with **empty passwords**, +which fails the check (`vault does not have a valid password entry`) — they +must be re-seeded with any non-empty password. `machines/bmc/site/root` is not +seeded at all; without it every run aborts with `MissingCredentials`. + +Beyond the preconditions, the **credential rotation flow** requires this exact +chain (all handled by `setup-machine-a-tron.sh` Phase 4): + +| Vault path | Value | Why | +|---|---|---| +| `machines/all_hosts/factory_default/bmc-metadata-items/dell` | `root`/`factory_password` | Host BMC factory default (mock's `DUMMY_FACTORY_PASSWORD`). Path segment is **lowercase** `dell` — `BMCVendor`'s `Display` impl lowercases. | +| `machines/all_dpus/factory_default/bmc-metadata-items/root` | `root`/`0penBmc` | DPU BMC factory default (mock's `DUMMY_FACTORY_DPU_PASSWORD`) — note it differs from the host factory password. | +| `machines/bmc/site/root` | `root`/<distinct> | Rotation target. **Must differ from both factory passwords**, or the rotation is a no-op and the mock rejects with `403 Factory-default password must be changed` forever. | + +site-explorer logs into each BMC with its factory default, rotates the password +to the site root value, then proceeds — using the wrong factory password (or a +site root equal to factory) yields `401 Unauthorized`, which latches a +self-perpetuating `AvoidLockout` (NICO-SITEEXPLORER-144) until an operator +clears it (`nico-admin-cli site-explorer refresh `). + +## 4. Deploying the Helm Chart + +### Site values file + +Copy `helm-prereqs/values/machine-a-tron.yaml` and fill in the site-specific values: + +| Field | Description | +|---|---| +| `image.tag` | Tag produced by step 2 (e.g. `8c35783af-amd64`) | +| `machines.dell-hosts.oobDhcpRelayAddress` | Gateway of the OOB/underlay network from nico-core site config | +| `machines.dell-hosts.adminDhcpRelayAddress` | Gateway of the admin network from nico-core site config | +| `machines.dell-hosts.hostCount` | Must not exceed available OOB DHCP addresses (`hostCount + hostCount×dpuPerHostCount`) | + +### Critical: SPIFFE URI override + +The cert-manager `Certificate` resource auto-generates a SPIFFE URI based on the +deployment namespace: `spiffe://nico.local/nico-mat/sa/nico-machine-a-tron`. + +nico-api's `spiffe_service_base_paths` only includes `/nico-system/sa/` (and two +others), so this URI is **not recognized**. The result is that machine-a-tron's +principal is only `TrustedCertificate` — not `SpiffeServiceIdentifier("machine-a-tron")` +— and every gRPC call beyond `Version` returns HTTP 403. + +The values file already includes the fix: + +```yaml +certificate: + uris: + - "spiffe://nico.local/nico-system/sa/machine-a-tron" +``` + +This overrides the auto-generated URI so nico-api can correctly identify and authorize +machine-a-tron as the `Machineatron` RBAC principal. + +### Deploy + +```bash +helm upgrade --install nico-machine-a-tron \ + helm/charts/nico-machine-a-tron \ + -n nico-mat \ + --create-namespace \ + -f helm-prereqs/values/machine-a-tron.yaml +``` + +## 5. Configuring Override Mode + +Configure nico-core's site-explorer to redirect all Redfish traffic to the mock. +Add this to the nico-core site config under `[site_explorer]`: + +```toml +[site_explorer] +bmc_proxy = "nico-machine-a-tron-bmc-mock.nico-mat.svc.cluster.local:1266" +``` + +> **Use the cross-namespace FQDN.** site-explorer runs inside nico-api in +> `nico-system`; a bare service name resolves against that namespace and fails +> ("connection refused" on every Redfish call) because the mock's Service lives +> in `nico-mat`. +> +> **This setting does not survive a nico-core `helm upgrade`** — the ConfigMap +> is chart-owned, so an upgrade silently reverts it. Re-run +> `setup-machine-a-tron.sh` (idempotent) after any nico-core upgrade. + +> **Field name matters.** The config field is `bmc_proxy` — a single +> `"host:port"` string (`crates/site-explorer/src/config.rs`). The older +> `override_target_ip` / `override_target_port` fields are **deprecated**, and +> `override_target_host` was never a valid field at all (earlier revisions of +> this guide were wrong — a value under that key is silently ignored). + +Setting `bmc_proxy` at launch also makes `allow_changing_bmc_proxy` default to +`true`. That is what allows the chart value `machineATron.configureBmcProxyHost` +to work: when set, machine-a-tron calls nico-api's `set_dynamic_config` to set +`bmc_proxy` at runtime, but that call is rejected with `PermissionDenied` unless +`allow_changing_bmc_proxy` is true. The two mechanisms are complementary — the +values file ships `configureBmcProxyHost: +"nico-machine-a-tron-bmc-mock.nico-mat.svc.cluster.local"` (FQDN, same +cross-namespace requirement), and the nico-core `bmc_proxy` setting both +enables that path and covers the case where the runtime call has not happened +yet. + +Apply via `helm upgrade` of the nico-core chart, or patch the configmap and +restart nico-api (site-explorer runs in-process in nico-api): + +```bash +kubectl rollout restart deployment/nico-api -n nico-system +``` + +### Why machines get created (expected_machines) + +site-explorer's `MachineCreator` refuses to create a managed host unless a +matching `expected_machines` row exists (by BMC MAC) — otherwise it logs +`Refusing to create managed host, expected machines entry not found`. machine-a-tron +auto-registers these when `machineATron.registerExpectedMachines: true` (the +default in the values file). DHCP discovery alone is **not** sufficient. + +## 6. Verifying Startup + +Check that machine-a-tron passes the initial API calls: + +```bash +kubectl logs -n nico-mat deployment/nico-machine-a-tron | grep -E "firmware|Got desired|Error:" +``` + +Expected: `Got desired firmware versions from the server: [...]` + +Check nico-api for denied requests (should be empty after the SPIFFE fix): + +```bash +kubectl logs -n nico-system deployment/nico-api | grep "Request denied.*machine-a-tron" +``` + +## 7. DHCP Address Space + +machine-a-tron allocates one OOB IP per BMC interface (1 per host + 1 per DPU). With +5 hosts and 2 DPUs each that is **15 IPs** (`5 + 5×2`). Ensure the OOB DHCP prefix in +nico-core is large enough. Note the usable count is smaller than the raw CIDR: a +`/28` (16 addresses) yields only ~10–13 usable after subtracting network, +broadcast, gateway, and any reserved addresses — on dev6 a `/28` yielded 10, so +`hostCount: 3 × 2 DPUs = 9` fit but `5 × 2 = 15` did not. Use at least a `/27` +for the default counts, or reduce `hostCount` / `dpuPerHostCount`. Symptom of +overflow: `No IP addresses left in prefix ...` and machines stuck in `BmcInit`. + +If the prefix is exhausted from previous runs, either: + +- Force-delete old machine records via the admin CLI, or +- Reprovision the site (which clears the database) and redeploy from scratch. + +> **Do NOT hand-delete rows** from `machine_interfaces`, `dhcp_entries`, or +> `machine_interface_addresses` to free leases. The `machine_dhcp_records` view +> inner-joins the singleton control row `machine_interfaces_deletion` (id=1); if +> that row is deleted (easy to do by accident when clearing related tables) the +> view returns zero rows and `DiscoverDhcp` fails for **every** BMC with +> `Database Error: no rows returned by a query that expected to return at least +> one row`. If you hit that, restore the row: +> ```sql +> INSERT INTO machine_interfaces_deletion (id) VALUES (1) ON CONFLICT (id) DO NOTHING; +> ``` + +## 8. Summary of Non-Obvious Fixes Discovered + +| Problem | Root cause | Fix | +|---|---|---| +| `exec format error` in pod | Image was built for `arm64`, nodes are `x86_64` | Cross-compile with `--platform linux/amd64` and `x86_64-unknown-linux-gnu` Rust target | +| `SIGSEGV` compiling `aws-lc-sys` | QEMU emulates the `.S` assembler, which crashes | True cross-compilation (native arm64 host → x86_64 target) instead of QEMU | +| `File not found: google/protobuf/timestamp.proto` | `libprotobuf-dev` absent in build image | Add `libprotobuf-dev` to `apt-get install` in builder stage | +| `git fetch ... (exit status: 127)` | `libredfish` is a git dependency, `git` not in slim image | Add `git` to builder stage | +| `--load` fails for cross-platform builds | Docker limitation | Use `--push` directly to registry | +| HTTP 403 on every gRPC call | machine-a-tron cert SPIFFE URI not in nico-api's `service_base_paths` | Set `certificate.uris: ["spiffe://nico.local/nico-system/sa/machine-a-tron"]` in values | +| `client error (Connect)` on every nico-api call after a reprovision | Stale `nico-roots` CA + client cert signed by the old CA | Re-copy `nico-roots` from nico-system; delete `nico-machine-a-tron-certificate` so cert-manager reissues from the current CA | +| site-explorer aborts with `MissingCredentials machines/bmc/site/root` | Site BMC root cred not in default `kvSeeds` | Seed `secrets/machines/bmc/site/root` = `root`/<non-factory password> in Vault | +| site-explorer aborts with `MissingCredentials .../uefi-metadata-items/auth` | kvSeeds create the UEFI creds with **empty** passwords, which fail validation | Re-seed both site_default UEFI creds with any non-empty password | +| Redfish redirect ignored; `endpoint_explorations=0` | Wrong config field (`override_target_host` is not real) | Use `bmc_proxy = "nico-machine-a-tron-bmc-mock.nico-mat.svc.cluster.local:1266"` under `[site_explorer]` | +| Redfish `connection refused` on every endpoint despite bmc_proxy set | Bare service name resolves against nico-system, not nico-mat | Use the cross-namespace FQDN in `bmc_proxy` | +| Host BMCs 401 while DPUs explore fine | Host and DPU factory passwords differ (`factory_password` vs `0penBmc`); host factory cred missing or wrong | Seed `machines/all_hosts/factory_default/bmc-metadata-items/dell` = `root`/`factory_password` (lowercase `dell`) | +| DPU explorations stuck at `403 Factory-default password must be changed` | Site root password equals the factory password → rotation is a no-op | Seed `machines/bmc/site/root` with a password distinct from both factory defaults | +| All endpoints latch `AvoidLockout` (NICO-SITEEXPLORER-144) after a cred fix | A previous Unauthorized is self-perpetuating in the exploration report | `nico-admin-cli site-explorer refresh ` per endpoint (or re-run setup-machine-a-tron.sh, which clears it) | +| `Refusing to create managed host`; machine-a-tron logs `PermissionDenied` on registration | nico-api build lacks the `Machineatron` → `AddExpectedMachine` RBAC grant | Rebuild nico-api with the grant (internal_rbac_rules.rs); the setup script also has a DB fallback | +| Machine creation fails `No IP addresses left in prefix ` | Admin pool too small: creation needs one host-PF IP per DPU | Fit `hostCount×dpuPerHostCount` ≤ usable admin-pool IPs (the script auto-fits) | +| `Refusing to create managed host, expected machines entry not found` | No `expected_machines` row for the discovered BMC MAC | Set `machineATron.registerExpectedMachines: true` (default) so machine-a-tron auto-registers them | +| `No IP addresses left in prefix ...`; machines stuck in `BmcInit` | OOB DHCP pool too small for host×DPU count | Sizing: `hostCount + hostCount×dpuPerHostCount` ≤ usable pool IPs; use ≥ /27 or reduce counts | +| `DiscoverDhcp`: `no rows ... expected to return at least one row` | `machine_interfaces_deletion` singleton (id=1) deleted; breaks `machine_dhcp_records` view | `INSERT INTO machine_interfaces_deletion (id) VALUES (1) ON CONFLICT DO NOTHING;` — never hand-delete lease rows | diff --git a/docs/development/machine-a-tron-scale-testing.md b/docs/development/machine-a-tron-scale-testing.md new file mode 100644 index 0000000000..48c0fa8d96 --- /dev/null +++ b/docs/development/machine-a-tron-scale-testing.md @@ -0,0 +1,168 @@ +# Scaling NICo with machine-a-tron: 100 → 1000 → 4500 simulated hosts + +> **Status: DRAFT — early feedback wanted.** Stage 1 (100 hosts × 2 DPUs = +> 300 BMCs → 264 machines) is complete on dev6. Stage 2 (1000 hosts) is +> running as this is written. Stage 3 targets 4500 hosts × 2 DPUs = 13,500 +> BMC endpoints, in support of scaling NICo to ~4500 nodes. +> +> **Rebased on #2764**: this work now sits on top of Alexander's ClusterIP +> migration, which removes nginx/MetalLB from the chart entirely (per-BMC +> ClusterIP Services + ServiceCIDR, multi-pod sharding via `pods..cidr`). +> That migration supersedes issues 10, 12 and 13 below (kept for the record — +> they document why the nginx/MetalLB path was abandoned) and independently +> confirms the direction of the proxy-direct pivot. The scripts' scale mode +> (`bmc_proxy` + client-injected Forwarded) remains valid with `bmcServices` +> disabled and is what all stage results below were measured with. +> +> Branch: `machine-a-tron-e2e-on-2764`. Everything below is reproducible +> with two commands: +> +> ```bash +> export KUBECONFIG=/path/to/site/kubeconfig +> helm-prereqs/cleanup-machine-a-tron.sh -y +> MAT_MODE=scale HOST_COUNT=1000 helm-prereqs/setup-machine-a-tron.sh -y +> ``` + +## What this work delivers + +1. **`helm-prereqs/setup-machine-a-tron.sh`** — one idempotent script that + takes a running NICo site from nothing to created machines: namespace, + pull secret, CA/Vault secret refresh, the full BMC/UEFI credential chain, + nico-core site-config changes, DHCP pool sizing with auto-fit, DB safety + checks, helm deploy, and a verification loop that actively shepherds the + ingestion pipeline (details below on why that is necessary). +2. **`helm-prereqs/cleanup-machine-a-tron.sh`** — the full inverse, so + from-scratch runs are reproducible (this caught several + "works-second-time-only" bugs). +3. **`MAT_MODE=scale`** — a scale profile + (`helm-prereqs/values/machine-a-tron-scale.yaml`) using a **proxy-direct** + transport architecture (see next section), simulated network segments + sized for 13.5k endpoints, and raised site-explorer throughput knobs. +4. A one-line RBAC fix in nico-api (`Machineatron` was missing the + `AddExpectedMachine` grant) plus chart fixes to the nginx/MetalLB mode. + +## The architecture decision: proxy-direct + +The chart offers an nginx/MetalLB mode for large scale: one LoadBalancer +Service per simulated BMC (cap 16,384), nginx terminating TLS and routing to +the mock. We started there and hit four independent failure modes at just 300 +endpoints (§ issues 10–13). The pivotal realization: + +**`site_explorer.bmc_proxy` alone already scales.** When it is set, the +Redfish client itself injects `Forwarded: host=` (RFC 7239, +`crates/redfish/src/libredfish/implementation.rs`), and the mock's shared +registry (`use_single_bmc_mock = true`) routes each request to the right +simulated BMC. One ClusterIP Service carries the whole fleet — no nginx, no +MetalLB pool, no per-BMC Services, no `externalTrafficPolicy: Local` +pitfalls. + +The nginx/MetalLB mode remains the right choice when simulated BMCs must +coexist with **real hardware** (each mock needs a real routable IP). For a +simulation-only cluster it only adds moving parts. The chart fixes we made to +that mode are kept for its real users. + +Result at 300 endpoints: exploration went from constant flapping +(Unreachable/ConnectionRefused under load) to rock-stable 300/300. + +## Complete issue log + +Every issue below was found live on dev6 and is fixed on the branch, encoded +in the scripts/charts with explanatory comments. + +### Baseline (override-mode) end-to-end + +| # | Issue | Root cause | Fix | +|---|---|---|---| +| 1 | Every nico-api call fails `client error (Connect)` after a site reprovision | machine-a-tron trusts the old CA (stale `nico-roots` copy) and presents a cert signed by it | Script refreshes `nico-roots` + Vault secrets from nico-system and deletes the client-cert secret so cert-manager reissues from the current CA | +| 2 | Redfish redirect silently ignored | Docs said `override_target_host` — never a valid field; the real field is `bmc_proxy = "host:port"`, and it must be the **cross-namespace FQDN** (site-explorer runs in nico-system; a bare service name doesn't resolve) | Script sets `bmc_proxy` correctly; docs fixed | +| 3 | site-explorer aborts every run: `MissingCredentials` | `machines/bmc/site/root` isn't in default kvSeeds; the seeded UEFI creds ship with **empty** passwords which fail validation | Script seeds the full chain | +| 4 | Host BMCs 401 while DPUs explore fine | Host and DPU mock factory passwords differ (`factory_password` vs `0penBmc`); the host factory Vault path vendor segment is **lowercase** (`…/dell` — `BMCVendor`'s `Display` lowercases; the earlier capital-`Dell` seed was read by nobody) | Script seeds both factory creds on the correct paths | +| 5 | machine-a-tron's expected-machine registration 403s (logged misleadingly as "likely already ingested") | `Machineatron` principal missing from the `AddExpectedMachine` RBAC grant — an oversight; it holds the sibling grants (`DiscoverDhcp`, `CreateNetworkSegment`, `GetExpectedSwitch`) | One-line fix in `internal_rbac_rules.rs`; script includes a DB fallback for nico-api builds without it | +| 6 | Endpoints permanently stuck `AvoidLockout` (NICO-SITEEXPLORER-144) on a fresh deploy | Per-MAC rotated creds (`machines/bmc//root`) survive cleanup; a fresh mock is at factory password but the per-MAC entry makes site-explorer present the old rotated one → 401 latch, self-perpetuating by design | Cleanup purges per-MAC creds; setup self-heals stale ones (only when the machine graph is truly empty — machines AND interfaces at 0) | +| 7 | `DiscoverDhcp` fails for every BMC: "no rows returned…" | The `machine_dhcp_records` VIEW inner-joins a singleton control row (`machine_interfaces_deletion` id=1); manual lease cleanup had deleted it | Script restores the singleton; documented: never hand-delete lease rows | +| 8 | Machines never created: admin pool exhausted | Real demand is OOB = `hosts×(1+dpus)` and admin = `hosts×(dpus+1)` (one host-PF IP per DPU **plus one per host at creation**); usable = `2^(32-mask) − reserve_first − 1` | Sizing check with auto-fit; `reserve_first` parsed from the live site config | + +### Scale mode (100 hosts × 2 DPUs and up) + +| # | Issue | Root cause | Fix | +|---|---|---|---| +| 9 | helm deploy aborts: hundreds of `connection reset by peer` | helm's default burst (100 concurrent API calls) overwhelms SOCKS/ssh tunnels when creating hundreds of Services | `--qps 15 --burst-limit 30` (env-overridable) | +| 10 | nginx bmc-proxy CrashLoopBackOff: `host not found in upstream` | Chart template pointed the upstream at the bare chart name, which is not a Service | Point at the `-bmc-mock` Service (chart fix) | +| 11 | Nothing listens on the mock port; probes kill the pod | `use_single_bmc_mock=false` makes each mock bind its **real BMC IP** on the pod netns (bare-metal mode). `true` is the shared-registry mode K8s needs | `useSingleBmcMock: true` | +| 12 | Every registry lookup 404s: `no router configured for host: 10.233.x.x` | nginx forwarded `host=$server_addr`, but kube-proxy DNATs the LB IP to the nginx **pod IP** before the connection arrives | `Forwarded "host=$host"` — the client-requested host is the LB IP end-to-end (chart fix) | +| 13 | LB IPs intermittently Unreachable in-cluster | Per-BMC Services use `externalTrafficPolicy: Local` and the chart's REQUIRED podAffinity stacked all proxies on the mat node | Required anti-affinity between proxy replicas (+ `maxUnavailable=1/maxSurge=0`; with replicas == nodes a surge pod deadlocks the rollout) — chart fix, kept for nginx-mode users | +| 14 | DHCP fails: `No network segment defined for relay addresses` | Config-driven segment creation is **bootstrap-once** — skipped entirely on multi-domain sites ("Multiple domains, skipping initial network creation") | Script clone-inserts the simulated segments from same-type templates; `allocation_strategy` forced to `dynamic` (templates may be `reserved`, which rejects all dynamic DHCP) | +| 15 | AvoidLockout storm on all DPU endpoints; preingestion pinned at exactly `hostCount` | The rotation dance is racy at scale: preingestion's initial BMC reset reboots the mock, which returns at the **factory** password while its per-MAC Vault entry says "rotated" | Pin mock passwords to the site root (`hostBmcPassword`/`dpuBmcPassword`) — site-explorer's documented fallback ("factory failed → sitewide root, no rotation") logs straight in; resets become harmless | +| 16 | Pipeline stalls at preingestion `initial`; manager idle | `waiting_for_explorer_refresh` (set when errors are cleared) gates endpoints out of preingestion and can linger after a healthy report lands (273/300 were parked) | Verification loop unparks endpoints whose reports are clean | +| 17 | Managed hosts identified but machines never created; cycles never finish | `explorations_per_run` was raised to 400 "for throughput" — but identification and creation only run **at the end of a completed explore cycle**, and 400 deep scans per cycle meant cycles stopped completing | Default lowered to 120: cycles complete in ~1–2 min and creation runs every cycle | +| 18 | `Resource pool lo-ip is empty` on the 3rd machine | Machine creation allocates one loopback IP per machine; pool **definitions are seed-once** ("Declaration has drifted since seed … not re-applying") so config widening is ignored; dev6 ships **3** lo-ip addresses | Script inserts free `resource_pool` rows directly for a simulated range (16k) when the pool is smaller than the machine target | + +### A note on the verification loop + +The script's final phase doesn't just poll — it actively shepherds: +re-clears `AvoidLockout`/`Unauthorized` latches (they are one-way by design; +on real hardware an operator runs `nico-admin-cli site-explorer refresh`) and +unparks healthy endpoints. On a simulation cluster with hundreds of +concurrent resets/explorations, transient races are guaranteed; the loop is +the "operator". Mocks have no lockout threshold, so this is safe here. + +## Where we are today + +| Stage | Scale | Result | +|---|---|---| +| Baseline | 1 host × 1 DPU (override mode) | ✅ end-to-end: machines created, full credential rotation exercised | +| Stage 1 | 100 hosts × 2 DPUs = 300 BMCs (proxy-direct) | ✅ 300/300 endpoints stable, machines created and advancing through `hostinit`/`dpuinit` | +| Stage 2 | 1000 hosts × 2 DPUs = 3000 BMCs | ✅ **END TO END OK — 3000/3000 machines** in a single unattended script run (~25 min total; creation ≈ 240 machines/min) | +| Stage 3 | 4500 hosts × 2 DPUs = 13,500 BMCs | ✅ **exploration sweep complete: 13,500/13,500 endpoints**; 10,353 machines at time of writing with the remainder completing autonomously; fleet actively progressing through machine-controller lifecycle states (`dpuinit`, `dpudiscoveringstate`) on dev-sized postgres (4 CPU) | + +Stage-3 observations worth reviewers' attention: + +- **The ingestion pipeline is fully autonomous once configured.** Client + connectivity to the cluster dropped twice for extended periods during + stage 3; ingestion continued unattended both times (e.g. +720 machines + through one outage, +4,000 through another). The shepherd loop's latch + clearing — critical in earlier iterations — was a no-op for the entire + stage-3 run thanks to pinned credentials. +- **Measured stage-3 rates on dev6 (3 nodes, dev-sized postgres):** DHCP + ~110 interfaces/min; exploration ~120–360 endpoints/cycle; creation + 40–240 machines per completed explore cycle, sawtoothing with cycle + phasing (identification rebuilds `explored_managed_hosts` each cycle). +- **Per-MAC Vault credential lifecycle needs batching at scale** (issue 19 + below): site-explorer stores one `machines/bmc//root` entry per BMC — + 13,500 entries; deleting them one API round-trip at a time takes hours, + batched server-side it takes seconds. +- `expected_machines` auto-registration worked at stage 3 (9,890+ registered + by machine-a-tron via the API), confirming the RBAC grant path. + +Additional issue found at stage 3: + +| # | Issue | Root cause | Fix | +|---|---|---|---| +| 19 | Stage-2→3 cleanup ran for over an hour "deleting credentials" | One `kubectl exec` per per-MAC Vault deletion × thousands of entries | Batch the deletion loop server-side on the vault pod — one exec total (both cleanup and setup self-heal) | + +## Open questions — feedback wanted + +1. **RBAC**: is granting `Machineatron` → `AddExpectedMachine` acceptable + (commit `9a9ba072a`)? Until a nico-api image with it is deployed, the + script registers expected machines via direct DB insert — okay as a + documented simulation-only fallback? +2. **Seed-once reconcile semantics**: networks, and resource-pool + definitions are all create-once; config changes on established sites are + silently ignored (or warn-only). The script works around this with direct + DB writes (segment clone-insert, pool row insertion). Should NICo support + declarative updates for these instead? +3. **AvoidLockout at scale**: one-way latches are right for real BMCs, but + simulation fleets guarantee latch storms during resets. Worth a + site-config escape hatch (e.g. `site_explorer.lockout_protection = false`) + instead of the script's DB-level clearing? +4. **Mock fidelity**: the mock returns to its configured password after a + BMC reset. Real BMCs persist a rotated password across resets. Should + bmc-mock persist rotated credentials so the rotation path can be exercised + at scale without pinning? +5. **lo-ip per machine**: is one loopback IP per machine the intended + allocation at 13.5k machines, and is there guidance for sizing this pool + in production site templates (dev templates ship 3)? +6. **Cycle economics**: identification/creation only run at the end of a + completed `explore_site` cycle, so `explorations_per_run` trades sweep + throughput against creation latency in a non-obvious way. Worth + documenting (or decoupling creation from the exploration cycle)? diff --git a/helm-prereqs/cleanup-machine-a-tron.sh b/helm-prereqs/cleanup-machine-a-tron.sh new file mode 100755 index 0000000000..e85f7f82e5 --- /dev/null +++ b/helm-prereqs/cleanup-machine-a-tron.sh @@ -0,0 +1,296 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# ============================================================================= +# cleanup-machine-a-tron.sh — tear down machine-a-tron for a from-scratch redeploy +# +# The inverse of setup-machine-a-tron.sh. By default it removes everything that +# script created EXCEPT the namespace and pull secret (so a re-run of setup does +# not need the registry API key again): +# * uninstalls the nico-machine-a-tron Helm release +# * deletes the client cert secret (so cert-manager reissues from the current CA) +# * resets the machine graph in the NICo DB (TRUNCATE ... CASCADE) while +# PRESERVING network config and the machine_interfaces_deletion singleton +# * reverts nico-core site_explorer.bmc_proxy (and any legacy +# override_target_* lines) and restarts nico-api +# * removes the Vault credential machines/bmc/site/root +# +# ONLY for simulation-only clusters. The DB reset truncates machines, +# machine_interfaces, explored_endpoints, explored_managed_hosts, and +# expected_machines with CASCADE — every machine on the cluster is assumed to +# be a machine-a-tron simulation. NEVER run this against a site with real +# hardware inventory. +# +# The machine_interfaces_deletion singleton (id=1) is preserved and its +# last_deletion watermark bumped — deleting it would break the +# machine_dhcp_records view and every subsequent DiscoverDhcp call. +# +# Required environment: +# KUBECONFIG Path to the target cluster kubeconfig. +# Optional environment: +# MAT_NAMESPACE Default: nico-mat +# NICO_SYSTEM_NS Default: nico-system +# POSTGRES_NS Default: postgres +# VAULT_NS Default: vault +# +# Usage: +# export KUBECONFIG=/path/to/kubeconfig +# ./cleanup-machine-a-tron.sh # prompt before each destructive step +# ./cleanup-machine-a-tron.sh -y # non-interactive +# ./cleanup-machine-a-tron.sh --delete-namespace # full teardown (also removes +# # pull secret + copied secrets) +# ./cleanup-machine-a-tron.sh --keep-db # leave the DB untouched +# ./cleanup-machine-a-tron.sh --keep-nico-core-config # leave bmc_proxy in place +# ./cleanup-machine-a-tron.sh --keep-vault-cred # leave BMC cred in Vault +# ============================================================================= + +set -euo pipefail + +MAT_NAMESPACE="${MAT_NAMESPACE:-nico-mat}" +NICO_SYSTEM_NS="${NICO_SYSTEM_NS:-nico-system}" +POSTGRES_NS="${POSTGRES_NS:-postgres}" +VAULT_NS="${VAULT_NS:-vault}" +RELEASE="nico-machine-a-tron" +NICO_DB="nico_system_nico" + +ASSUME_YES=false +DELETE_NAMESPACE=false +KEEP_DB=false +KEEP_NICO_CORE_CONFIG=false +KEEP_VAULT_CRED=false + +for arg in "$@"; do + case "$arg" in + -y|--yes) ASSUME_YES=true ;; + --delete-namespace) DELETE_NAMESPACE=true ;; + --keep-db) KEEP_DB=true ;; + --keep-nico-core-config) KEEP_NICO_CORE_CONFIG=true ;; + --keep-vault-cred) KEEP_VAULT_CRED=true ;; + -h|--help) grep '^#' "$0" | sed 's/^# \{0,1\}//' | head -70; exit 0 ;; + *) echo "Unknown argument: $arg" >&2; exit 2 ;; + esac +done + +# --- helpers (match setup-machine-a-tron.sh) -------------------------------- +_c() { printf '\033[%sm' "$1"; } +BOLD="$(_c 1)"; RED="$(_c 31)"; GREEN="$(_c 32)"; YEL="$(_c 33)"; BLU="$(_c 34)"; NC="$(_c 0)" +phase() { echo; echo "${BOLD}${BLU}== $* ==${NC}"; } +info() { echo " $*"; } +ok() { echo " ${GREEN}✓${NC} $*"; } +warn() { echo " ${YEL}!${NC} $*" >&2; } +die() { echo "${RED}ERROR:${NC} $*" >&2; exit 1; } +confirm() { $ASSUME_YES && return 0; read -r -p " $* [y/N] " a; [[ "$a" == "y" || "$a" == "Y" ]]; } + +CM_JSON="" +cleanup_tmp() { rm -f "$CM_JSON" 2>/dev/null || true; } +trap cleanup_tmp EXIT + +PG_PRIMARY="" +_pg_primary() { + [[ -n "$PG_PRIMARY" ]] && { echo "$PG_PRIMARY"; return; } + PG_PRIMARY="$(kubectl get pods -n "$POSTGRES_NS" -l application=spilo \ + -o jsonpath='{range .items[*]}{.metadata.name} {.metadata.labels.spilo-role}{"\n"}{end}' 2>/dev/null \ + | awk '$2=="master"{print $1}' | head -1)" + echo "$PG_PRIMARY" +} +psql_run() { # runs SQL, returns raw psql output (not -tAc, so NOTICEs show) + local pg; pg="$(_pg_primary)"; [[ -n "$pg" ]] || die "no Patroni primary in $POSTGRES_NS" + kubectl exec -n "$POSTGRES_NS" "$pg" -- su postgres -c "psql -d $NICO_DB -v ON_ERROR_STOP=1 -c \"$1\"" 2>&1 +} +psql_q() { + local pg; pg="$(_pg_primary)"; [[ -n "$pg" ]] || die "no Patroni primary in $POSTGRES_NS" + kubectl exec -n "$POSTGRES_NS" "$pg" -- su postgres -c "psql -d $NICO_DB -tAc \"$1\"" 2>/dev/null +} +_VAULT_TOKEN="" +vault_cmd() { + if [[ -z "$_VAULT_TOKEN" ]]; then + _VAULT_TOKEN="$(kubectl get secret nico-vault-token -n "$NICO_SYSTEM_NS" -o jsonpath='{.data.token}' | base64 -d)" + [[ -n "$_VAULT_TOKEN" ]] || die "could not read nico-vault-token from $NICO_SYSTEM_NS" + fi + kubectl exec -n "$VAULT_NS" vault-0 -c vault -- sh -c \ + "export VAULT_TOKEN='$_VAULT_TOKEN' VAULT_ADDR=https://127.0.0.1:8200 VAULT_SKIP_VERIFY=true; $1" 2>/dev/null +} + +# ============================================================================= +# Phase 0 — preflight +# ============================================================================= +phase "Phase 0 — preflight" +for t in kubectl helm; do command -v "$t" >/dev/null || die "$t not found in PATH"; done +kubectl cluster-info >/dev/null 2>&1 || die "cannot reach the cluster (check KUBECONFIG)" +ok "cluster reachable" +echo " This will tear down machine-a-tron in namespace ${MAT_NAMESPACE} and reset" +echo " the machine graph in DB ${NICO_DB}. Simulation clusters only." +confirm "Proceed with cleanup?" || die "aborted" + +# ============================================================================= +# Phase 1 — uninstall Helm release +# ============================================================================= +phase "Phase 1 — uninstall Helm release" +if helm status "$RELEASE" -n "$MAT_NAMESPACE" >/dev/null 2>&1; then + # retry — transient "http2: client connection lost" can abort the uninstall + _uninstalled=false + for _i in 1 2 3; do + if helm uninstall "$RELEASE" -n "$MAT_NAMESPACE" >/dev/null 2>&1; then _uninstalled=true; break; fi + helm status "$RELEASE" -n "$MAT_NAMESPACE" >/dev/null 2>&1 || { _uninstalled=true; break; } + warn "uninstall attempt ${_i} failed (transient?) — retrying" + sleep 5 + done + $_uninstalled && ok "release ${RELEASE} uninstalled" || die "could not uninstall ${RELEASE}" +else + ok "release ${RELEASE} not installed" +fi + +# ============================================================================= +# Phase 2 — namespace / cert secret +# ============================================================================= +phase "Phase 2 — namespace / cert secret" +if $DELETE_NAMESPACE; then + if kubectl get ns "$MAT_NAMESPACE" >/dev/null 2>&1; then + confirm "Delete the entire ${MAT_NAMESPACE} namespace (removes pull secret + copied secrets)?" \ + && { kubectl delete namespace "$MAT_NAMESPACE" --wait=false >/dev/null; ok "namespace ${MAT_NAMESPACE} deletion requested"; } \ + || warn "skipped namespace deletion" + else + ok "namespace ${MAT_NAMESPACE} already absent" + fi +else + kubectl delete secret "${RELEASE}-certificate" -n "$MAT_NAMESPACE" --ignore-not-found >/dev/null 2>&1 \ + && ok "client cert secret deleted (cert-manager reissues on next deploy)" + info "namespace + pull secret kept (use --delete-namespace for full teardown)" +fi + +# ============================================================================= +# Phase 3 — reset machine graph in the DB (preserve singleton + network config) +# ============================================================================= +phase "Phase 3 — reset machine graph in DB" +if $KEEP_DB; then + warn "--keep-db set; leaving DB untouched" +else + BEFORE="$(psql_q "SELECT + (SELECT count(*) FROM machines) || '/' || + (SELECT count(*) FROM machine_interfaces) || '/' || + (SELECT count(*) FROM expected_machines) || '/' || + (SELECT count(*) FROM explored_endpoints);")" + info "before (machines/interfaces/expected/endpoints): ${BEFORE:-?}" + if confirm "TRUNCATE machine graph (machines, machine_interfaces, explored_endpoints, explored_managed_hosts, expected_machines) CASCADE?"; then + OUT="$(psql_run "BEGIN; + TRUNCATE machines, machine_interfaces, explored_endpoints, explored_managed_hosts, expected_machines RESTART IDENTITY CASCADE; + UPDATE machine_interfaces_deletion SET last_deletion = now() WHERE id = 1; + INSERT INTO machine_interfaces_deletion (id) VALUES (1) ON CONFLICT (id) DO NOTHING; + COMMIT;")" || die "DB reset failed (rolled back): $OUT" + # surface which tables CASCADE touched + printf '%s\n' "$OUT" | grep -i "truncate cascades" | sed 's/^/ /' || true + AFTER="$(psql_q "SELECT + (SELECT count(*) FROM machines) || '/' || + (SELECT count(*) FROM machine_interfaces) || '/' || + (SELECT count(*) FROM expected_machines) || '/' || + (SELECT count(*) FROM explored_endpoints);")" + SINGLETON="$(psql_q "SELECT count(*) FROM machine_interfaces_deletion WHERE id=1;")" + ok "machine graph reset; after: ${AFTER} singleton_present: ${SINGLETON}" + [[ "$SINGLETON" == "1" ]] || die "singleton row missing after reset — investigate before redeploy" + else + warn "skipped DB reset" + fi +fi + +# ============================================================================= +# Phase 4 — revert nico-core site_explorer bmc_proxy +# ============================================================================= +phase "Phase 4 — revert nico-core site_explorer config" +if $KEEP_NICO_CORE_CONFIG; then + warn "--keep-nico-core-config set; leaving bmc_proxy in place" +else + CM_JSON="$(mktemp)" + if kubectl get cm nico-api-site-config-files -n "$NICO_SYSTEM_NS" -o json > "$CM_JSON" 2>/dev/null; then + if grep -qE "bmc_proxy|override_target_(host|ip|port)" "$CM_JSON"; then + python3 - "$CM_JSON" <<'PY' +import json, sys +path = sys.argv[1] +cm = json.load(open(path)) +drop = ("bmc_proxy", "override_target_host", "override_target_ip", "override_target_port") +for k, v in cm["data"].items(): + lines = [ln for ln in v.splitlines() if not any(t in ln for t in drop)] + cm["data"][k] = "\n".join(lines) + ("\n" if v.endswith("\n") else "") +for f in ("resourceVersion","uid","creationTimestamp","managedFields"): + cm["metadata"].pop(f, None) +json.dump(cm, open(path, "w")) +PY + kubectl apply -f "$CM_JSON" >/dev/null + info "removed bmc_proxy / override_target_* lines; restarting nico-api" + kubectl rollout restart deployment/nico-api -n "$NICO_SYSTEM_NS" >/dev/null + kubectl rollout status deployment/nico-api -n "$NICO_SYSTEM_NS" --timeout=180s >/dev/null \ + || warn "nico-api rollout slow; continuing" + ok "site_explorer config reverted" + else + ok "no bmc_proxy / override_target_* present" + fi + else + warn "nico-api-site-config-files configmap not found; skipping" + fi +fi + +# ============================================================================= +# Phase 5 — remove site BMC root Vault credential +# ============================================================================= +phase "Phase 5 — remove machine-a-tron Vault credentials" +# site root + the host factory cred that setup-machine-a-tron.sh seeds. +# (The DPU factory + UEFI creds belong to nico-prereqs kvSeeds — left alone.) +# NB: vendor segment is lowercase ("dell") — BMCVendor's Display impl +# lowercases; must match what setup-machine-a-tron.sh seeds. +MAT_VAULT_PATHS="machines/bmc/site/root machines/all_hosts/factory_default/bmc-metadata-items/dell" +if $KEEP_VAULT_CRED; then + warn "--keep-vault-cred set; leaving credentials in place" +elif confirm "Delete Vault credentials seeded/rotated for machine-a-tron (site root, host factory, per-MAC)?"; then + for p in $MAT_VAULT_PATHS; do + if vault_cmd "vault kv get secrets/$p" >/dev/null 2>&1; then + vault_cmd "vault kv metadata delete secrets/$p" >/dev/null 2>&1 \ + || vault_cmd "vault kv delete secrets/$p" >/dev/null 2>&1 || true + ok "removed $p" + else + ok "already absent: $p" + fi + done + # Per-MAC rotated creds (machines/bmc//root) — written by site-explorer + # when it rotates each BMC's password to the site root. MUST be purged: a + # fresh mock boots at the factory password, but a surviving per-MAC entry + # tells site-explorer the BMC was already rotated, so it presents the old + # rotated password, gets 401 Unauthorized, and permanently latches + # AvoidLockout (NICO-SITEEXPLORER-144) on the endpoint. + # Batch server-side in ONE kubectl exec: at scale there are thousands of + # per-MAC entries and one exec per deletion takes HOURS (one round-trip + + # exec setup each); a shell loop on the vault pod does them all in seconds. + _n="$(vault_cmd 'count=0 +for m in $(vault kv list -format=yaml secrets/machines/bmc 2>/dev/null | sed "s/^- //" | grep -v "^site/"); do + vault kv metadata delete "secrets/machines/bmc/${m%/}/root" >/dev/null 2>&1 && count=$((count+1)) +done +echo $count' || echo 0)" + ok "removed ${_n:-0} per-MAC rotated creds (batched server-side)" +else + warn "kept credentials" +fi + +# ============================================================================= +# Phase 6 — verify clean state +# ============================================================================= +phase "Phase 6 — verify" +helm status "$RELEASE" -n "$MAT_NAMESPACE" >/dev/null 2>&1 && warn "release still present" || ok "release gone" +if ! $KEEP_DB; then + M="$(psql_q "SELECT count(*) FROM machines;" || echo '?')" + I="$(psql_q "SELECT count(*) FROM machine_interfaces;" || echo '?')" + info "machines=${M} machine_interfaces=${I} (expect 0/0)" +fi +phase "Done" +info "Cluster reset for a from-scratch machine-a-tron deploy." +info "Next: ./setup-machine-a-tron.sh (set REGISTRY_PULL_SECRET if you used --delete-namespace)" diff --git a/helm-prereqs/setup-machine-a-tron.sh b/helm-prereqs/setup-machine-a-tron.sh new file mode 100755 index 0000000000..38c8a846b3 --- /dev/null +++ b/helm-prereqs/setup-machine-a-tron.sh @@ -0,0 +1,869 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# ============================================================================= +# setup-machine-a-tron.sh — deploy machine-a-tron end to end on a NICo site +# +# machine-a-tron is a bare-metal simulator: it hosts mock DPUs and servers via a +# Redfish BMC mock so NICo can run full ingestion flows without real hardware. +# This script sets up EVERYTHING needed for a running NICo Core site to discover +# and create the simulated machines, in setup.sh style (phased, idempotent). +# +# It assumes NICo Core (nico-api, postgres/nico-pg-cluster, Vault, ESO, +# cert-manager) is already deployed on the target cluster (i.e. setup.sh has run +# and the site is bootstrapped). It does NOT deploy NICo Core. +# +# --------------------------------------------------------------------------- +# WHY EACH STEP EXISTS — the non-obvious failure modes this script prevents +# (learned the hard way; do not remove without understanding them): +# +# * CA refresh (Phase 3): after a site reprovision the nico-system CA and +# nico-api are recreated. A machine-a-tron left over from before still trusts +# the OLD CA (stale nico-roots) and presents a client cert signed by the OLD +# CA, so mTLS to nico-api fails with "client error (Connect)" on every call. +# We always re-copy nico-roots from nico-system and (Phase 8) delete the old +# cert secret so cert-manager reissues from the CURRENT CA. +# +# * BMC site-root credential (Phase 5): site-explorer's check_preconditions +# requires the Vault credential machines/bmc/site/root. It is NOT in the +# default nico-prereqs kvSeeds, so without it site-explorer aborts every run +# with MissingCredentials and never explores anything. +# +# * bmc_proxy field name (Phase 6): the site_explorer config field is +# `bmc_proxy = "host:port"` (a single string). `override_target_host` is NOT +# a real field (older docs were wrong); override_target_ip/port are +# DEPRECATED. Setting bmc_proxy at launch also makes allow_changing_bmc_proxy +# default true, which is what lets machine-a-tron's configureBmcProxyHost +# runtime call succeed instead of being PermissionDenied. +# +# * expected_machines (chart value registerExpectedMachines: true): without a +# matching expected_machines row (by BMC MAC), MachineCreator refuses to +# create the managed host. machine-a-tron auto-registers them when the value +# is true — the script asserts it. +# +# * DHCP pool sizing (Phase 7): machine-a-tron needs one OOB IP per BMC — +# hostCount + hostCount*dpuPerHostCount. Overflowing the OOB pool yields +# "No IP addresses left in prefix ..." and machines never register. +# +# * machine_interfaces_deletion singleton (Phase 10 check): the +# machine_dhcp_records VIEW inner-joins the singleton row id=1. If it is ever +# deleted (e.g. by manual lease cleanup) the view returns zero rows and +# DiscoverDhcp fails with "no rows ... expected to return at least one row". +# The script restores the row if missing. NEVER delete it. To free stale +# leases, force-delete machine records via the admin CLI or reprovision — +# do NOT hand-delete interface/dhcp rows. +# +# --------------------------------------------------------------------------- +# Tool requirements: kubectl, helm, jq +# +# Required environment: +# KUBECONFIG Path to the target cluster kubeconfig (or current +# kubectl context already points at it). +# +# Optional environment: +# NICO_IMAGE_REGISTRY REQUIRED unless image.repository is set in the +# values file. Registry/repository prefix, without +# http(s):// (same convention as setup.sh). The +# machine-a-tron image is pulled from +# ${NICO_IMAGE_REGISTRY}/machine-a-tron. +# MAT_IMAGE_TAG REQUIRED unless image.tag is set in the values +# file. machine-a-tron image tag. +# REGISTRY_PULL_SECRET Registry password/API key. Only needed if the +# pull secret does not already exist in the +# machine-a-tron namespace. +# REGISTRY_PULL_USERNAME Username for the pull secret. Default: $oauthtoken +# (correct for nvcr.io). +# MAT_NAMESPACE Deployment namespace. Default: nico-mat +# NICO_SYSTEM_NS NICo Core namespace. Default: nico-system +# POSTGRES_NS Postgres namespace. Default: postgres +# VAULT_NS Vault namespace. Default: vault +# BMC_USERNAME Site BMC root username. Default: root +# BMC_PASSWORD Site BMC root password (rotation target). MUST +# differ from the mock factory defaults. +# Default: NicoSiteRoot1 +# OOB_DHCP_RELAY OOB/underlay gateway (BMC DHCP relay). Auto-detected +# from nico-core site config if unset. +# ADMIN_DHCP_RELAY Admin network gateway. Auto-detected if unset. +# HOST_COUNT Override machines.dell-hosts.hostCount. +# DPU_PER_HOST Override machines.dell-hosts.dpuPerHostCount. +# CHART_DIR Path to the nico-machine-a-tron chart. +# Default: /helm/charts/nico-machine-a-tron +# VALUES_FILE Base values template. +# Default: /values/machine-a-tron.yaml +# +# Usage: +# export KUBECONFIG=/path/to/kubeconfig +# ./setup-machine-a-tron.sh # prompt before deploy +# ./setup-machine-a-tron.sh -y # non-interactive +# ./setup-machine-a-tron.sh --skip-nico-core-config # don't touch nico-core +# ============================================================================= + +set -euo pipefail + +# --- config / defaults ------------------------------------------------------- +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "${SCRIPT_DIR}/.." && pwd)" + +MAT_NAMESPACE="${MAT_NAMESPACE:-nico-mat}" +NICO_SYSTEM_NS="${NICO_SYSTEM_NS:-nico-system}" +POSTGRES_NS="${POSTGRES_NS:-postgres}" +VAULT_NS="${VAULT_NS:-vault}" +BMC_USERNAME="${BMC_USERNAME:-root}" +# Site-wide BMC root password — the ROTATION TARGET. site-explorer logs into +# each mock BMC with its factory-default password, then rotates it to this +# value. It MUST DIFFER from both factory defaults below, or the rotation is a +# no-op and the mock keeps rejecting with "Factory-default password must be +# changed" (403) forever. +BMC_PASSWORD="${BMC_PASSWORD:-NicoSiteRoot1}" +# Factory defaults HARDCODED in the bmc-mock binary (crates/bmc-mock/src/lib.rs): +# host BMCs: DUMMY_FACTORY_PASSWORD = "factory_password" +# DPU BMCs: DUMMY_FACTORY_DPU_PASSWORD = "0penBmc" +# Do not change unless the mock changes. +FACTORY_HOST_BMC_PASSWORD="factory_password" +FACTORY_DPU_BMC_PASSWORD="0penBmc" +# Vendor path segment for the host factory cred. LOWERCASE is required: the +# credential path is built with format!("{vendor}") and BMCVendor's Display +# impl lowercases the variant name ("Dell Inc." → BMCVendor::Dell → "dell", +# crates/bmc-vendor/src/lib.rs impl Display). to_pascalcase() exists but is +# NOT used for Vault paths. +HOST_BMC_VENDOR="${HOST_BMC_VENDOR:-dell}" +# site-default UEFI passwords — check_preconditions requires them NON-EMPTY. +# The mock BMC does not validate them, so any non-empty value works. +UEFI_DPU_PASSWORD="${UEFI_DPU_PASSWORD:-bluefield}" +UEFI_HOST_PASSWORD="${UEFI_HOST_PASSWORD:-bluefield}" +REGISTRY_PULL_USERNAME="${REGISTRY_PULL_USERNAME:-\$oauthtoken}" +PULL_SECRET_NAME="${PULL_SECRET_NAME:-machine-a-tron-pull}" +RELEASE="nico-machine-a-tron" +BMC_MOCK_SVC="nico-machine-a-tron-bmc-mock" +BMC_MOCK_PORT="1266" +# site-explorer runs in nico-system, so it CANNOT resolve the bare service name +# (which resolves against its own namespace). bmc_proxy MUST use the +# cross-namespace FQDN of the bmc-mock service in the machine-a-tron namespace. +BMC_MOCK_FQDN="${BMC_MOCK_SVC}.${MAT_NAMESPACE}.svc.cluster.local" +NICO_DB="nico_system_nico" + +# --- deployment mode --------------------------------------------------------- +# override (default): all Redfish through site_explorer.bmc_proxy → one mock. +# scale: MetalLB/nginx mode — one LB Service per BMC, bmc_proxy UNSET, mock +# routes per-BMC via the Forwarded header. Uses values/machine-a-tron-scale.yaml +# plus two SIMULATED networks added to the nico-core site config (below) and +# raised site_explorer throughput knobs. See the chart README "METALLB MODE". +MAT_MODE="${MAT_MODE:-override}" +# Simulated networks for scale mode — must match the scale values file +# (relay addresses) and the MetalLB ipRange (inside the OOB prefix). +SCALE_OOB_PREFIX="10.100.0.0/17"; SCALE_OOB_GW="10.100.0.1" +SCALE_ADMIN_PREFIX="10.102.0.0/18"; SCALE_ADMIN_GW="10.102.0.1" +SCALE_RESERVE=1 +# site_explorer throughput knobs applied in scale mode (defaults 30/90/4 make +# 4500-host ingestion take ~9h; these bring it to ~1-2h). +SCALE_CONCURRENT_EXPLORATIONS="${SCALE_CONCURRENT_EXPLORATIONS:-100}" +# NB: keep explorations_per_run MODERATE. Identification and machine creation +# only run at the END of a completed explore_site cycle — a huge per-run value +# makes every cycle deep-scan hundreds of endpoints (dozens of Redfish calls +# each) and cycles stop completing, so machines are never created. ~120 keeps +# cycles under ~2 min while still sweeping the fleet quickly. +SCALE_EXPLORATIONS_PER_RUN="${SCALE_EXPLORATIONS_PER_RUN:-120}" +SCALE_MACHINES_CREATED_PER_RUN="${SCALE_MACHINES_CREATED_PER_RUN:-40}" + +CHART_DIR="${CHART_DIR:-${REPO_ROOT}/helm/charts/nico-machine-a-tron}" + +ASSUME_YES=false +SKIP_NICO_CORE_CONFIG=false +CM_JSON="" +MERGED_VALUES="" +cleanup() { rm -f "$CM_JSON" "$MERGED_VALUES" 2>/dev/null || true; } +trap cleanup EXIT + +for arg in "$@"; do + case "$arg" in + -y|--yes) ASSUME_YES=true ;; + --scale) MAT_MODE="scale" ;; + --skip-nico-core-config) SKIP_NICO_CORE_CONFIG=true ;; + -h|--help) grep '^#' "$0" | sed 's/^# \{0,1\}//' | head -110; exit 0 ;; + *) echo "Unknown argument: $arg" >&2; exit 2 ;; + esac +done + +if [[ "$MAT_MODE" == "scale" ]]; then + VALUES_FILE="${VALUES_FILE:-${SCRIPT_DIR}/values/machine-a-tron-scale.yaml}" +else + VALUES_FILE="${VALUES_FILE:-${SCRIPT_DIR}/values/machine-a-tron.yaml}" +fi + +# --- helpers ----------------------------------------------------------------- +_c() { printf '\033[%sm' "$1"; } +BOLD="$(_c 1)"; RED="$(_c 31)"; GREEN="$(_c 32)"; YEL="$(_c 33)"; BLU="$(_c 34)"; NC="$(_c 0)" +phase() { echo; echo "${BOLD}${BLU}== $* ==${NC}"; } +info() { echo " $*"; } +ok() { echo " ${GREEN}✓${NC} $*"; } +warn() { echo " ${YEL}!${NC} $*" >&2; } +die() { echo "${RED}ERROR:${NC} $*" >&2; exit 1; } +confirm() { + $ASSUME_YES && return 0 + read -r -p " $* [y/N] " ans + [[ "$ans" == "y" || "$ans" == "Y" ]] +} + +# psql against the consolidated NICo DB on the Patroni primary +PG_PRIMARY="" +_pg_primary() { + [[ -n "$PG_PRIMARY" ]] && { echo "$PG_PRIMARY"; return; } + PG_PRIMARY="$(kubectl get pods -n "$POSTGRES_NS" -l application=spilo \ + -o jsonpath='{range .items[*]}{.metadata.name} {.metadata.labels.spilo-role}{"\n"}{end}' 2>/dev/null \ + | awk '$2=="master"{print $1}' | head -1)" + echo "$PG_PRIMARY" +} +psql_q() { + local pg; pg="$(_pg_primary)" + [[ -n "$pg" ]] || die "no Patroni primary found in namespace $POSTGRES_NS" + kubectl exec -n "$POSTGRES_NS" "$pg" -- su postgres -c "psql -d $NICO_DB -v ON_ERROR_STOP=1 -tAc \"$1\"" 2>/dev/null +} +# count query that always yields a number — a transient kubectl/psql failure +# returns "0" instead of an empty string that would blow up (( )) arithmetic. +psql_count() { local r; r="$(psql_q "$1" || true)"; echo "${r:-0}"; } +# vault CLI on vault-0 using the root token stored in nico-system/nico-vault-token. +# Token is cached after the first read (it does not change within a run). +# env vars are exported (not inline-prefixed) so they apply across pipes, e.g. +# `echo ... | vault kv put ... -` — an inline prefix would bind them to echo only. +_VAULT_TOKEN="" +vault_cmd() { + if [[ -z "$_VAULT_TOKEN" ]]; then + _VAULT_TOKEN="$(kubectl get secret nico-vault-token -n "$NICO_SYSTEM_NS" -o jsonpath='{.data.token}' | base64 -d)" + [[ -n "$_VAULT_TOKEN" ]] || die "could not read nico-vault-token from $NICO_SYSTEM_NS" + fi + kubectl exec -n "$VAULT_NS" vault-0 -c vault -- sh -c \ + "export VAULT_TOKEN='$_VAULT_TOKEN' VAULT_ADDR=https://127.0.0.1:8200 VAULT_SKIP_VERIFY=true; $1" 2>/dev/null +} +# copy a secret from nico-system into the machine-a-tron namespace (strip metadata) +copy_secret() { + local name="$1" + kubectl get secret "$name" -n "$NICO_SYSTEM_NS" -o json 2>/dev/null \ + | jq 'del(.metadata.namespace,.metadata.resourceVersion,.metadata.uid,.metadata.creationTimestamp,.metadata.ownerReferences,.metadata.annotations,.metadata.managedFields)' \ + | kubectl apply -n "$MAT_NAMESPACE" -f - >/dev/null +} + +# ============================================================================= +# Phase 0 — preflight +# ============================================================================= +phase "Phase 0 — preflight" +for t in kubectl helm jq; do command -v "$t" >/dev/null || die "$t not found in PATH"; done +kubectl version -o json >/dev/null 2>&1 || kubectl cluster-info >/dev/null 2>&1 || die "cannot reach the cluster (check KUBECONFIG)" +ok "tools present, cluster reachable" +[[ -d "$CHART_DIR" ]] || die "chart dir not found: $CHART_DIR" +[[ -f "$VALUES_FILE" ]] || die "values file not found: $VALUES_FILE" +kubectl get deploy nico-api -n "$NICO_SYSTEM_NS" >/dev/null 2>&1 || die "nico-api not found in $NICO_SYSTEM_NS — deploy NICo Core (setup.sh) first" +[[ -n "$(_pg_primary)" ]] || die "no Postgres primary in $POSTGRES_NS" +kubectl get pod vault-0 -n "$VAULT_NS" >/dev/null 2>&1 || die "vault-0 not found in $VAULT_NS" +ok "NICo Core present: nico-api, postgres primary $(_pg_primary), vault-0" + +# portable extraction (macOS BSD sed/grep lack \s): [[:space:]] + awk on quotes +MAT_IMAGE_TAG="${MAT_IMAGE_TAG:-$(grep -E '^[[:space:]]*tag:' "$VALUES_FILE" | head -1 | awk -F'"' '{print $2}')}" +MAT_IMAGE_REPO="${MAT_IMAGE_REPO:-$(grep -E '^[[:space:]]*repository:' "$VALUES_FILE" | head -1 | awk -F'"' '{print $2}')}" +# Registry-agnostic (mirrors setup.sh): the image location comes from the +# environment, never from committed defaults. +if [[ -z "$MAT_IMAGE_REPO" ]]; then + [[ -n "${NICO_IMAGE_REGISTRY:-}" ]] || die "NICO_IMAGE_REGISTRY is unset and the values file has no image.repository (see setup.sh conventions)" + MAT_IMAGE_REPO="${NICO_IMAGE_REGISTRY}/machine-a-tron" +fi +HOST_COUNT="${HOST_COUNT:-$(grep -E '^[[:space:]]*hostCount:' "$VALUES_FILE" | head -1 | grep -oE '[0-9]+')}" +DPU_PER_HOST="${DPU_PER_HOST:-$(grep -E '^[[:space:]]*dpuPerHostCount:' "$VALUES_FILE" | head -1 | grep -oE '[0-9]+')}" +[[ -n "$MAT_IMAGE_TAG" ]] || die "MAT_IMAGE_TAG is unset and the values file has no image.tag" +[[ "$HOST_COUNT" =~ ^[0-9]+$ && "$DPU_PER_HOST" =~ ^[0-9]+$ ]] \ + || die "could not determine hostCount/dpuPerHostCount from $VALUES_FILE (set HOST_COUNT / DPU_PER_HOST)" +# Passwords are inlined into sh -c JSON heredocs on the vault pod; quotes, +# backslashes, or whitespace would break quoting or corrupt the JSON silently. +for _pw in "$BMC_PASSWORD" "$UEFI_DPU_PASSWORD" "$UEFI_HOST_PASSWORD"; do + case "$_pw" in + *[\'\"\\\ ]*) die "passwords must not contain quotes, backslashes, or spaces (BMC_PASSWORD / UEFI_*_PASSWORD)" ;; + esac +done +info "image: ${MAT_IMAGE_REPO}:${MAT_IMAGE_TAG} hosts: ${HOST_COUNT} dpus/host: ${DPU_PER_HOST}" + +# ============================================================================= +# Phase 1 — namespace +# ============================================================================= +phase "Phase 1 — namespace ${MAT_NAMESPACE}" +kubectl create namespace "$MAT_NAMESPACE" --dry-run=client -o yaml | kubectl apply -f - >/dev/null +# label so ESO / nico-roots sync treats it as a managed namespace +kubectl label namespace "$MAT_NAMESPACE" nico.nvidia.com/managed=true --overwrite >/dev/null +ok "namespace ready" + +# ============================================================================= +# Phase 2 — image pull secret +# ============================================================================= +phase "Phase 2 — image pull secret" +if kubectl get secret "$PULL_SECRET_NAME" -n "$MAT_NAMESPACE" >/dev/null 2>&1; then + ok "pull secret ${PULL_SECRET_NAME} already exists" +elif [[ -n "${REGISTRY_PULL_SECRET:-}" ]]; then + kubectl create secret docker-registry "$PULL_SECRET_NAME" -n "$MAT_NAMESPACE" \ + --docker-server="${MAT_IMAGE_REPO%%/*}" --docker-username="$REGISTRY_PULL_USERNAME" \ + --docker-password="$REGISTRY_PULL_SECRET" \ + --dry-run=client -o yaml | kubectl apply -f - >/dev/null + ok "pull secret ${PULL_SECRET_NAME} created" +else + warn "pull secret ${PULL_SECRET_NAME} missing and REGISTRY_PULL_SECRET unset" + warn " → set REGISTRY_PULL_SECRET, or ensure the image is already cached on nodes" +fi + +# ============================================================================= +# Phase 3 — refresh CA + Vault secrets from nico-system (GOTCHA: stale CA) +# ============================================================================= +phase "Phase 3 — refresh nico-roots CA + Vault secrets" +copy_secret nico-roots +ok "nico-roots synced from ${NICO_SYSTEM_NS} (current CA)" +for s in nico-vault-approle-tokens nico-vault-token; do + if kubectl get secret "$s" -n "$NICO_SYSTEM_NS" >/dev/null 2>&1; then + copy_secret "$s"; ok "$s synced" + else + warn "$s not present in ${NICO_SYSTEM_NS}; skipping (may produce log noise only)" + fi +done + +# ============================================================================= +# Phase 4 — seed site BMC root credential (GOTCHA: not in default kvSeeds) +# ============================================================================= +phase "Phase 4 — seed Vault BMC + UEFI credentials" +_kv_put() { # $1=path $2=username $3=password + vault_cmd "echo '{\"UsernamePassword\":{\"username\":\"$2\",\"password\":\"$3\"}}' | vault kv put secrets/$1 -" >/dev/null \ + || die "failed to write $1 to Vault" +} +_kv_password() { # $1=path → prints current password (empty if absent) + # `|| true` is load-bearing: when the path is absent, vault kv get fails and + # under `set -e` a failing $(...) assignment would kill the whole script. + vault_cmd "vault kv get -format=json secrets/$1" 2>/dev/null | jq -r '.data.data.UsernamePassword.password // empty' 2>/dev/null || true +} +# Factory-default creds — the INITIAL login site-explorer uses before rotating. +# Host and DPU factories DIFFER (see constants above); the wrong one yields 401 +# Unauthorized on every host BMC and a permanent AvoidLockout latch. +_kv_put "machines/all_hosts/factory_default/bmc-metadata-items/${HOST_BMC_VENDOR}" root "$FACTORY_HOST_BMC_PASSWORD" +ok "factory host cred: .../${HOST_BMC_VENDOR} = root/${FACTORY_HOST_BMC_PASSWORD}" +_kv_put "machines/all_dpus/factory_default/bmc-metadata-items/root" root "$FACTORY_DPU_BMC_PASSWORD" +ok "factory DPU cred: .../root = root/${FACTORY_DPU_BMC_PASSWORD}" +# Site-wide root — the rotation target; must differ from both factory passwords. +_cur="$(_kv_password machines/bmc/site/root)" +if [[ -z "$_cur" || "$_cur" == "$FACTORY_HOST_BMC_PASSWORD" || "$_cur" == "$FACTORY_DPU_BMC_PASSWORD" ]]; then + _kv_put "machines/bmc/site/root" "$BMC_USERNAME" "$BMC_PASSWORD" + ok "machines/bmc/site/root seeded (${BMC_USERNAME}/**** — distinct from factory)" +else + ok "machines/bmc/site/root already present with a non-factory password" +fi +# Self-heal stale per-MAC rotated creds (machines/bmc//root). site-explorer +# writes one per BMC after rotating its password to the site root; entries +# surviving from a previous deployment poison a fresh one — the fresh mock is at +# the factory password, but the per-MAC entry makes site-explorer present the +# old rotated password: 401 → permanent AvoidLockout. Only safe to purge when +# the machine graph is empty (live machines' per-MAC creds are real). +# "Fresh deployment" = no machines AND no interfaces. machines==0 alone is +# NOT enough: mid-ingestion (interfaces DHCP'd, endpoints explored, machines +# not yet created) the per-MAC creds are LIVE — purging them forces every +# endpoint back through the credential fallback. +if [[ "$(psql_count "SELECT count(*) FROM machines;")" == "0" \ + && "$(psql_count "SELECT count(*) FROM machine_interfaces;")" == "0" ]]; then + # Batched server-side in ONE kubectl exec — thousands of entries at scale; + # one exec per deletion takes hours. + _n="$(vault_cmd 'count=0 +for m in $(vault kv list -format=yaml secrets/machines/bmc 2>/dev/null | sed "s/^- //" | grep -v "^site/"); do + vault kv metadata delete "secrets/machines/bmc/${m%/}/root" >/dev/null 2>&1 && count=$((count+1)) +done +echo $count' || echo 0)" + if [[ "${_n:-0}" != "0" ]]; then + warn "purged ${_n} stale per-MAC BMC creds from a previous deployment (machine graph was empty)" + fi +fi +# site-explorer's check_preconditions ALSO requires the DPU + Host site_default +# UEFI creds to have a NON-EMPTY password. The nico-prereqs kvSeeds create these +# with empty passwords ("SITE SECRET: populate per site"), which fails the check +# with "vault does not have a valid password entry". Seed a non-empty value if +# the current password is empty/absent (the mock BMC does not validate it). +_seed_uefi() { # $1=vault path $2=password + local path="$1" pw="$2" cur + cur="$(_kv_password "$path")" + if [[ -n "$cur" ]]; then + ok "precondition cred present + valid: $path" + else + vault_cmd "echo '{\"UsernamePassword\":{\"username\":\"admin\",\"password\":\"${pw}\"}}' | vault kv put secrets/$path -" >/dev/null \ + || die "failed to seed UEFI cred $path" + ok "seeded UEFI cred (was empty): $path" + fi +} +_seed_uefi "machines/all_dpus/site_default/uefi-metadata-items/auth" "$UEFI_DPU_PASSWORD" +_seed_uefi "machines/all_hosts/site_default/uefi-metadata-items/auth" "$UEFI_HOST_PASSWORD" + +# ============================================================================= +# Phase 5 — configure nico-core site config for the selected mode +# override: set site_explorer.bmc_proxy (GOTCHA: field name; FQDN required) +# scale: REMOVE bmc_proxy, add simulated networks + throughput knobs +# ============================================================================= +phase "Phase 5 — nico-core site config (${MAT_MODE} mode)" +if $SKIP_NICO_CORE_CONFIG; then + warn "--skip-nico-core-config set; configure [site_explorer]/[networks] manually for ${MAT_MODE} mode" +elif [[ "$MAT_MODE" == "scale" ]]; then + CM_JSON="$(mktemp)" + kubectl get cm nico-api-site-config-files -n "$NICO_SYSTEM_NS" -o json > "$CM_JSON" 2>/dev/null \ + || die "nico-api-site-config-files configmap not found" + _PATCH_RESULT="$(SCALE_OOB_PREFIX="$SCALE_OOB_PREFIX" SCALE_OOB_GW="$SCALE_OOB_GW" \ + SCALE_ADMIN_PREFIX="$SCALE_ADMIN_PREFIX" SCALE_ADMIN_GW="$SCALE_ADMIN_GW" \ + SCALE_RESERVE="$SCALE_RESERVE" \ + BMC_PROXY="${BMC_MOCK_FQDN}:${BMC_MOCK_PORT}" \ + KNOB_CONC="$SCALE_CONCURRENT_EXPLORATIONS" KNOB_EPR="$SCALE_EXPLORATIONS_PER_RUN" \ + KNOB_MCPR="$SCALE_MACHINES_CREATED_PER_RUN" python3 - "$CM_JSON" <<'PY' +import json, os, sys +path = sys.argv[1] +cm = json.load(open(path)) +env = os.environ +# lines managed by this script inside [site_explorer] +drop = ("bmc_proxy", "override_target_host", "override_target_ip", "override_target_port", + "concurrent_explorations", "explorations_per_run", "machines_created_per_run") +knobs = [ + # PROXY-DIRECT: the Redfish client injects "Forwarded: host=" + # whenever bmc_proxy is set; the mock's registry routes on it. One + # ClusterIP service serves the whole simulated fleet. + f' bmc_proxy = "{env["BMC_PROXY"]}"', + f' concurrent_explorations = {env["KNOB_CONC"]}', + f' explorations_per_run = {env["KNOB_EPR"]}', + f' machines_created_per_run = {env["KNOB_MCPR"]}', +] +networks = f''' +# --- simulated networks for machine-a-tron scale testing (managed by +# --- setup-machine-a-tron.sh --scale; safe to leave in place) --- +[networks.simulated-oob] +type = "underlay" +prefix = "{env["SCALE_OOB_PREFIX"]}" +gateway = "{env["SCALE_OOB_GW"]}" +mtu = 9000 +reserve_first = {env["SCALE_RESERVE"]} + +[networks.simulated-admin] +type = "admin" +prefix = "{env["SCALE_ADMIN_PREFIX"]}" +gateway = "{env["SCALE_ADMIN_GW"]}" +mtu = 9000 +reserve_first = {env["SCALE_RESERVE"]} +''' +# Machine creation allocates one loopback IP per machine from pools.lo-ip — +# site templates ship tiny ranges (dev6: 3 addresses) that exhaust instantly +# ("Resource pool lo-ip is empty"). Pools DO reconcile at startup (unlike +# networks), so appending a simulated range takes effect on restart. +SIM_LO = ', { start = "10.103.0.1", end = "10.103.63.254" }]' +changed = False +for k, v in cm["data"].items(): + if "[site_explorer]" not in v: + continue + out, in_lo = [], False + for ln in v.splitlines(): + s = ln.strip() + if any(t in ln for t in drop): + continue + if s.startswith("[pools."): + in_lo = (s == "[pools.lo-ip]") + if in_lo and s.startswith("ranges") and "10.103.0.1" not in ln: + r = ln.rstrip(); idx = r.rfind("]") + ln = r[:idx] + SIM_LO + r[idx+1:] + out.append(ln) + if s == "[site_explorer]": + out.extend(knobs) + new = "\n".join(out) + ("\n" if v.endswith("\n") else "") + if "[networks.simulated-oob]" not in new: + new = new.rstrip("\n") + "\n" + networks + if new != v: + cm["data"][k] = new + changed = True +for f in ("resourceVersion","uid","creationTimestamp","managedFields"): + cm["metadata"].pop(f, None) +json.dump(cm, open(path, "w")) +print("changed" if changed else "nochange") +PY +)" + if [[ "$_PATCH_RESULT" == "changed" ]]; then + kubectl apply -f "$CM_JSON" >/dev/null + info "scale config applied (proxy-direct bmc_proxy, simulated networks, knobs, lo-ip); restarting nico-api" + kubectl rollout restart deployment/nico-api -n "$NICO_SYSTEM_NS" >/dev/null + kubectl rollout status deployment/nico-api -n "$NICO_SYSTEM_NS" --timeout=180s >/dev/null \ + || warn "nico-api rollout did not complete in time; continuing" + ok "scale networks: oob ${SCALE_OOB_PREFIX} (gw ${SCALE_OOB_GW}), admin ${SCALE_ADMIN_PREFIX} (gw ${SCALE_ADMIN_GW})" + ok "site_explorer knobs: concurrent=${SCALE_CONCURRENT_EXPLORATIONS} per_run=${SCALE_EXPLORATIONS_PER_RUN} create/run=${SCALE_MACHINES_CREATED_PER_RUN}" + else + ok "scale config already in place" + fi + + # --- ensure the simulated network SEGMENTS exist ------------------------- + # Config-driven segment creation (create_initial_networks) is bootstrap- + # once: it SKIPS entirely when the DB has multiple DNS domains ("we + # probably created the network much earlier", crates/api-core/src/db_init.rs). + # On an established site the new [networks.*] stanzas therefore never + # materialize and every mat DHCP fails with "No network segment defined + # for relay addresses". Fallback: clone an existing segment of the same + # type (identity fields overridden, vlan/vni cleared) + insert the prefix. + _ensure_segment() { # $1=name $2=type-ilike $3=prefix $4=gateway $5=reserve + local name="$1" typ="$2" pfx="$3" gw="$4" rsv="$5" + if [[ "$(psql_count "SELECT count(*) FROM network_segments WHERE name='${name}';")" != "0" ]]; then + ok "segment ${name} present" + return + fi + warn "segment ${name} missing (config seeding is bootstrap-once on multi-domain sites) — creating from template" + # allocation_strategy is forced to 'dynamic': templates may be + # 'reserved' (static-assignments segments), which rejects every mat + # DHCP with "configured for static DHCP leases only". + psql_q "INSERT INTO network_segments + SELECT (jsonb_populate_record(ns, jsonb_build_object( + 'id', gen_random_uuid()::text, 'name', '${name}', + 'allocation_strategy', 'dynamic', + 'vlan_id', NULL, 'vni_id', NULL))).* + FROM network_segments ns + WHERE ns.network_segment_type::text ILIKE '${typ}' LIMIT 1;" >/dev/null \ + || die "failed to create segment ${name} (no ${typ} template segment?)" + psql_q "INSERT INTO network_prefixes (segment_id, prefix, gateway, num_reserved) + SELECT id, '${pfx}'::cidr, '${gw}'::inet, ${rsv} + FROM network_segments WHERE name='${name}';" >/dev/null \ + || die "failed to add prefix ${pfx} to segment ${name}" + ok "segment ${name} created: ${pfx} (gw ${gw}, reserved ${rsv})" + } + _ensure_segment "simulated-oob" "underlay" "$SCALE_OOB_PREFIX" "$SCALE_OOB_GW" "$SCALE_RESERVE" + _ensure_segment "simulated-admin" "admin" "$SCALE_ADMIN_PREFIX" "$SCALE_ADMIN_GW" "$SCALE_RESERVE" + + # --- widen the lo-ip resource pool --------------------------------------- + # Machine creation allocates one loopback IP per machine from resource_pool + # rows. Pool DEFINITIONS are seed-once ("Declaration has drifted since + # seed ... not re-applying", crates/api-db/src/resource_pool.rs), so config + # changes to [pools.lo-ip] are IGNORED on established sites — rows must be + # inserted directly. Site templates ship tiny ranges (dev6: 3 addresses) + # that exhaust instantly ("Resource pool lo-ip is empty"). + _LO_FREE="$(psql_count "SELECT count(*) FROM resource_pool WHERE name='lo-ip' AND allocated IS NULL;")" + if (( _LO_FREE < HOST_COUNT * (1 + DPU_PER_HOST) )); then + info "widening lo-ip pool (${_LO_FREE} free < needed) with simulated range 10.103.0.1-10.103.63.254" + psql_q "INSERT INTO resource_pool (name, value, value_type, auto_assign, state, state_version, created) + SELECT 'lo-ip', host('10.103.0.0'::inet + g), 'ipv4', true, + '{\\\"state\\\":\\\"free\\\"}'::jsonb, + (SELECT state_version FROM resource_pool WHERE name='lo-ip' LIMIT 1), + now() + FROM generate_series(1, 16382) g + WHERE NOT EXISTS (SELECT 1 FROM resource_pool rp WHERE rp.name='lo-ip' AND rp.value = host('10.103.0.0'::inet + g));" >/dev/null \ + || die "failed to widen lo-ip pool" + ok "lo-ip pool: $(psql_count "SELECT count(*) FROM resource_pool WHERE name='lo-ip' AND allocated IS NULL;") free" + else + ok "lo-ip pool has ${_LO_FREE} free addresses" + fi +else + CM_JSON="$(mktemp)" + kubectl get cm nico-api-site-config-files -n "$NICO_SYSTEM_NS" -o json > "$CM_JSON" 2>/dev/null \ + || die "nico-api-site-config-files configmap not found" + if grep -q "bmc_proxy = \"${BMC_MOCK_FQDN}:${BMC_MOCK_PORT}\"" "$CM_JSON"; then + ok "bmc_proxy already configured" + else + _PATCH_RESULT="$(BMC_PROXY="${BMC_MOCK_FQDN}:${BMC_MOCK_PORT}" python3 - "$CM_JSON" <<'PY' +import json, os, sys +proxy = os.environ["BMC_PROXY"] +path = sys.argv[1] +cm = json.load(open(path)) +line = f' bmc_proxy = "{proxy}"' +drop = ("bmc_proxy", "override_target_host", "override_target_ip", "override_target_port") +changed = False +for k, v in cm["data"].items(): + if "[site_explorer]" not in v: + continue + out = [] + for ln in v.splitlines(): + if any(t in ln for t in drop): # strip any stale/legacy proxy lines + continue + out.append(ln) + if ln.strip() == "[site_explorer]": + out.append(line) # insert the correct FQDN bmc_proxy + cm["data"][k] = "\n".join(out) + ("\n" if v.endswith("\n") else "") + changed = True +for f in ("resourceVersion","uid","creationTimestamp","managedFields"): + cm["metadata"].pop(f, None) +json.dump(cm, open(path, "w")) +print("changed" if changed else "nochange") +PY +)" + if [[ "$_PATCH_RESULT" == "changed" ]]; then + kubectl apply -f "$CM_JSON" >/dev/null + info "configmap patched; restarting nico-api to load bmc_proxy" + kubectl rollout restart deployment/nico-api -n "$NICO_SYSTEM_NS" >/dev/null + kubectl rollout status deployment/nico-api -n "$NICO_SYSTEM_NS" --timeout=180s >/dev/null \ + || warn "nico-api rollout did not complete in time; continuing" + ok "bmc_proxy set to ${BMC_MOCK_FQDN}:${BMC_MOCK_PORT}" + else + ok "no [site_explorer] section found to patch — check the configmap manually" + fi + fi +fi + +# ============================================================================= +# Phase 6 — resolve DHCP relays + sizing check +# ============================================================================= +phase "Phase 6 — DHCP relays + pool sizing (${MAT_MODE})" +if [[ "$MAT_MODE" == "scale" ]]; then + # scale mode uses the SIMULATED networks added in Phase 5 — constants, + # no live-config parsing needed. + OOB_PREFIX="$SCALE_OOB_PREFIX"; OOB_DHCP_RELAY="${OOB_DHCP_RELAY:-$SCALE_OOB_GW}" + ADMIN_PREFIX="$SCALE_ADMIN_PREFIX"; ADMIN_DHCP_RELAY="${ADMIN_DHCP_RELAY:-$SCALE_ADMIN_GW}" + OOB_RESERVE="$SCALE_RESERVE"; ADMIN_RESERVE="$SCALE_RESERVE" +else + SITE_CFG="$(kubectl get cm nico-api-site-config-files -n "$NICO_SYSTEM_NS" \ + -o jsonpath='{.data.nico-api-site-config\.toml}' 2>/dev/null || true)" + # older deployments carry the config under the carbide-* key only + [[ -n "$SITE_CFG" ]] || SITE_CFG="$(kubectl get cm nico-api-site-config-files -n "$NICO_SYSTEM_NS" \ + -o jsonpath='{.data.carbide-api-site-config\.toml}' 2>/dev/null || true)" + # gateway lines appear as: gateway = "10.x.y.z" (admin then underlay in template order). + # Portable parse (no mapfile / negative index — macOS ships bash 3.2). + # Comment lines are stripped first: the config template carries commented + # examples (e.g. "# reserve_first = 5") that would otherwise be picked up. + SITE_CFG_CODE="$(printf '%s\n' "$SITE_CFG" | grep -vE '^[[:space:]]*#' || true)" + GW_LIST="$(printf '%s\n' "$SITE_CFG_CODE" | grep -oE 'gateway = "[0-9.]+"' | grep -oE '[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+' || true)" + PFX_LIST="$(printf '%s\n' "$SITE_CFG_CODE" | grep -oE 'prefix = "[0-9./]+"' | grep -oE '[0-9./]+' || true)" + RSV_LIST="$(printf '%s\n' "$SITE_CFG_CODE" | grep -oE 'reserve_first = [0-9]+' | grep -oE '[0-9]+' || true)" + # admin segment is the FIRST prefix/gateway pair, OOB/underlay the LAST. + ADMIN_PREFIX="$(printf '%s\n' "$PFX_LIST" | head -1)" + OOB_PREFIX="$(printf '%s\n' "$PFX_LIST" | tail -1)" + OOB_DHCP_RELAY="${OOB_DHCP_RELAY:-$(printf '%s\n' "$GW_LIST" | tail -1)}" + ADMIN_DHCP_RELAY="${ADMIN_DHCP_RELAY:-$(printf '%s\n' "$GW_LIST" | head -1)}" + # Usable IPs per pool = 2^(32-mask) − reserve_first − 1 (broadcast). + # reserve_first covers the network address, gateway, and operator-reserved + # leading addresses — verified live: /28 with reserve_first=5 → 10 usable. + ADMIN_RESERVE="$(printf '%s\n' "$RSV_LIST" | head -1)"; ADMIN_RESERVE="${ADMIN_RESERVE:-5}" + OOB_RESERVE="$(printf '%s\n' "$RSV_LIST" | tail -1)"; OOB_RESERVE="${OOB_RESERVE:-5}" +fi +[[ -n "$OOB_DHCP_RELAY" && -n "$ADMIN_DHCP_RELAY" ]] \ + || die "could not resolve DHCP relays; set OOB_DHCP_RELAY and ADMIN_DHCP_RELAY" +ok "OOB: relay ${OOB_DHCP_RELAY} prefix ${OOB_PREFIX:-unknown}" +ok "admin: relay ${ADMIN_DHCP_RELAY} prefix ${ADMIN_PREFIX:-unknown}" +_usable() { local m="${1##*/}" r="$2"; local u=$(( (1 << (32 - m)) - r - 1 )); (( u < 0 )) && u=0; echo "$u"; } +# Demand per pool (measured live): +# OOB = hostCount*(1 + dpuPerHost) — one BMC IP per host and per DPU +# admin = hostCount*(dpuPerHost + 1) — one host-PF IP per DPU at DHCP time, +# PLUS one admin IP per host allocated by machine creation (creation +# fails with "No IP addresses left in prefix " without it) +if [[ "${OOB_PREFIX:-}" == */* && "${ADMIN_PREFIX:-}" == */* ]]; then + OOB_USABLE="$(_usable "$OOB_PREFIX" "$OOB_RESERVE")"; ADMIN_USABLE="$(_usable "$ADMIN_PREFIX" "$ADMIN_RESERVE")" + # max hosts each pool supports, then take the min + FIT_OOB=$(( OOB_USABLE / (1 + DPU_PER_HOST) )) + FIT_ADMIN=$(( ADMIN_USABLE / (DPU_PER_HOST + 1) )) + FIT=$(( FIT_OOB < FIT_ADMIN ? FIT_OOB : FIT_ADMIN )) + info "pool fit: OOB ${OOB_PREFIX} ≈${OOB_USABLE} usable → ≤${FIT_OOB} hosts; admin ${ADMIN_PREFIX} ≈${ADMIN_USABLE} usable → ≤${FIT_ADMIN} hosts" + if (( HOST_COUNT > FIT )); then + (( FIT < 1 )) && die "pools too small for even 1 host × ${DPU_PER_HOST} DPUs — widen the admin/OOB prefixes or lower DPU_PER_HOST" + warn "requested ${HOST_COUNT} hosts exceeds pool capacity (${FIT}) — auto-fitting hostCount=${FIT}" + warn " (override with HOST_COUNT/DPU_PER_HOST env vars, or widen the site's DHCP prefixes)" + confirm "Proceed with hostCount=${FIT} × ${DPU_PER_HOST} DPUs?" || die "aborted on sizing" + HOST_COUNT="$FIT" + fi + NEED=$(( HOST_COUNT + HOST_COUNT * DPU_PER_HOST )) + ok "sizing: ${HOST_COUNT} hosts × ${DPU_PER_HOST} DPUs → ${NEED} OOB + $(( HOST_COUNT * (DPU_PER_HOST + 1) )) admin IPs" +else + NEED=$(( HOST_COUNT + HOST_COUNT * DPU_PER_HOST )) + warn "could not parse both pool prefixes — skipping sizing check (need ${NEED} OOB IPs)" +fi + +# ============================================================================= +# Phase 7 — DB safety: restore machine_interfaces_deletion singleton (GOTCHA) +# ============================================================================= +phase "Phase 7 — DB safety checks" +# "!= 1" (not "== 0") so a transient query failure (empty result) also takes +# the restore path — the INSERT is idempotent (ON CONFLICT DO NOTHING). +SINGLETON="$(psql_count "SELECT count(*) FROM machine_interfaces_deletion WHERE id=1;")" +if [[ "$SINGLETON" != "1" ]]; then + warn "machine_interfaces_deletion singleton (id=1) missing — restoring" + warn " (its absence breaks the machine_dhcp_records view → DiscoverDhcp 'no rows' errors)" + psql_q "INSERT INTO machine_interfaces_deletion (id) VALUES (1) ON CONFLICT (id) DO NOTHING;" >/dev/null + ok "singleton restored" +else + ok "machine_interfaces_deletion singleton present" +fi +ORPHANS="$(psql_count "SELECT count(*) FROM machine_interfaces mi WHERE NOT EXISTS (SELECT 1 FROM machines m WHERE m.id = mi.machine_id);")" +MACHINES_NOW="$(psql_count "SELECT count(*) FROM machines;")" +if [[ "${ORPHANS:-0}" -gt 0 && "${MACHINES_NOW:-0}" == "0" ]]; then + warn "${ORPHANS} orphaned machine_interfaces (no parent machine) may hold OOB leases" + warn " → if DHCP later reports exhaustion, force-delete stale records via the admin CLI" + warn " or reprovision. Do NOT hand-delete interface/dhcp rows (breaks the singleton)." +fi + +# ============================================================================= +# Phase 8 — reissue client cert from current CA (GOTCHA: stale cert) +# ============================================================================= +phase "Phase 8 — reissue machine-a-tron client cert" +# Always delete: a cert issued under a previous CA fails mTLS to nico-api with +# "client error (Connect)" on every call. cert-manager reissues from the +# current CA within seconds of the deploy — there is no reason to keep it. +kubectl delete secret "${RELEASE}-certificate" -n "$MAT_NAMESPACE" --ignore-not-found >/dev/null 2>&1 +ok "client cert cleared; cert-manager reissues from the current CA on deploy" + +# ============================================================================= +# Phase 9 — deploy the chart +# ============================================================================= +phase "Phase 9 — helm upgrade --install ${RELEASE}" +MERGED_VALUES="$(mktemp)" +# Site-specific overrides ONLY (never committed). Passed as a second -f so Helm +# deep-merges it over the base template (last -f wins per key) — avoids +# unreliable duplicate top-level keys within a single YAML file. +cat > "$MERGED_VALUES" <> "$MERGED_VALUES" </dev/null 2>&1 \ + && ok "client certificate Ready" || warn "certificate not Ready yet — check cert-manager" + +# wait windows scale with the deployment size (scale mode: hundreds-thousands) +IFACE_WAIT=$(( 90 + NEED )); (( IFACE_WAIT > 1800 )) && IFACE_WAIT=1800 +info "giving machine-a-tron time to register + DHCP (up to ${IFACE_WAIT}s)..." +_end=$((SECONDS+IFACE_WAIT)) +IFACES=0 +while (( SECONDS < _end )); do + IFACES="$(psql_count "SELECT count(*) FROM machine_interfaces;")" + (( IFACES >= NEED )) && break + sleep 10 +done +IPS="$(psql_count "SELECT count(*) FROM machine_interface_addresses;")" +info "machine_interfaces=${IFACES} (need ${NEED}) ips_allocated=${IPS}" +(( IFACES >= NEED )) && ok "BMC interfaces registered + DHCP allocated" \ + || warn "fewer interfaces than expected — check pool sizing / bmc DHCP" + +# --- expected_machines: required for machine creation (matched by BMC MAC) --- +# machine-a-tron auto-registers them (registerExpectedMachines: true), but on +# nico-api builds without the Machineatron→AddExpectedMachine RBAC grant +# (crates/api-core/src/auth/internal_rbac_rules.rs) the call is 403'd. Fall +# back to direct DB registration mirroring what the API call would create. +info "waiting for expected_machines (auto-registration)..." +_end=$((SECONDS+45)); EXPECTED=0 +while (( SECONDS < _end )); do + EXPECTED="$(psql_count "SELECT count(*) FROM expected_machines;")" + (( EXPECTED > 0 )) && break + sleep 5 +done +if (( EXPECTED > 0 )); then + ok "expected_machines=${EXPECTED} (registerExpectedMachines worked — RBAC grant present)" +else + warn "no expected_machines — this nico-api build lacks the Machineatron" + warn " AddExpectedMachine RBAC grant (403). Falling back to direct DB registration." + # scope strictly to BMC interfaces in the OOB prefix — with an unparsed + # prefix the filter would match admin-segment interfaces too and register + # them with the wrong factory password. + [[ "${OOB_PREFIX:-}" == */* ]] || die "cannot scope the expected_machines fallback: OOB prefix unknown" + psql_q "INSERT INTO expected_machines (id, serial_number, bmc_mac_address, bmc_username, bmc_password) + SELECT gen_random_uuid(), 'MAT-' || replace(mi.mac_address::text, ':', ''), mi.mac_address, 'root', '${FACTORY_HOST_BMC_PASSWORD}' + FROM machine_interfaces mi + JOIN machine_interface_addresses mia ON mia.interface_id = mi.id + WHERE mia.address << '${OOB_PREFIX}'::inet + AND NOT EXISTS (SELECT 1 FROM expected_machines em WHERE em.bmc_mac_address = mi.mac_address);" >/dev/null \ + || die "expected_machines DB fallback INSERT failed" + EXPECTED="$(psql_count "SELECT count(*) FROM expected_machines;")" + (( EXPECTED > 0 )) || die "expected_machines still 0 after fallback — machine creation cannot proceed" + ok "expected_machines=${EXPECTED} registered via DB fallback" +fi + +# --- kick exploration: clear any AvoidLockout latched before creds existed --- +# Exploration cycles that ran before Phase 4/5 completed record Unauthorized, +# which latches a self-perpetuating AvoidLockout in the exploration report. +# This mirrors the API's clear_last_known_error + request_exploration pair +# (crates/api-db/src/explored_endpoints.rs) — including the +# waiting_for_explorer_refresh flag, which gates preingestion until a fresh +# probe lands (skipping it would let preingestion act on the stale report). +psql_q "UPDATE explored_endpoints + SET exploration_report = jsonb_set(exploration_report, '{LastExplorationError}', 'null'::jsonb), + exploration_requested = true, + waiting_for_explorer_refresh = true;" >/dev/null || true +ok "cleared exploration lockouts + requested re-exploration" + +# machine target = one row per host + per DPU; wait scales with host count +MACHINE_TARGET=$(( HOST_COUNT * (1 + DPU_PER_HOST) )) +MACHINE_WAIT=$(( 420 + HOST_COUNT * 3 )); (( MACHINE_WAIT > 5400 )) && MACHINE_WAIT=5400 +info "waiting for explore → rotate → preingest → identify → create (target ${MACHINE_TARGET} machines, up to ${MACHINE_WAIT}s)..." +_end=$((SECONDS+MACHINE_WAIT)); MACHINES=0 +while (( SECONDS < _end )); do + MACHINES="$(psql_count "SELECT count(*) FROM machines;")" + (( MACHINES >= MACHINE_TARGET )) && break + # single round-trip for the progress line + _prog="$(psql_q "SELECT + (SELECT count(*) FILTER (WHERE exploration_report->'LastExplorationError' = 'null'::jsonb) FROM explored_endpoints) || '/' || + (SELECT count(*) FROM explored_managed_hosts);" || echo '?/?')" + info " endpoints_ok=${_prog%%/*}/${IFACES} managed_hosts=${_prog##*/} machines=${MACHINES} ..." + # Re-clear any AvoidLockout that latched during the wait (e.g. an + # exploration racing a mock reboot from preingestion's initial BMC reset). + # Idempotent; scoped to latched endpoints only so successful reports keep + # their state. + psql_q "UPDATE explored_endpoints + SET exploration_report = jsonb_set(exploration_report, '{LastExplorationError}', 'null'::jsonb), + exploration_requested = true, + waiting_for_explorer_refresh = true + WHERE exploration_report->'LastExplorationError'->>'Type' IN ('AvoidLockout','Unauthorized');" >/dev/null || true + # ...and UNPARK endpoints that have since explored clean: a lingering + # waiting_for_explorer_refresh gates them out of preingestion + # (find_preingest_not_waiting) even after a healthy report lands, stalling + # the pipeline at 'initial' indefinitely. + psql_q "UPDATE explored_endpoints + SET waiting_for_explorer_refresh = false, exploration_requested = false + WHERE waiting_for_explorer_refresh + AND exploration_report->'LastExplorationError' = 'null'::jsonb;" >/dev/null || true + sleep 25 +done +ENDPOINTS="$(psql_count "SELECT count(*) FROM explored_endpoints;")" +MHOSTS="$(psql_count "SELECT count(*) FROM explored_managed_hosts;")" +echo +if (( MACHINES >= MACHINE_TARGET )); then + ok "${GREEN}END TO END OK${NC} — endpoints=${ENDPOINTS}, managed_hosts=${MHOSTS}, machines=${MACHINES}/${MACHINE_TARGET}" +elif (( MACHINES > 0 )); then + ok "${GREEN}MACHINES CREATED${NC} (partial) — ${MACHINES}/${MACHINE_TARGET}; ingestion continuing in the background" +else + warn "machines not created yet (endpoints=${ENDPOINTS}, managed_hosts=${MHOSTS})" + warn " check: kubectl logs -n ${NICO_SYSTEM_NS} deploy/nico-api | grep -i 'site.explor\\|MissingCred\\|Refusing\\|Failed to create'" + warn " check: kubectl logs -n ${MAT_NAMESPACE} deploy/${RELEASE} | grep -iE 'No IP addresses|error'" + warn " a common cause: admin/OOB pool exhaustion — see the sizing output of Phase 6" +fi + +phase "Done" +info "machine-a-tron release ${RELEASE} deployed to ${MAT_NAMESPACE}." +info "Redeploy/iterate: re-run this script (idempotent) after 'export KUBECONFIG=...'." diff --git a/helm-prereqs/values/machine-a-tron-scale.yaml b/helm-prereqs/values/machine-a-tron-scale.yaml new file mode 100644 index 0000000000..856097927c --- /dev/null +++ b/helm-prereqs/values/machine-a-tron-scale.yaml @@ -0,0 +1,98 @@ +# ============================================================================= +# machine-a-tron-scale.yaml — LARGE-SCALE site values (MetalLB/nginx BMC mode) +# +# For scale testing NICo ingestion with hundreds to thousands of simulated +# hosts. Used by: MAT_MODE=scale setup-machine-a-tron.sh +# See docs/development/machine-a-tron-deployment.md and the chart README's +# "METALLB MODE" section. +# +# How this differs from Override Mode (values/machine-a-tron.yaml): +# - nginxBmcProxy.enabled: true — every simulated BMC gets its own +# LoadBalancer IP (one Service per BMC, chart cap 16384). nginx routes to +# the right mock via "Forwarded: host=". +# - machineATron.useSingleBmcMock: false — the mock registers one router per +# BMC keyed by its DHCP'd IP instead of one combined server. +# - site_explorer.bmc_proxy MUST BE UNSET in nico-core — site-explorer dials +# each BMC's dedicated IP directly (setup script removes it in scale mode). +# - Requires the simulated-oob / simulated-admin networks in the nico-core +# site config (added by the setup script's scale mode): +# [networks.simulated-oob] prefix 10.100.0.0/17 gateway 10.100.0.1 +# [networks.simulated-admin] prefix 10.102.0.0/18 gateway 10.102.0.1 +# The MetalLB ipRange below MUST sit inside the simulated-oob prefix so +# NICo-allocated BMC IPs match the MetalLB Service IPs (allocation is +# sequential on both sides). +# ============================================================================= + +## Registry-agnostic (like the NICo REST values): setup-machine-a-tron.sh +## injects image.repository = ${NICO_IMAGE_REGISTRY}/machine-a-tron and +## image.tag = ${MAT_IMAGE_TAG} at install time. Set them here only for +## direct helm installs without the script. +image: + repository: "" + tag: "" + pullPolicy: IfNotPresent + +global: + imagePullSecrets: + - name: machine-a-tron-pull + +# Same SPIFFE identity requirement as Override Mode (see machine-a-tron.yaml). +certificate: + uris: + - "spiffe://nico.local/nico-system/sa/machine-a-tron" + +machineATron: + nicoApiUrl: "https://nico-api.nico-system.svc.cluster.local:1079" + # Scale mode: no bmc_proxy anywhere — leave unset. + configureBmcProxyHost: "" + registerExpectedMachines: true + # true = SHARED REGISTRY mode (crates/machine-a-tron/src/main.rs): one + # server on bmc_mock_port routes to per-BMC mock routers via the + # "Forwarded: host=" header nginx injects. This is REQUIRED in K8s. + # false would make each mock bind its real BMC IP on the pod netns (bare- + # metal style) — nothing listens on 1266 and the pod crash-loops on probes. + useSingleBmcMock: true + +# Larger resources: one tokio task per host + per DPU; at thousands of +# machines the FSM/API chatter is CPU-bound. +resources: + limits: + cpu: "8" + memory: 8Gi + requests: + cpu: "2" + memory: 2Gi + +pods: + default: + machines: + # disable the chart-default example group (deep-merge would keep it) + rack-machines: null + dell-hosts: + hwType: dell_poweredge_r750 + # STAGE 1 default: 100 hosts × 2 DPUs = 300 BMC endpoints / LB Services. + # Raise per stage (1000 → 4500). Chart caps total BMCs at 16384. + # Override per run: HOST_COUNT / DPU_PER_HOST env to the setup script. + hostCount: 100 + dpuPerHostCount: 2 + vpcCount: 0 + subnetsPerVpc: 0 + dpuRebootDelay: 1 + hostRebootDelay: 1 + # Slower cadences than the dev profile — at thousands of machines the + # aggregate API/CPU load scales linearly with these. + scoutRunInterval: "300s" + runIntervalWorking: "5s" + runIntervalIdle: "60s" + networkStatusRunInterval: "120s" + templateDir: "/opt/machine-a-tron/templates" + # Gateways of the SIMULATED networks (added to nico-core by scale mode). + oobDhcpRelayAddress: "10.100.0.1" + adminDhcpRelayAddress: "10.102.0.1" + networkVirtualizationType: "" + dpusInNicMode: false + +extraEnv: + - name: RUST_LOG + value: "info" + diff --git a/helm-prereqs/values/machine-a-tron.yaml b/helm-prereqs/values/machine-a-tron.yaml new file mode 100644 index 0000000000..d25d4c385b --- /dev/null +++ b/helm-prereqs/values/machine-a-tron.yaml @@ -0,0 +1,105 @@ +# ============================================================================= +# machine-a-tron.yaml — site values template +# +# Copy this file and fill in site-specific values before deploying, OR let +# setup-machine-a-tron.sh consume it directly (it fills the site-specific +# fields from the running nico-core site config). +# See docs/development/machine-a-tron-deployment.md for the full guide. +# +# Deployment mode: Override Mode (nginxBmcProxy.enabled: false) +# site-explorer's Redfish calls are redirected to machine-a-tron's mock BMC. +# Not compatible with real hardware — use only on simulation-only clusters. +# +# IMPORTANT — the redirect is driven by nico-core, not this chart. nico-core's +# site_explorer config must contain (note the CROSS-NAMESPACE FQDN): +# [site_explorer] +# bmc_proxy = "nico-machine-a-tron-bmc-mock.nico-mat.svc.cluster.local:1266" +# (Field is `bmc_proxy`, a single "host:port" string. The older +# override_target_ip/override_target_port fields are DEPRECATED, and +# override_target_host was never a valid field. setup-machine-a-tron.sh sets +# bmc_proxy for you.) The FQDN is REQUIRED: site-explorer runs in nico-system +# and cannot resolve the bare service name (which resolves against its own +# namespace) — a bare name yields a Redfish "connection refused". Setting +# bmc_proxy at launch also makes allow_changing_bmc_proxy default to true, +# which is what lets configureBmcProxyHost (below) work at runtime. +# ============================================================================= + +## Registry-agnostic (like the NICo REST values): setup-machine-a-tron.sh +## injects image.repository = ${NICO_IMAGE_REGISTRY}/machine-a-tron and +## image.tag = ${MAT_IMAGE_TAG} at install time. Set them here only for +## direct helm installs without the script. +image: + repository: "" + tag: "" + pullPolicy: IfNotPresent + +global: + imagePullSecrets: + - name: machine-a-tron-pull + +# Override the SPIFFE URI so nico-api can authorize machine-a-tron calls. +# nico-api recognizes paths under /nico-system/sa/ and the RBAC rule for +# Machineatron expects SpiffeServiceIdentifier("machine-a-tron"). Without +# this override the cert-manager-issued cert uses the deployment namespace +# (/nico-mat/sa/nico-machine-a-tron) which is not in nico-api's trusted +# service_base_paths list and results in a 403 on every API call. +certificate: + uris: + - "spiffe://nico.local/nico-system/sa/machine-a-tron" + +machineATron: + nicoApiUrl: "https://nico-api.nico-system.svc.cluster.local:1079" + # When set, machine-a-tron calls nico-api set_dynamic_config to point + # site_explorer.bmc_proxy at ":" at runtime. + # This ONLY succeeds if nico-core has allow_changing_bmc_proxy=true (which is + # the default once nico-core launches with bmc_proxy set). It is a + # belt-and-suspenders companion to the nico-core bmc_proxy setting — harmless + # when both are set to the same target. + # MUST be the cross-namespace FQDN (site-explorer is in nico-system and cannot + # resolve the bare name). Update the ".nico-mat." segment if you deploy to a + # different namespace. + configureBmcProxyHost: "nico-machine-a-tron-bmc-mock.nico-mat.svc.cluster.local" + # Auto-register each mock host as an expected_machine (matched by BMC MAC). + # REQUIRED for end-to-end: site-explorer's MachineCreator refuses to create a + # managed host when no expected_machines row matches the discovered BMC MAC + # ("Refusing to create managed host, expected machines entry not found"). + registerExpectedMachines: true + +pods: + default: + machines: + # disable the chart-default example group (deep-merge would keep it) + rack-machines: null + dell-hosts: + hwType: dell_poweredge_r750 + # SIZING: machine-a-tron allocates one OOB DHCP IP per BMC interface — + # 1 per host + 1 per DPU. Total = hostCount + hostCount*dpuPerHostCount. + # This MUST fit the usable addresses in the OOB/underlay DHCP pool of your + # nico-core site config, or discovery reports "No IP addresses left in + # prefix ..." and machines never register. + # e.g. a /28 pool with a gateway and a few reserved addresses yields + # ~10 usable → hostCount:3 * dpuPerHostCount:2 = 9 fits; 5*2=15 does not. + # setup-machine-a-tron.sh estimates the pool size and warns if you overflow. + hostCount: 3 + dpuPerHostCount: 2 + vpcCount: 0 + subnetsPerVpc: 0 + dpuRebootDelay: 1 + hostRebootDelay: 1 + scoutRunInterval: "60s" + runIntervalWorking: "1s" + runIntervalIdle: "10s" + networkStatusRunInterval: "20s" + templateDir: "/opt/machine-a-tron/templates" + # Set to the gateway of the OOB/underlay network in nico-core site config. + # (setup-machine-a-tron.sh fills this from the running nico-core config.) + oobDhcpRelayAddress: "FILL_IN" + # Set to the gateway of the admin network in nico-core site config. + adminDhcpRelayAddress: "FILL_IN" + networkVirtualizationType: "" + dpusInNicMode: false + +extraEnv: + - name: RUST_LOG + value: "info" + From 65d444866b6b6cb6f778b7b1766a6418945fc1fe Mon Sep 17 00:00:00 2001 From: Shayan Namaghi Date: Thu, 9 Jul 2026 13:12:25 -0700 Subject: [PATCH 18/28] docs(machine-a-tron): 4500-node quick start, measured timings, final results MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Follow-ups on top of the adopted e2e commit: - deployment guide: add a "Quick start: a 4500-host simulated fleet" section (registry-agnostic env exports + one command) and a measured timing table from the live runs: DHCP ~60-90 min, exploration sweep ~3-5 h, creation drain ~2-3 h, ~6-9 h end to end unattended (100 hosts ~12 min, 1000 hosts ~25 min for calibration). - scale-testing writeup: record the final consolidated result — 13,500/13,500 machines, every pipeline counter at 100% (13,500 explored / 13,500 preingestion-complete / 4,500 hosts identified) on a freshly provisioned cluster. - Dockerfile: build the builder stage on $BUILDPLATFORM instead of a hardcoded linux/arm64. The pin existed because rustc under QEMU SEGFAULTs assembling aws-lc-sys's x86-64 assembly (s2n-bignum) on Apple Silicon hosts; with $BUILDPLATFORM the builder is always native (cross-compiling from arm64, natively compiling on x86_64 CI) and the cross toolchain is installed only when actually cross-building. - drop a registry-specific comment left in the setup script header. --- crates/machine-a-tron/Dockerfile | 26 +++++++---- docs/development/machine-a-tron-deployment.md | 46 ++++++++++++++++++- .../machine-a-tron-scale-testing.md | 2 +- helm-prereqs/setup-machine-a-tron.sh | 1 - 4 files changed, 64 insertions(+), 11 deletions(-) diff --git a/crates/machine-a-tron/Dockerfile b/crates/machine-a-tron/Dockerfile index 041e1f10c2..9ede8d9b09 100644 --- a/crates/machine-a-tron/Dockerfile +++ b/crates/machine-a-tron/Dockerfile @@ -1,8 +1,12 @@ # syntax=docker/dockerfile:1.7 # -# Cross-compilation: native arm64 Rust compiler targeting x86_64-unknown-linux-gnu. -# Avoids QEMU emulation so that aws-lc-sys (s2n-bignum assembly) compiles correctly — -# QEMU SEGFAULTs on the .S files even though the binary runs fine on real x86_64. +# Cross-compilation: the BUILDER always runs NATIVE on the build host +# ($BUILDPLATFORM) and cross-compiles to x86_64-unknown-linux-gnu. This is not +# a Mac convenience: building the amd64 stage the naive way on an Apple +# Silicon host runs rustc under QEMU emulation, and QEMU SEGFAULTs assembling +# aws-lc-sys's hand-written x86-64 assembly (s2n-bignum) — even though the +# resulting binary runs fine on real x86_64. On an x86_64 build host the +# builder is simply native and the "cross" compile is a no-op. # # Build from the repo root: # @@ -13,15 +17,19 @@ # -t //machine-a-tron:-amd64 \ # . # -FROM --platform=linux/arm64 rust:1.96.0-slim-bookworm AS builder +FROM --platform=$BUILDPLATFORM rust:1.96.0-slim-bookworm AS builder RUN apt-get update && apt-get install -y --no-install-recommends \ git \ pkg-config \ protobuf-compiler \ libprotobuf-dev \ - gcc-x86-64-linux-gnu \ - libc6-dev-amd64-cross \ + && if [ "$(uname -m)" != "x86_64" ]; then \ + apt-get install -y --no-install-recommends \ + gcc-x86-64-linux-gnu libc6-dev-amd64-cross; \ + else \ + apt-get install -y --no-install-recommends gcc libc6-dev; \ + fi \ && rm -rf /var/lib/apt/lists/* RUN rustup target add x86_64-unknown-linux-gnu @@ -29,8 +37,6 @@ RUN rustup target add x86_64-unknown-linux-gnu ENV CARGO_HOME=/cargo-home ENV CARGO_NET_GIT_FETCH_WITH_CLI=true ENV CARGO_TARGET_DIR=/cargo-target -ENV CARGO_TARGET_X86_64_UNKNOWN_LINUX_GNU_LINKER=x86_64-linux-gnu-gcc -ENV CC_x86_64_unknown_linux_gnu=x86_64-linux-gnu-gcc ENV RUST_BACKTRACE=1 WORKDIR /workspace @@ -38,6 +44,10 @@ COPY . . RUN --mount=type=cache,id=nico-mat-cross-cargo-home,target=/cargo-home,sharing=locked \ --mount=type=cache,id=nico-mat-cross-cargo-target,target=/cargo-target,sharing=locked \ + if [ "$(uname -m)" != "x86_64" ]; then \ + export CARGO_TARGET_X86_64_UNKNOWN_LINUX_GNU_LINKER=x86_64-linux-gnu-gcc \ + CC_x86_64_unknown_linux_gnu=x86_64-linux-gnu-gcc; \ + fi && \ cargo build -p carbide-machine-a-tron --bin machine-a-tron --locked \ --target x86_64-unknown-linux-gnu && \ mkdir -p /artifacts && \ diff --git a/docs/development/machine-a-tron-deployment.md b/docs/development/machine-a-tron-deployment.md index eba185e6db..077acd6a96 100644 --- a/docs/development/machine-a-tron-deployment.md +++ b/docs/development/machine-a-tron-deployment.md @@ -25,6 +25,50 @@ export REGISTRY_PULL_SECRET= # only if the pull secret is abse helm-prereqs/setup-machine-a-tron.sh # add -y for non-interactive ``` +## Quick start: a 4500-host simulated fleet + +Build and push the image to the registry of your choice, then run the setup +script in scale mode. Nothing registry-specific is committed — the image +location, tag, and pull credentials all come from the environment, exactly +like `setup.sh`: + +```bash +export KUBECONFIG=/path/to/site/kubeconfig +export NICO_IMAGE_REGISTRY=/ # e.g. registry.example.com/nico +export MAT_IMAGE_TAG= # tag you built and pushed (see §2) +export REGISTRY_PULL_SECRET= # omit if the pull secret already exists + +MAT_MODE=scale HOST_COUNT=4500 helm-prereqs/setup-machine-a-tron.sh -y +``` + +That is the whole procedure. The script exits after its bounded verification +window while ingestion continues in-cluster; progress is visible with: + +```bash +kubectl exec -n postgres -- su postgres -c \ + "psql -d nico_system_nico -tAc \"SELECT + (SELECT count(*) FROM explored_endpoints) || ' explored / ' || + (SELECT count(*) FROM machines) || ' machines';\"" +``` + +### What to expect (measured on a 3-node dev cluster, dev-sized postgres) + +4500 hosts × 2 DPUs = 13,500 BMC endpoints → 13,500 machines. + +| Phase | Duration | Notes | +|---|---|---| +| Deploy + DHCP registration | ~60–90 min | ~150–180 interfaces/min while 13.5k mock FSMs boot | +| Exploration sweep | ~3–5 h | overlaps DHCP; `explorations_per_run=120` per cycle | +| Preingestion | tracks the sweep | ~90% conversion, completes shortly after it | +| Identification + creation | final ~2–3 h | hosts identify in waves; creation drains at ~150–300 machines per explore cycle | +| **End to end** | **~6–9 h, unattended** | 100 hosts ≈ 12 min and 1000 hosts ≈ 25 min, for calibration | + +The pipeline is autonomous once the script completes — it has run through +multi-hour client connectivity outages without intervention. Occasional +nico-api restarts under peak ingestion load are absorbed by the pipeline +(machines resume within a cycle). Re-running the script is always safe +(idempotent) and re-registers any expected machines that arrived late. + The rest of this document explains what that script does and why, and is the reference for manual deployment or debugging. The script's header comments enumerate the non-obvious failure modes it guards against. @@ -83,7 +127,7 @@ docker login "${NICO_IMAGE_REGISTRY%%/*}" \ -p "${REGISTRY_PULL_SECRET}" ``` -For `nvcr.io` the username is the literal string `$oauthtoken`. +Some registries use a fixed username with API-key auth — set `REGISTRY_PULL_USERNAME` accordingly (default: `$oauthtoken`). ## 3. Cluster Prerequisites diff --git a/docs/development/machine-a-tron-scale-testing.md b/docs/development/machine-a-tron-scale-testing.md index 48c0fa8d96..0088a9c561 100644 --- a/docs/development/machine-a-tron-scale-testing.md +++ b/docs/development/machine-a-tron-scale-testing.md @@ -113,7 +113,7 @@ the "operator". Mocks have no lockout threshold, so this is safe here. | Baseline | 1 host × 1 DPU (override mode) | ✅ end-to-end: machines created, full credential rotation exercised | | Stage 1 | 100 hosts × 2 DPUs = 300 BMCs (proxy-direct) | ✅ 300/300 endpoints stable, machines created and advancing through `hostinit`/`dpuinit` | | Stage 2 | 1000 hosts × 2 DPUs = 3000 BMCs | ✅ **END TO END OK — 3000/3000 machines** in a single unattended script run (~25 min total; creation ≈ 240 machines/min) | -| Stage 3 | 4500 hosts × 2 DPUs = 13,500 BMCs | ✅ **exploration sweep complete: 13,500/13,500 endpoints**; 10,353 machines at time of writing with the remainder completing autonomously; fleet actively progressing through machine-controller lifecycle states (`dpuinit`, `dpudiscoveringstate`) on dev-sized postgres (4 CPU) | +| Stage 3 | 4500 hosts × 2 DPUs = 13,500 BMCs | ✅ **13,500/13,500 machines — 100% fleet** (first run: 13,500 endpoints explored, 10k+ machines; consolidated rerun on the #2764 ClusterIP chart + a freshly provisioned cluster: every counter at 100% — 13,500 explored / 13,500 preingestion-complete / 4,500 hosts / 13,500 machines, ~15 h wall clock unattended incl. connectivity outages) | Stage-3 observations worth reviewers' attention: diff --git a/helm-prereqs/setup-machine-a-tron.sh b/helm-prereqs/setup-machine-a-tron.sh index 38c8a846b3..de42b65f11 100755 --- a/helm-prereqs/setup-machine-a-tron.sh +++ b/helm-prereqs/setup-machine-a-tron.sh @@ -85,7 +85,6 @@ # pull secret does not already exist in the # machine-a-tron namespace. # REGISTRY_PULL_USERNAME Username for the pull secret. Default: $oauthtoken -# (correct for nvcr.io). # MAT_NAMESPACE Deployment namespace. Default: nico-mat # NICO_SYSTEM_NS NICo Core namespace. Default: nico-system # POSTGRES_NS Postgres namespace. Default: postgres From eb8bf8a318e0ff3e2c96e6909933d8f027da43b7 Mon Sep 17 00:00:00 2001 From: Alexander Korobkov Date: Thu, 9 Jul 2026 14:49:07 -0600 Subject: [PATCH 19/28] feat(devspace): k3s and machine-a-tron namespace support --- dev/deployment/devspace/values.base.yaml | 1 + devspace.yaml | 6 ++++++ .../nico-machine-a-tron/templates/namespace.yaml | 10 ++++++++++ helm/charts/nico-machine-a-tron/values.yaml | 5 +++++ 4 files changed, 22 insertions(+) create mode 100644 helm/charts/nico-machine-a-tron/templates/namespace.yaml diff --git a/dev/deployment/devspace/values.base.yaml b/dev/deployment/devspace/values.base.yaml index e4e70fae3f..efeef49369 100644 --- a/dev/deployment/devspace/values.base.yaml +++ b/dev/deployment/devspace/values.base.yaml @@ -141,6 +141,7 @@ nico-bmc-proxy: nico-machine-a-tron: enabled: true + namespaceOverride: "nico-mat" machineATron: nicoApiUrl: "https://nico-api.nico-system.svc.cluster.local:1079" pods: diff --git a/devspace.yaml b/devspace.yaml index aae7e68790..262aac786e 100644 --- a/devspace.yaml +++ b/devspace.yaml @@ -293,6 +293,12 @@ hooks: "${runtime.images.nico-bmc-proxy.image}:${runtime.images.nico-bmc-proxy.tag}" \ "${runtime.images.machine-a-tron.image}:${runtime.images.machine-a-tron.tag}" ;; + k3s-*) + echo "Loading images into k3s cluster..." + docker save "${runtime.images.nico-api.image}:${runtime.images.nico-api.tag}" | sudo k3s ctr images import - + docker save "${runtime.images.nico-bmc-proxy.image}:${runtime.images.nico-bmc-proxy.tag}" | sudo k3s ctr images import - + docker save "${runtime.images.machine-a-tron.image}:${runtime.images.machine-a-tron.tag}" | sudo k3s ctr images import - + ;; # Add new local K8s tools here esac - name: load-rest-images-into-local-cluster diff --git a/helm/charts/nico-machine-a-tron/templates/namespace.yaml b/helm/charts/nico-machine-a-tron/templates/namespace.yaml new file mode 100644 index 0000000000..bc0d92cdfd --- /dev/null +++ b/helm/charts/nico-machine-a-tron/templates/namespace.yaml @@ -0,0 +1,10 @@ +{{- if and .Values.namespaceOverride .Values.createNamespace }} +apiVersion: v1 +kind: Namespace +metadata: + name: {{ include "nico-machine-a-tron.namespace" . }} + annotations: + helm.sh/resource-policy: keep + labels: + {{- include "nico-machine-a-tron.labels" . | nindent 4 }} +{{- end }} diff --git a/helm/charts/nico-machine-a-tron/values.yaml b/helm/charts/nico-machine-a-tron/values.yaml index 0970c74f27..5d3fa1f308 100644 --- a/helm/charts/nico-machine-a-tron/values.yaml +++ b/helm/charts/nico-machine-a-tron/values.yaml @@ -36,12 +36,17 @@ global: app.kubernetes.io/part-of: site-controller image: + ## Container image repository. When empty, uses global.image.repository or defaults to nico:latest repository: "" + ## Container image tag. Defaults to chart appVersion tag: "" pullPolicy: IfNotPresent namespaceOverride: "" +## Create namespace when namespaceOverride is set +createNamespace: true + automountServiceAccountToken: true ## Graceful shutdown timeout (seconds) From c5fe4a7babd12120283fd4c929ce6ba915c0f22e Mon Sep 17 00:00:00 2001 From: Alexander Korobkov Date: Thu, 9 Jul 2026 14:51:08 -0600 Subject: [PATCH 20/28] feat(ci): machine-a-tron container build --- .github/workflows/ci.yaml | 32 ++++++++++++++++++ crates/machine-a-tron/Dockerfile | 56 ++++++++++++++++++++++---------- 2 files changed, 70 insertions(+), 18 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index a0b266c7c7..c7edb8abd1 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -751,6 +751,35 @@ jobs: ${{ needs.prepare.outputs.image_registry }}/nvmetal-carbide@${{ needs.build-release-container-x86_64.outputs.image_digest }} \ ${{ needs.prepare.outputs.image_registry }}/nvmetal-carbide-aarch64@${{ needs.build-release-container-aarch64.outputs.image_digest }} + build-machine-a-tron: + if: >- + ${{ + always() + && github.event_name != 'schedule' + && needs.prepare.result == 'success' + && needs.prepare.outputs.source_files_changed == 'true' + }} + needs: + - prepare + uses: ./.github/workflows/docker-build.yml + with: + dockerfile_path: crates/machine-a-tron/Dockerfile + context_path: . + build_args: >- + ${{ format('{{"VERSION":"{0}","CI_COMMIT_SHORT_SHA":"{1}"}}', + needs.prepare.outputs.version, + needs.prepare.outputs.short_sha) }} + image_name: ${{ needs.prepare.outputs.image_registry }}/machine-a-tron + image_tag: ${{ needs.prepare.outputs.version }} + platforms: linux/amd64 + runner: linux-amd64-cpu16 + push: ${{ needs.prepare.outputs.publish_images == 'true' }} + load: false + scan: true + tag_latest: false + timeout_minutes: 60 + secrets: inherit + test-release-container-services: if: >- ${{ @@ -1738,6 +1767,7 @@ jobs: - build-artifacts-container-aarch64 - build-release-container-x86_64 - build-release-container-aarch64 + - build-machine-a-tron - test-release-container-services - build-boot-artifacts-x86 - build-boot-artifacts-bfb @@ -1838,6 +1868,7 @@ jobs: runs-on: linux-amd64-cpu4 needs: - build-release-container-x86_64 + - build-machine-a-tron - build-forge-cli-x86_64 - build-release-artifacts-x86-host - build-release-artifacts-arm-host @@ -1868,6 +1899,7 @@ jobs: - build-artifacts-container-aarch64 - build-release-container-x86_64 - build-release-container-aarch64 + - build-machine-a-tron - test-release-container-services - build-boot-artifacts-x86 - build-boot-artifacts-bfb diff --git a/crates/machine-a-tron/Dockerfile b/crates/machine-a-tron/Dockerfile index 9ede8d9b09..bfb77dd48d 100644 --- a/crates/machine-a-tron/Dockerfile +++ b/crates/machine-a-tron/Dockerfile @@ -13,11 +13,18 @@ # docker buildx build \ # --platform linux/amd64 \ # -f crates/machine-a-tron/Dockerfile \ -# --push \ -# -t //machine-a-tron:-amd64 \ +# --build-arg VERSION=0.1.0 \ +# -t machine-a-tron:latest \ # . # -FROM --platform=$BUILDPLATFORM rust:1.96.0-slim-bookworm AS builder +ARG RUST_VERSION=1.96.0 + +FROM --platform=$BUILDPLATFORM rust:${RUST_VERSION}-slim-bookworm AS builder + +ARG VERSION +ARG CI_COMMIT_SHORT_SHA +ENV VERSION=${VERSION} +ENV CI_COMMIT_SHORT_SHA=${CI_COMMIT_SHORT_SHA} RUN apt-get update && apt-get install -y --no-install-recommends \ git \ @@ -42,28 +49,41 @@ ENV RUST_BACKTRACE=1 WORKDIR /workspace COPY . . +# Build release binary with cross-compilation RUN --mount=type=cache,id=nico-mat-cross-cargo-home,target=/cargo-home,sharing=locked \ - --mount=type=cache,id=nico-mat-cross-cargo-target,target=/cargo-target,sharing=locked \ - if [ "$(uname -m)" != "x86_64" ]; then \ - export CARGO_TARGET_X86_64_UNKNOWN_LINUX_GNU_LINKER=x86_64-linux-gnu-gcc \ - CC_x86_64_unknown_linux_gnu=x86_64-linux-gnu-gcc; \ - fi && \ - cargo build -p carbide-machine-a-tron --bin machine-a-tron --locked \ - --target x86_64-unknown-linux-gnu && \ - mkdir -p /artifacts && \ - cp /cargo-target/x86_64-unknown-linux-gnu/debug/machine-a-tron /artifacts/machine-a-tron + --mount=type=cache,id=nico-mat-cross-cargo-target,target=/cargo-target,sharing=locked \ + if [ "$(uname -m)" != "x86_64" ]; then \ + export CARGO_TARGET_X86_64_UNKNOWN_LINUX_GNU_LINKER=x86_64-linux-gnu-gcc \ + CC_x86_64_unknown_linux_gnu=x86_64-linux-gnu-gcc; \ + fi && \ + cargo build -p carbide-machine-a-tron --bin machine-a-tron --locked --release \ + --target x86_64-unknown-linux-gnu && \ + mkdir -p /artifacts && \ + cp /cargo-target/x86_64-unknown-linux-gnu/release/machine-a-tron /artifacts/machine-a-tron +# Runtime image FROM --platform=linux/amd64 debian:bookworm-slim +ARG VERSION +ARG CI_COMMIT_SHORT_SHA +LABEL org.opencontainers.image.version="${VERSION}" +LABEL org.opencontainers.image.revision="${CI_COMMIT_SHORT_SHA}" +LABEL org.opencontainers.image.title="machine-a-tron" +LABEL org.opencontainers.image.description="NICo Hardware Simulation Tool" +LABEL org.opencontainers.image.vendor="NVIDIA" + RUN apt-get update && apt-get install -y --no-install-recommends \ - ca-certificates \ - iproute2 \ - iputils-ping \ - libssl3 \ - libudev1 \ - && rm -rf /var/lib/apt/lists/* + ca-certificates \ + iproute2 \ + iputils-ping \ + libssl3 \ + libudev1 \ + && rm -rf /var/lib/apt/lists/* RUN mkdir -p /opt/machine-a-tron/bin /opt/machine-a-tron/templates /tmp/machine-a-tron-data COPY --from=builder /artifacts/machine-a-tron /opt/machine-a-tron/bin/machine-a-tron COPY crates/machine-a-tron/templates /opt/machine-a-tron/templates + +WORKDIR /opt/machine-a-tron +ENTRYPOINT ["/opt/machine-a-tron/bin/machine-a-tron"] From 251c79326a55456d2f6554ca00703a160b513802 Mon Sep 17 00:00:00 2001 From: Alexander Korobkov Date: Thu, 9 Jul 2026 15:15:29 -0600 Subject: [PATCH 21/28] chore(format): rbac rules --- crates/api-core/src/auth/internal_rbac_rules.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/crates/api-core/src/auth/internal_rbac_rules.rs b/crates/api-core/src/auth/internal_rbac_rules.rs index 0b04966f00..995e4f1f17 100644 --- a/crates/api-core/src/auth/internal_rbac_rules.rs +++ b/crates/api-core/src/auth/internal_rbac_rules.rs @@ -322,7 +322,10 @@ impl InternalRBACRules { x.perm("UpdateInstancePhoneHomeLastContact", vec![Agent]); x.perm("SetHostUefiPassword", vec![ForgeAdminCLI]); x.perm("ClearHostUefiPassword", vec![ForgeAdminCLI]); - x.perm("AddExpectedMachine", vec![ForgeAdminCLI, SiteAgent, Flow, Machineatron]); + x.perm( + "AddExpectedMachine", + vec![ForgeAdminCLI, SiteAgent, Flow, Machineatron], + ); x.perm("DeleteExpectedMachine", vec![ForgeAdminCLI, SiteAgent]); x.perm("UpdateExpectedMachine", vec![ForgeAdminCLI, SiteAgent]); x.perm("CreateExpectedMachines", vec![ForgeAdminCLI, SiteAgent]); From 80c4aba5e7730af9a027343eb94deee203adbb79 Mon Sep 17 00:00:00 2001 From: Shayan Namaghi Date: Fri, 10 Jul 2026 14:12:37 -0700 Subject: [PATCH 22/28] =?UTF-8?q?docs(machine-a-tron):=20multi-pod=20bmcSe?= =?UTF-8?q?rvices=20mode=20=E2=80=94=20validated=20example=20+=20findings?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Validated two-pod bmcServices simulation end to end on a dev cluster (2 pods x 5 wiwynn_gb200_nvl hosts x 2 DPUs -> 30/30 machines, 15 per pod) and captured the requirements it surfaced: * ServiceCIDR needs Kubernetes 1.29+; on older clusters carve pod CIDRs from --service-cluster-ip-range with serviceCIDR.create=false * direct ClusterIP dialing requires the bmc-mock Host-header fallback (PR #3190) since no Forwarded header is sent without a bmc_proxy * every pod needs disjoint mac_address_pool / hw_mac_address_ranges bases (identical defaults -> second pod rejected with 'Network segment mismatch for existing MAC address'); the chart has no per-pod MAC knob, so the example uses full configFiles.matConfigs overrides * one NICo network segment per pod CIDR (one-IPv4-prefix-per-segment constraint) * unrecognized Redfish vendors (WIWYNN) resolve to the 'unknown' factory credential path, and GB-class hosts require expected_machines rows with the pinned credential before exploration completes --- docs/development/machine-a-tron-deployment.md | 45 ++++ .../values/machine-a-tron-multipod.yaml | 194 ++++++++++++++++++ 2 files changed, 239 insertions(+) create mode 100644 helm-prereqs/values/machine-a-tron-multipod.yaml diff --git a/docs/development/machine-a-tron-deployment.md b/docs/development/machine-a-tron-deployment.md index 077acd6a96..383b38157d 100644 --- a/docs/development/machine-a-tron-deployment.md +++ b/docs/development/machine-a-tron-deployment.md @@ -299,6 +299,51 @@ matching `expected_machines` row exists (by BMC MAC) — otherwise it logs auto-registers these when `machineATron.registerExpectedMachines: true` (the default in the values file). DHCP discovery alone is **not** sufficient. +## 5b. Multi-pod simulation with per-BMC ClusterIP services (`bmcServices`) + +The chart can shard the simulated fleet across several machine-a-tron pods, +each with a dedicated ClusterIP range where every simulated BMC gets its own +Service (`pods..cidr` + `bmcServices.enabled: true`). NICo then dials +each BMC IP directly — no `bmc_proxy`. A validated two-pod example lives at +`helm-prereqs/values/machine-a-tron-multipod.yaml` (2 pods × 5 +`wiwynn_gb200_nvl` hosts × 2 DPUs → 30 machines, 15 per pod). + +Everything single-pod mode needs still applies (namespaces, CA copy, Vault +seeds, SPIFFE URI). Multi-pod adds five requirements, all hit in practice: + +1. **Kubernetes 1.29+** for the `ServiceCIDR` object. On older clusters set + `bmcServices.serviceCIDR.create: false` and pick pod CIDRs **inside** the + apiserver's `--service-cluster-ip-range` — a static `clusterIP` outside + that range is rejected ("the provided IP is not in the valid range"). + Check the chosen sub-ranges are free of existing ClusterIPs first. +2. **Image with the `Host`-header routing fallback (PR #3190).** Without a + `bmc_proxy` the Redfish client sends no `Forwarded` header; older bmc-mock + builds then 404 every request with `no router configured`. Leave + `site_explorer.bmc_proxy` unset in this mode. +3. **Disjoint MAC pools per pod.** All machine-a-tron instances derive MACs + from the same default pools, so the pod that leases second is rejected on + every DHCP with `Network segment mismatch for existing MAC address` and + simulates nothing. Until the chart grows a per-pod MAC knob, supply full + per-pod TOML overrides via `configFiles.matConfigs.` setting distinct + `mac_address_pool` / `hw_mac_address_ranges` bases (see the example + values file). +4. **One NICo network segment per pod CIDR.** `network_prefixes` allows one + IPv4 prefix per segment, so each pod CIDR needs its own cloned underlay + segment (same technique as the scale-mode segment fallback), gateway `.1`, + `num_reserved 1` — BMC Service IPs start at `.2`, matching the DHCP + allocator. +5. **Hardware-type specifics.** Vendors libredfish does not recognize (e.g. + `wiwynn_gb200_nvl` reports `WIWYNN`) resolve to `unknown`, so seed the + host factory credential at + `machines/all_hosts/factory_default/bmc-metadata-items/unknown`. GB-class + hosts (`NvidiaGBx00` flow) additionally require `expected_machines` rows + with a working BMC credential before exploration completes + (`NICO-SITEEXPLORER-141 Missing credential expected_machine`); on nico-api + builds without the Machineatron `AddExpectedMachine` RBAC grant the + auto-registration is 403'd and the rows must be inserted directly with the + **pinned** password (`hostBmcPassword`), not the factory default. DPU BMCs + explore without expected rows. + ## 6. Verifying Startup Check that machine-a-tron passes the initial API calls: diff --git a/helm-prereqs/values/machine-a-tron-multipod.yaml b/helm-prereqs/values/machine-a-tron-multipod.yaml new file mode 100644 index 0000000000..723d24e3a4 --- /dev/null +++ b/helm-prereqs/values/machine-a-tron-multipod.yaml @@ -0,0 +1,194 @@ +# machine-a-tron site values — MULTI-POD bmcServices mode (validated example) +# +# Two machine-a-tron pods, each simulating 5 hosts x 2 DPUs (30 BMCs total), +# each with a dedicated per-BMC ClusterIP Service range. Copy this file and +# adjust CIDRs/counts for your site, then deploy with: +# +# helm upgrade -i nico-machine-a-tron helm/charts/nico-machine-a-tron \ +# -n nico-mat -f .yaml +# +# READ FIRST — requirements this mode adds on top of single-pod mode: +# +# * Kubernetes 1.29+ for the ServiceCIDR object (beta 1.31+). On OLDER +# clusters set bmcServices.serviceCIDR.create=false and pick pod CIDRs +# INSIDE the apiserver's --service-cluster-ip-range — any clusterIP outside +# that range is rejected outright ("the provided IP is not in the valid +# range"). Verify the chosen sub-ranges are free of existing ClusterIPs: +# kubectl get svc -A -o jsonpath='{range .items[*]}{.spec.clusterIP}{"\n"}{end}' +# +# * The machine-a-tron image must include the bmc-mock Host-header routing +# fallback (PR #3190). Without a bmc_proxy the Redfish client sends no +# "Forwarded" header, and older mocks 404 every request with +# "no router configured". Do NOT set site_explorer.bmc_proxy in this mode — +# NICo dials each BMC's ClusterIP directly. +# +# * Every pod MUST get disjoint MAC pools (the per-pod TOML overrides below). +# All machine-a-tron instances otherwise derive identical MACs from the +# default pools (02:00:00:00:00:00 / 06:00:00:00:00:00); the pod that +# leases second is rejected on every DHCP with "Network segment mismatch +# for existing MAC address" and simulates nothing. The chart has no +# per-pod MAC knob yet, so the configFiles.matConfigs overrides are the +# only way to set mac_address_pool / hw_mac_address_ranges per pod. +# +# * One NICo network segment per pod CIDR: network_prefixes has a +# one-IPv4-prefix-per-segment constraint, so the pod CIDRs cannot be added +# as two prefixes of one OOB segment. Clone one underlay segment per pod +# (same jsonb_populate_record technique as setup-machine-a-tron.sh scale +# mode), prefix = the pod CIDR, gateway = .1, num_reserved = 1 — the chart +# allocates BMC Service IPs starting at .2, which then lines up with the +# DHCP allocator. +# +# * Vendor-specific factory credential path: hw types whose Redfish vendor +# string is not recognized by libredfish (e.g. wiwynn_gb200_nvl reports +# "WIWYNN") resolve to BMCVendor::Unknown, so the host factory credential +# must be seeded at machines/all_hosts/factory_default/bmc-metadata-items/unknown. +# +# * GB-class hosts (NvidiaGBx00 flow, e.g. wiwynn_gb200_nvl) require +# expected_machines rows with a WORKING BMC credential before exploration +# completes (error NICO-SITEEXPLORER-141 "Missing credential +# expected_machine"). machine-a-tron registers them itself when +# registerExpectedMachines=true, but only on nico-api builds with the +# Machineatron AddExpectedMachine RBAC grant — on older builds the call is +# 403'd and the rows must be inserted directly (see the fallback in +# setup-machine-a-tron.sh; the row password must be the PINNED password +# below, not the factory default). DPU BMCs explore without expected rows. + +certificate: + uris: + - spiffe://nico.local/nico-system/sa/machine-a-tron + +extraEnv: + - name: RUST_LOG + value: info + +global: + imagePullSecrets: + - name: machine-a-tron-pull + +# Build and push the image to the registry of your choice (see the deployment +# guide); the setup script injects these from NICO_IMAGE_REGISTRY/MAT_IMAGE_TAG +# when deploying via setup-machine-a-tron.sh. +image: + pullPolicy: IfNotPresent + repository: "" + tag: "" + +machineATron: + configureBmcProxyHost: "" + hostBmcPassword: NicoSiteRoot1 + dpuBmcPassword: NicoSiteRoot1 + nicoApiUrl: https://nico-api.nico-system.svc.cluster.local:1079 + registerExpectedMachines: true + useSingleBmcMock: true + +bmcServices: + enabled: true + servicePort: 443 + serviceCIDR: + # Requires Kubernetes 1.29+. On older clusters leave false and carve the + # pod CIDRs from the cluster's existing service range (see header). + create: false + +pods: + mat-0: + cidr: "10.233.58.0/23" + machines: + compute: + hwType: wiwynn_gb200_nvl + hostCount: 5 + dpuPerHostCount: 2 + oobDhcpRelayAddress: "10.233.58.1" + adminDhcpRelayAddress: "10.102.0.1" + mat-1: + cidr: "10.233.60.0/23" + machines: + compute: + hwType: wiwynn_gb200_nvl + hostCount: 5 + dpuPerHostCount: 2 + oobDhcpRelayAddress: "10.233.60.1" + adminDhcpRelayAddress: "10.102.0.1" + +resources: + limits: + cpu: "2" + memory: 2Gi + requests: + cpu: 500m + memory: 512Mi + +# Full per-pod TOML overrides. The chart uses these VERBATIM instead of +# generating mat.toml from pods..machines, so they must repeat the +# machine sections — keep them in sync with the pods block above. They exist +# solely to give each pod disjoint MAC pools (see header). +configFiles: + matConfigs: + mat-0: | + # Pod: mat-0 CIDR: 10.233.58.0/23 + carbide_api_url = "https://nico-api.nico-system.svc.cluster.local:1079" + interface = "NOTUSED" + tui_enabled = false + bmc_mock_port = 1266 + use_single_bmc_mock = true + mock_bmc_ssh_server = true + mock_bmc_ssh_port = 2222 + cleanup_on_quit = false + register_expected_machines = true + host_bmc_password = "NicoSiteRoot1" + dpu_bmc_password = "NicoSiteRoot1" + api_refresh_interval = "2s" + + mac_address_pool = { base = "02:00:00:00:00:00", host_bits = 24 } + hw_mac_address_ranges = { base = "06:00:00:00:00:00", host_bits = 32, range_host_bits = 8 } + + [machines.compute] + hw_type = "wiwynn_gb200_nvl" + host_count = 5 + vpc_count = 0 + subnets_per_vpc = 0 + dpu_per_host_count = 2 + dpu_reboot_delay = 1 + host_reboot_delay = 1 + scout_run_interval = "60s" + run_interval_working = "1s" + run_interval_idle = "10s" + network_status_run_interval = "20s" + template_dir = "/opt/machine-a-tron/templates" + oob_dhcp_relay_address = "10.233.58.1" + admin_dhcp_relay_address = "10.102.0.1" + dpus_in_nic_mode = false + mat-1: | + # Pod: mat-1 CIDR: 10.233.60.0/23 + carbide_api_url = "https://nico-api.nico-system.svc.cluster.local:1079" + interface = "NOTUSED" + tui_enabled = false + bmc_mock_port = 1266 + use_single_bmc_mock = true + mock_bmc_ssh_server = true + mock_bmc_ssh_port = 2222 + cleanup_on_quit = false + register_expected_machines = true + host_bmc_password = "NicoSiteRoot1" + dpu_bmc_password = "NicoSiteRoot1" + api_refresh_interval = "2s" + + # Disjoint MAC spaces from mat-0 (outside its 2^24 / 2^32 host spans) + mac_address_pool = { base = "02:00:01:00:00:00", host_bits = 24 } + hw_mac_address_ranges = { base = "06:01:00:00:00:00", host_bits = 32, range_host_bits = 8 } + + [machines.compute] + hw_type = "wiwynn_gb200_nvl" + host_count = 5 + vpc_count = 0 + subnets_per_vpc = 0 + dpu_per_host_count = 2 + dpu_reboot_delay = 1 + host_reboot_delay = 1 + scout_run_interval = "60s" + run_interval_working = "1s" + run_interval_idle = "10s" + network_status_run_interval = "20s" + template_dir = "/opt/machine-a-tron/templates" + oob_dhcp_relay_address = "10.233.60.1" + admin_dhcp_relay_address = "10.102.0.1" + dpus_in_nic_mode = false From 5288159706842c9d521b4c7ef3ee9cf95838e0a7 Mon Sep 17 00:00:00 2001 From: Alexander Korobkov Date: Tue, 14 Jul 2026 10:27:26 -0600 Subject: [PATCH 23/28] fix(devspace): machine-a-tron values --- dev/deployment/devspace/values.base.yaml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/dev/deployment/devspace/values.base.yaml b/dev/deployment/devspace/values.base.yaml index efeef49369..6f352a9386 100644 --- a/dev/deployment/devspace/values.base.yaml +++ b/dev/deployment/devspace/values.base.yaml @@ -141,7 +141,6 @@ nico-bmc-proxy: nico-machine-a-tron: enabled: true - namespaceOverride: "nico-mat" machineATron: nicoApiUrl: "https://nico-api.nico-system.svc.cluster.local:1079" pods: @@ -149,7 +148,7 @@ nico-machine-a-tron: machines: config: hwType: wiwynn_gb200_nvl - hostCount: 10 + hostCount: 2 dpuPerHostCount: 2 oobDhcpRelayAddress: "192.168.192.1" adminDhcpRelayAddress: "192.168.176.1" From 836ff8a6b9d0bd4ba32514e04b563b50b1437361 Mon Sep 17 00:00:00 2001 From: Alexander Korobkov Date: Tue, 14 Jul 2026 11:35:56 -0600 Subject: [PATCH 24/28] fix(devspace): machine-a-tron values --- dev/deployment/devspace/values.base.yaml | 11 +++++++++-- devspace.yaml | 13 +++++++++---- 2 files changed, 18 insertions(+), 6 deletions(-) diff --git a/dev/deployment/devspace/values.base.yaml b/dev/deployment/devspace/values.base.yaml index 6f352a9386..8cc6dfca7b 100644 --- a/dev/deployment/devspace/values.base.yaml +++ b/dev/deployment/devspace/values.base.yaml @@ -141,18 +141,25 @@ nico-bmc-proxy: nico-machine-a-tron: enabled: true + # Deploy to nico-system (same namespace as nico-api) for simpler dev setup + # No need to copy secrets or configure cross-namespace access machineATron: nicoApiUrl: "https://nico-api.nico-system.svc.cluster.local:1079" + # Auto-configure site-explorer to use this BMC mock endpoint + configureBmcProxyHost: "nico-machine-a-tron-bmc-mock.nico-system.svc.cluster.local" + # SPIFFE URI must match nico-api's allowed service_base_paths + certificate: + uris: + - "spiffe://nico.local/nico-system/sa/machine-a-tron" pods: default: machines: - config: + rack-machines: hwType: wiwynn_gb200_nvl hostCount: 2 dpuPerHostCount: 2 oobDhcpRelayAddress: "192.168.192.1" adminDhcpRelayAddress: "192.168.176.1" - nico-dhcp: enabled: false nico-dns: diff --git a/devspace.yaml b/devspace.yaml index 262aac786e..839a005821 100644 --- a/devspace.yaml +++ b/devspace.yaml @@ -294,10 +294,15 @@ hooks: "${runtime.images.machine-a-tron.image}:${runtime.images.machine-a-tron.tag}" ;; k3s-*) - echo "Loading images into k3s cluster..." - docker save "${runtime.images.nico-api.image}:${runtime.images.nico-api.tag}" | sudo k3s ctr images import - - docker save "${runtime.images.nico-bmc-proxy.image}:${runtime.images.nico-bmc-proxy.tag}" | sudo k3s ctr images import - - docker save "${runtime.images.machine-a-tron.image}:${runtime.images.machine-a-tron.tag}" | sudo k3s ctr images import - + # k3s in uses 'default' context by default. + # Rename it to match this pattern: + # sudo kubectl config rename-context default k3s-nico + if command -v k3s &> /dev/null; then + echo "Loading images into k3s cluster..." + docker save "${runtime.images.nico-api.image}:${runtime.images.nico-api.tag}" | sudo k3s ctr images import - + docker save "${runtime.images.nico-bmc-proxy.image}:${runtime.images.nico-bmc-proxy.tag}" | sudo k3s ctr images import - + docker save "${runtime.images.machine-a-tron.image}:${runtime.images.machine-a-tron.tag}" | sudo k3s ctr images import - + fi ;; # Add new local K8s tools here esac From ce883768de90298363a778b523da960f0314c420 Mon Sep 17 00:00:00 2001 From: Alexander Korobkov Date: Tue, 14 Jul 2026 11:52:45 -0600 Subject: [PATCH 25/28] fix(ci): machine-a-tron grype image scan --- .github/workflows/ci.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index c7edb8abd1..b9547aff09 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -774,7 +774,7 @@ jobs: platforms: linux/amd64 runner: linux-amd64-cpu16 push: ${{ needs.prepare.outputs.publish_images == 'true' }} - load: false + load: true scan: true tag_latest: false timeout_minutes: 60 From c8b39f5f8ec0e63001b41672e7df48da411b3b2f Mon Sep 17 00:00:00 2001 From: Alexander Korobkov Date: Tue, 14 Jul 2026 11:54:58 -0600 Subject: [PATCH 26/28] fix(devspace): machine-a-tron values --- dev/deployment/devspace/values.base.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/dev/deployment/devspace/values.base.yaml b/dev/deployment/devspace/values.base.yaml index 8cc6dfca7b..6dff64ef9e 100644 --- a/dev/deployment/devspace/values.base.yaml +++ b/dev/deployment/devspace/values.base.yaml @@ -1,4 +1,7 @@ nico-api: + # Disable legacy forge-system namespace alias for local dev + legacyAlias: + enabled: false resources: limits: cpu: "16" From 80e6de15742a169bc8ad4c21ecb735ca4f452d77 Mon Sep 17 00:00:00 2001 From: Alexander Korobkov Date: Tue, 14 Jul 2026 12:04:00 -0600 Subject: [PATCH 27/28] fix(devspace): devspace k3s support --- devspace.yaml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/devspace.yaml b/devspace.yaml index 839a005821..52e851b51b 100644 --- a/devspace.yaml +++ b/devspace.yaml @@ -294,7 +294,7 @@ hooks: "${runtime.images.machine-a-tron.image}:${runtime.images.machine-a-tron.tag}" ;; k3s-*) - # k3s in uses 'default' context by default. + # k3s uses 'default' context by default. # Rename it to match this pattern: # sudo kubectl config rename-context default k3s-nico if command -v k3s &> /dev/null; then @@ -302,6 +302,9 @@ hooks: docker save "${runtime.images.nico-api.image}:${runtime.images.nico-api.tag}" | sudo k3s ctr images import - docker save "${runtime.images.nico-bmc-proxy.image}:${runtime.images.nico-bmc-proxy.tag}" | sudo k3s ctr images import - docker save "${runtime.images.machine-a-tron.image}:${runtime.images.machine-a-tron.tag}" | sudo k3s ctr images import - + else + echo "ERROR: k3s command is required to load local images for a k3s-* context" >&2 + exit 1 fi ;; # Add new local K8s tools here From c891bc7c75974c8d6aaef170b1aabba41e6ba9af Mon Sep 17 00:00:00 2001 From: Alexander Korobkov Date: Tue, 14 Jul 2026 18:19:19 -0600 Subject: [PATCH 28/28] fix(devspace): address code review feedback --- dev/deployment/devspace/values.base.yaml | 6 ++---- devspace.yaml | 14 -------------- 2 files changed, 2 insertions(+), 18 deletions(-) diff --git a/dev/deployment/devspace/values.base.yaml b/dev/deployment/devspace/values.base.yaml index 6dff64ef9e..d1839a90c7 100644 --- a/dev/deployment/devspace/values.base.yaml +++ b/dev/deployment/devspace/values.base.yaml @@ -94,7 +94,7 @@ nico-api: enabled = true create_machines = true allow_changing_bmc_proxy = true - bmc_proxy = "machine-a-tron-bmc-mock.nico-system.svc.cluster.local:1266" + bmc_proxy = "nico-machine-a-tron-bmc-mock.nico-system.svc.cluster.local:1266" run_interval = "10s" machines_created_per_run = 1000 explorations_per_run = 1000 @@ -116,7 +116,7 @@ nico-bmc-proxy: metrics_endpoint = "[::]:1080" database_url = "postgres://replaced-by-env-var" allowed_principals = ["trusted-certificate"] - bmc_proxy = "machine-a-tron-bmc-mock.nico-system.svc.cluster.local:1266" + bmc_proxy = "nico-machine-a-tron-bmc-mock.nico-system.svc.cluster.local:1266" [tls] identity_pemfile_path = "/var/run/secrets/spiffe.io/tls.crt" @@ -148,8 +148,6 @@ nico-machine-a-tron: # No need to copy secrets or configure cross-namespace access machineATron: nicoApiUrl: "https://nico-api.nico-system.svc.cluster.local:1079" - # Auto-configure site-explorer to use this BMC mock endpoint - configureBmcProxyHost: "nico-machine-a-tron-bmc-mock.nico-system.svc.cluster.local" # SPIFFE URI must match nico-api's allowed service_base_paths certificate: uris: diff --git a/devspace.yaml b/devspace.yaml index 52e851b51b..aae7e68790 100644 --- a/devspace.yaml +++ b/devspace.yaml @@ -293,20 +293,6 @@ hooks: "${runtime.images.nico-bmc-proxy.image}:${runtime.images.nico-bmc-proxy.tag}" \ "${runtime.images.machine-a-tron.image}:${runtime.images.machine-a-tron.tag}" ;; - k3s-*) - # k3s uses 'default' context by default. - # Rename it to match this pattern: - # sudo kubectl config rename-context default k3s-nico - if command -v k3s &> /dev/null; then - echo "Loading images into k3s cluster..." - docker save "${runtime.images.nico-api.image}:${runtime.images.nico-api.tag}" | sudo k3s ctr images import - - docker save "${runtime.images.nico-bmc-proxy.image}:${runtime.images.nico-bmc-proxy.tag}" | sudo k3s ctr images import - - docker save "${runtime.images.machine-a-tron.image}:${runtime.images.machine-a-tron.tag}" | sudo k3s ctr images import - - else - echo "ERROR: k3s command is required to load local images for a k3s-* context" >&2 - exit 1 - fi - ;; # Add new local K8s tools here esac - name: load-rest-images-into-local-cluster