diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index a0b266c7c7..b9547aff09 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -751,6 +751,35 @@ jobs: ${{ needs.prepare.outputs.image_registry }}/nvmetal-carbide@${{ needs.build-release-container-x86_64.outputs.image_digest }} \ ${{ needs.prepare.outputs.image_registry }}/nvmetal-carbide-aarch64@${{ needs.build-release-container-aarch64.outputs.image_digest }} + build-machine-a-tron: + if: >- + ${{ + always() + && github.event_name != 'schedule' + && needs.prepare.result == 'success' + && needs.prepare.outputs.source_files_changed == 'true' + }} + needs: + - prepare + uses: ./.github/workflows/docker-build.yml + with: + dockerfile_path: crates/machine-a-tron/Dockerfile + context_path: . + build_args: >- + ${{ format('{{"VERSION":"{0}","CI_COMMIT_SHORT_SHA":"{1}"}}', + needs.prepare.outputs.version, + needs.prepare.outputs.short_sha) }} + image_name: ${{ needs.prepare.outputs.image_registry }}/machine-a-tron + image_tag: ${{ needs.prepare.outputs.version }} + platforms: linux/amd64 + runner: linux-amd64-cpu16 + push: ${{ needs.prepare.outputs.publish_images == 'true' }} + load: true + scan: true + tag_latest: false + timeout_minutes: 60 + secrets: inherit + test-release-container-services: if: >- ${{ @@ -1738,6 +1767,7 @@ jobs: - build-artifacts-container-aarch64 - build-release-container-x86_64 - build-release-container-aarch64 + - build-machine-a-tron - test-release-container-services - build-boot-artifacts-x86 - build-boot-artifacts-bfb @@ -1838,6 +1868,7 @@ jobs: runs-on: linux-amd64-cpu4 needs: - build-release-container-x86_64 + - build-machine-a-tron - build-forge-cli-x86_64 - build-release-artifacts-x86-host - build-release-artifacts-arm-host @@ -1868,6 +1899,7 @@ jobs: - build-artifacts-container-aarch64 - build-release-container-x86_64 - build-release-container-aarch64 + - build-machine-a-tron - test-release-container-services - build-boot-artifacts-x86 - build-boot-artifacts-bfb diff --git a/crates/api-core/src/auth/internal_rbac_rules.rs b/crates/api-core/src/auth/internal_rbac_rules.rs index f331e5d4ed..995e4f1f17 100644 --- a/crates/api-core/src/auth/internal_rbac_rules.rs +++ b/crates/api-core/src/auth/internal_rbac_rules.rs @@ -322,7 +322,10 @@ impl InternalRBACRules { x.perm("UpdateInstancePhoneHomeLastContact", vec![Agent]); x.perm("SetHostUefiPassword", vec![ForgeAdminCLI]); x.perm("ClearHostUefiPassword", vec![ForgeAdminCLI]); - x.perm("AddExpectedMachine", vec![ForgeAdminCLI, SiteAgent, Flow]); + x.perm( + "AddExpectedMachine", + vec![ForgeAdminCLI, SiteAgent, Flow, Machineatron], + ); x.perm("DeleteExpectedMachine", vec![ForgeAdminCLI, SiteAgent]); x.perm("UpdateExpectedMachine", vec![ForgeAdminCLI, SiteAgent]); x.perm("CreateExpectedMachines", vec![ForgeAdminCLI, SiteAgent]); diff --git a/crates/machine-a-tron/Dockerfile b/crates/machine-a-tron/Dockerfile new file mode 100644 index 0000000000..bfb77dd48d --- /dev/null +++ b/crates/machine-a-tron/Dockerfile @@ -0,0 +1,89 @@ +# syntax=docker/dockerfile:1.7 +# +# Cross-compilation: the BUILDER always runs NATIVE on the build host +# ($BUILDPLATFORM) and cross-compiles to x86_64-unknown-linux-gnu. This is not +# a Mac convenience: building the amd64 stage the naive way on an Apple +# Silicon host runs rustc under QEMU emulation, and QEMU SEGFAULTs assembling +# aws-lc-sys's hand-written x86-64 assembly (s2n-bignum) — even though the +# resulting binary runs fine on real x86_64. On an x86_64 build host the +# builder is simply native and the "cross" compile is a no-op. +# +# Build from the repo root: +# +# docker buildx build \ +# --platform linux/amd64 \ +# -f crates/machine-a-tron/Dockerfile \ +# --build-arg VERSION=0.1.0 \ +# -t machine-a-tron:latest \ +# . +# +ARG RUST_VERSION=1.96.0 + +FROM --platform=$BUILDPLATFORM rust:${RUST_VERSION}-slim-bookworm AS builder + +ARG VERSION +ARG CI_COMMIT_SHORT_SHA +ENV VERSION=${VERSION} +ENV CI_COMMIT_SHORT_SHA=${CI_COMMIT_SHORT_SHA} + +RUN apt-get update && apt-get install -y --no-install-recommends \ + git \ + pkg-config \ + protobuf-compiler \ + libprotobuf-dev \ + && if [ "$(uname -m)" != "x86_64" ]; then \ + apt-get install -y --no-install-recommends \ + gcc-x86-64-linux-gnu libc6-dev-amd64-cross; \ + else \ + apt-get install -y --no-install-recommends gcc libc6-dev; \ + fi \ + && rm -rf /var/lib/apt/lists/* + +RUN rustup target add x86_64-unknown-linux-gnu + +ENV CARGO_HOME=/cargo-home +ENV CARGO_NET_GIT_FETCH_WITH_CLI=true +ENV CARGO_TARGET_DIR=/cargo-target +ENV RUST_BACKTRACE=1 + +WORKDIR /workspace +COPY . . + +# Build release binary with cross-compilation +RUN --mount=type=cache,id=nico-mat-cross-cargo-home,target=/cargo-home,sharing=locked \ + --mount=type=cache,id=nico-mat-cross-cargo-target,target=/cargo-target,sharing=locked \ + if [ "$(uname -m)" != "x86_64" ]; then \ + export CARGO_TARGET_X86_64_UNKNOWN_LINUX_GNU_LINKER=x86_64-linux-gnu-gcc \ + CC_x86_64_unknown_linux_gnu=x86_64-linux-gnu-gcc; \ + fi && \ + cargo build -p carbide-machine-a-tron --bin machine-a-tron --locked --release \ + --target x86_64-unknown-linux-gnu && \ + mkdir -p /artifacts && \ + cp /cargo-target/x86_64-unknown-linux-gnu/release/machine-a-tron /artifacts/machine-a-tron + +# Runtime image +FROM --platform=linux/amd64 debian:bookworm-slim + +ARG VERSION +ARG CI_COMMIT_SHORT_SHA +LABEL org.opencontainers.image.version="${VERSION}" +LABEL org.opencontainers.image.revision="${CI_COMMIT_SHORT_SHA}" +LABEL org.opencontainers.image.title="machine-a-tron" +LABEL org.opencontainers.image.description="NICo Hardware Simulation Tool" +LABEL org.opencontainers.image.vendor="NVIDIA" + +RUN apt-get update && apt-get install -y --no-install-recommends \ + ca-certificates \ + iproute2 \ + iputils-ping \ + libssl3 \ + libudev1 \ + && rm -rf /var/lib/apt/lists/* + +RUN mkdir -p /opt/machine-a-tron/bin /opt/machine-a-tron/templates /tmp/machine-a-tron-data + +COPY --from=builder /artifacts/machine-a-tron /opt/machine-a-tron/bin/machine-a-tron +COPY crates/machine-a-tron/templates /opt/machine-a-tron/templates + +WORKDIR /opt/machine-a-tron +ENTRYPOINT ["/opt/machine-a-tron/bin/machine-a-tron"] diff --git a/dev/deployment/devspace/README.md b/dev/deployment/devspace/README.md index 46c184233b..626d1a014a 100644 --- a/dev/deployment/devspace/README.md +++ b/dev/deployment/devspace/README.md @@ -99,7 +99,7 @@ DevSpace will: - build the local runtime images from [`Dockerfile.api`](Dockerfile.api), [`Dockerfile.bmc-proxy`](Dockerfile.bmc-proxy), and [`Dockerfile.machine-a-tron`](Dockerfile.machine-a-tron) - build the REST API, workflow, site-manager, site-agent, database migration, certificate-manager, and MCP images from [`rest-api/docker/local`](../../../rest-api/docker/local) -- deploy the Helm chart in [`helm/`](../../../helm) +- deploy the Helm chart in [`helm/`](../../../helm) (including `nico-machine-a-tron`) - deploy the REST umbrella, site-agent, and MCP charts in [`helm/rest`](../../../helm/rest) - apply the local-only `machine-a-tron` Kubernetes objects from [`machine-a-tron.yaml`](machine-a-tron.yaml) with `kubectl` - inject the built image names and DevSpace-generated tags into both deployments at runtime @@ -109,11 +109,9 @@ The image builds are configured in [`devspace.yaml`](../../../devspace.yaml). Th The DevSpace images also use Dockerfile-specific ignore files: [`Dockerfile.api.dockerignore`](Dockerfile.api.dockerignore), [`Dockerfile.bmc-proxy.dockerignore`](Dockerfile.bmc-proxy.dockerignore), and [`Dockerfile.machine-a-tron.dockerignore`](Dockerfile.machine-a-tron.dockerignore). This keeps the top-level [`.dockerignore`](../../../.dockerignore) aligned with the main branch for CI and release builds, while still giving the local DevSpace builds a small Docker context. -DevSpace watches the Rust workspace, toolchain metadata, and the runtime Dockerfiles to decide when images need rebuilding. +DevSpace watches the Rust workspace, toolchain metadata, and the runtime Dockerfiles to decide when images need rebuilding. On kind clusters, the pre-deploy hooks load all Core and REST images into the cluster selected by the current kube context. -The REST images use the existing local Dockerfiles and one shared per-run tag because the REST Helm charts intentionally consume a common repository and tag. On kind clusters, the pre-deploy hooks load all Core and REST images into the cluster selected by the current kube context. - -The production Helm chart is still only responsible for the product services. `machine-a-tron` is deployed separately as plain local-only Kubernetes objects in [`machine-a-tron.yaml`](machine-a-tron.yaml), with DevSpace wiring in the local image tag and certificate issuer from [`devspace.yaml`](../../../devspace.yaml). The local API and BMC proxy configs in [`values.base.yaml`](values.base.yaml) point BMC traffic at `machine-a-tron-bmc-mock.nico-system.svc.cluster.local:1266`. +The `nico-machine-a-tron` Helm subchart configuration is in [`values.base.yaml`](values.base.yaml). The local API and BMC proxy configs point BMC traffic at `nico-machine-a-tron-bmc-mock.nico-system.svc.cluster.local:1266`. Common usage: @@ -163,7 +161,13 @@ docker build -t "nico-bmc-proxy:" -f dev/deployment/devs docker build -t "machine-a-tron:" -f dev/deployment/devspace/Dockerfile.machine-a-tron . ``` -DevSpace then deploys the Helm chart with the built `nico-api` image wired into `global.image.repository` and `global.image.tag`, the built `nico-bmc-proxy` image wired into the `nico-bmc-proxy` chart values, and applies the local-only `machine-a-tron` manifest with its image wired into the `Deployment` spec. The REST images are built from the existing `rest-api/docker/local` Dockerfiles and are passed to the three existing REST Helm charts with the same generated tag. +DevSpace then deploys the Helm chart with: +- the built `nico-api` image wired into `global.image.repository` and `global.image.tag` +- the built `nico-bmc-proxy` image wired into the `nico-bmc-proxy` chart values +- the built `machine-a-tron` image wired into the `nico-machine-a-tron` chart values +- certificate issuer settings from the DevSpace environment variables + +The REST images are built from the existing `rest-api/docker/local` Dockerfiles and are passed to the three existing REST Helm charts with the same generated tag. ## Resetting the local environment @@ -175,7 +179,7 @@ Reset the complete local environment by running: devspace purge -n nico-system ``` -When the current context is `kind-`, the purge pipeline deletes and recreates that kind cluster with the same node image, then bootstraps clean prerequisites. This removes all Kubernetes state, including the Core and REST databases, Temporal namespaces and history, Vault data, Keycloak data, certificates, site registration, Helm releases, CRDs, and persistent volumes. +When the current context is `kind-`, the purge pipeline deletes and recreates that kind cluster with the same node image, then bootstraps clean prerequisites. This removes all Kubernetes state, including the Core and REST databases, Temporal namespaces and history, Vault data, Keycloak data, certificates, site registration, Helm releases (including machine-a-tron), CRDs, and persistent volumes. On any other Kubernetes context, the pipeline delegates to DevSpace's default purge behavior. It removes the deployments managed by this project without replacing the cluster or reinstalling separately managed prerequisites. diff --git a/dev/deployment/devspace/machine-a-tron.yaml b/dev/deployment/devspace/machine-a-tron.yaml deleted file mode 100644 index d825bcd748..0000000000 --- a/dev/deployment/devspace/machine-a-tron.yaml +++ /dev/null @@ -1,140 +0,0 @@ -apiVersion: v1 -kind: ServiceAccount -metadata: - name: machine-a-tron ---- -apiVersion: v1 -kind: ConfigMap -metadata: - name: machine-a-tron-config -data: - mat.toml: | - carbide_api_url = "https://nico-api.nico-system.svc.cluster.local:1079" - interface = "NOTUSED" - tui_enabled = false - use_pxe_api = true - bmc_mock_host_tar = "/opt/machine-a-tron/dell_poweredge_r750.tar.gz" - bmc_mock_dpu_tar = "/opt/machine-a-tron/nvidia_dpu.tar.gz" - bmc_mock_port = 1266 - use_single_bmc_mock = true - mock_bmc_ssh_server = true - mock_bmc_ssh_port = 2222 - persist_dir = "/tmp/machine-a-tron-data" - register_expected_machines = true - - # machine-a-tron config generated from "just setup-k3s-env-ips" - [machines.config] - host_count = 10 - dpu_per_host_count = 2 - dpu_reboot_delay = 1 # in units of seconds - host_reboot_delay = 1 # in units of seconds - vpc_count = 0 - admin_dhcp_relay_address = "192.168.64.1" - oob_dhcp_relay_address = "192.168.192.1" - subnets_per_vpc = 0 - run_interval_working = "1s" - run_interval_idle = "10s" - template_dir = "/opt/machine-a-tron/templates" ---- -apiVersion: cert-manager.io/v1 -kind: Certificate -metadata: - name: machine-a-tron-certificate -spec: - duration: 720h0m0s - renewBefore: 360h0m0s - secretName: machine-a-tron-certificate - privateKey: - algorithm: ECDSA - size: 384 - issuerRef: - kind: Issuer - name: local-ca-issuer - group: cert-manager.io - dnsNames: - - machine-a-tron.nico-system.svc.cluster.local - - machine-a-tron-bmc-mock.nico-system.svc.cluster.local - uris: - - spiffe://nico.local/nico-system/sa/machine-a-tron ---- -apiVersion: v1 -kind: Service -metadata: - name: machine-a-tron-bmc-mock - labels: - app: machine-a-tron -spec: - selector: - app: machine-a-tron - ports: - - name: redfish - port: 1266 - targetPort: redfish - protocol: TCP - - name: ssh - port: 22 - targetPort: ssh - protocol: TCP ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - name: machine-a-tron - labels: - app: machine-a-tron -spec: - replicas: 1 - strategy: - type: Recreate - selector: - matchLabels: - app: machine-a-tron - template: - metadata: - labels: - app: machine-a-tron - spec: - serviceAccountName: machine-a-tron - containers: - - name: machine-a-tron - image: machine-a-tron - imagePullPolicy: IfNotPresent - command: - - /opt/machine-a-tron/bin/machine-a-tron - env: - - name: FORGE_ROOT_CA_PATH - value: /var/run/secrets/spiffe.io/ca.crt - - name: CLIENT_CERT_PATH - value: /var/run/secrets/spiffe.io/tls.crt - - name: CLIENT_KEY_PATH - value: /var/run/secrets/spiffe.io/tls.key - - name: MACHINE_A_TRON_CONFIG_PATH - value: /opt/machine-a-tron/config/mat.toml - ports: - - name: redfish - containerPort: 1266 - - name: ssh - containerPort: 2222 - readinessProbe: - tcpSocket: - port: redfish - initialDelaySeconds: 5 - periodSeconds: 10 - volumeMounts: - - name: spiffe - mountPath: /var/run/secrets/spiffe.io - readOnly: true - - name: config - mountPath: /opt/machine-a-tron/config - readOnly: true - - name: persist - mountPath: /tmp/machine-a-tron-data - volumes: - - name: spiffe - secret: - secretName: machine-a-tron-certificate - - name: config - configMap: - name: machine-a-tron-config - - name: persist - emptyDir: {} diff --git a/dev/deployment/devspace/values.base.yaml b/dev/deployment/devspace/values.base.yaml index cba862e532..d1839a90c7 100644 --- a/dev/deployment/devspace/values.base.yaml +++ b/dev/deployment/devspace/values.base.yaml @@ -1,4 +1,7 @@ nico-api: + # Disable legacy forge-system namespace alias for local dev + legacyAlias: + enabled: false resources: limits: cpu: "16" @@ -91,7 +94,7 @@ nico-api: enabled = true create_machines = true allow_changing_bmc_proxy = true - bmc_proxy = "machine-a-tron-bmc-mock.nico-system.svc.cluster.local:1266" + bmc_proxy = "nico-machine-a-tron-bmc-mock.nico-system.svc.cluster.local:1266" run_interval = "10s" machines_created_per_run = 1000 explorations_per_run = 1000 @@ -113,7 +116,7 @@ nico-bmc-proxy: metrics_endpoint = "[::]:1080" database_url = "postgres://replaced-by-env-var" allowed_principals = ["trusted-certificate"] - bmc_proxy = "machine-a-tron-bmc-mock.nico-system.svc.cluster.local:1266" + bmc_proxy = "nico-machine-a-tron-bmc-mock.nico-system.svc.cluster.local:1266" [tls] identity_pemfile_path = "/var/run/secrets/spiffe.io/tls.crt" @@ -139,6 +142,25 @@ nico-bmc-proxy: client_cert = "/var/run/secrets/spiffe.io/tls.crt" client_key = "/var/run/secrets/spiffe.io/tls.key" +nico-machine-a-tron: + enabled: true + # Deploy to nico-system (same namespace as nico-api) for simpler dev setup + # No need to copy secrets or configure cross-namespace access + machineATron: + nicoApiUrl: "https://nico-api.nico-system.svc.cluster.local:1079" + # SPIFFE URI must match nico-api's allowed service_base_paths + certificate: + uris: + - "spiffe://nico.local/nico-system/sa/machine-a-tron" + pods: + default: + machines: + rack-machines: + hwType: wiwynn_gb200_nvl + hostCount: 2 + dpuPerHostCount: 2 + oobDhcpRelayAddress: "192.168.192.1" + adminDhcpRelayAddress: "192.168.176.1" nico-dhcp: enabled: false nico-dns: diff --git a/devspace.yaml b/devspace.yaml index c0261daa28..aae7e68790 100644 --- a/devspace.yaml +++ b/devspace.yaml @@ -162,42 +162,24 @@ deployments: repository: ${runtime.images.nico-api.image} tag: ${runtime.images.nico-api.tag} pullPolicy: IfNotPresent + certificate: + issuerRef: + kind: ${LOCAL_DEV_CERT_ISSUER_KIND} + name: ${LOCAL_DEV_CERT_ISSUER_NAME} + group: ${LOCAL_DEV_CERT_ISSUER_GROUP} nico-bmc-proxy: image: repository: ${runtime.images.nico-bmc-proxy.image} tag: ${runtime.images.nico-bmc-proxy.tag} pullPolicy: IfNotPresent + nico-machine-a-tron: + image: + repository: ${runtime.images.machine-a-tron.image} + tag: ${runtime.images.machine-a-tron.tag} + pullPolicy: IfNotPresent valuesFiles: - dev/deployment/devspace/values.base.yaml - dev/deployment/devspace/values.generated.yaml - machine-a-tron-local: - namespace: ${LOCAL_DEV_NAMESPACE} - updateImageTags: true - kubectl: - manifests: - - dev/deployment/devspace/machine-a-tron.yaml - patches: - - target: - apiVersion: cert-manager.io/v1 - kind: Certificate - name: machine-a-tron-certificate - op: replace - path: spec.issuerRef.kind - value: ${LOCAL_DEV_CERT_ISSUER_KIND} - - target: - apiVersion: cert-manager.io/v1 - kind: Certificate - name: machine-a-tron-certificate - op: replace - path: spec.issuerRef.name - value: ${LOCAL_DEV_CERT_ISSUER_NAME} - - target: - apiVersion: cert-manager.io/v1 - kind: Certificate - name: machine-a-tron-certificate - op: replace - path: spec.issuerRef.group - value: ${LOCAL_DEV_CERT_ISSUER_GROUP} nico-rest: namespace: nico-rest updateImageTags: false diff --git a/docs/development/machine-a-tron-deployment.md b/docs/development/machine-a-tron-deployment.md new file mode 100644 index 0000000000..383b38157d --- /dev/null +++ b/docs/development/machine-a-tron-deployment.md @@ -0,0 +1,412 @@ +# machine-a-tron — Build & Deployment Guide + +machine-a-tron is a bare-metal simulator for NICo testing. It hosts mock DPUs and +servers via Redfish BMC, allowing end-to-end NICo flows without real hardware. This +guide documents everything needed to build the container image and deploy it on a +cluster. + +## Overview + +machine-a-tron runs in **Override Mode**: site-explorer redirects all Redfish traffic +to the mock BMC server running inside the pod. It is only suitable for +simulation-only clusters (no real hardware). + +## Quick path: `setup-machine-a-tron.sh` + +For a running NICo Core site, the fastest and most reliable way to deploy is the +end-to-end script, which performs every step in this guide (namespace, pull +secret, CA/Vault secret refresh, BMC credential seeding, `bmc_proxy` +configuration, DHCP-pool sizing, cert reissue, deploy, and verification) and is +idempotent: + +```bash +export KUBECONFIG=/path/to/kubeconfig +export REGISTRY_PULL_SECRET= # only if the pull secret is absent +helm-prereqs/setup-machine-a-tron.sh # add -y for non-interactive +``` + +## Quick start: a 4500-host simulated fleet + +Build and push the image to the registry of your choice, then run the setup +script in scale mode. Nothing registry-specific is committed — the image +location, tag, and pull credentials all come from the environment, exactly +like `setup.sh`: + +```bash +export KUBECONFIG=/path/to/site/kubeconfig +export NICO_IMAGE_REGISTRY=/ # e.g. registry.example.com/nico +export MAT_IMAGE_TAG= # tag you built and pushed (see §2) +export REGISTRY_PULL_SECRET= # omit if the pull secret already exists + +MAT_MODE=scale HOST_COUNT=4500 helm-prereqs/setup-machine-a-tron.sh -y +``` + +That is the whole procedure. The script exits after its bounded verification +window while ingestion continues in-cluster; progress is visible with: + +```bash +kubectl exec -n postgres -- su postgres -c \ + "psql -d nico_system_nico -tAc \"SELECT + (SELECT count(*) FROM explored_endpoints) || ' explored / ' || + (SELECT count(*) FROM machines) || ' machines';\"" +``` + +### What to expect (measured on a 3-node dev cluster, dev-sized postgres) + +4500 hosts × 2 DPUs = 13,500 BMC endpoints → 13,500 machines. + +| Phase | Duration | Notes | +|---|---|---| +| Deploy + DHCP registration | ~60–90 min | ~150–180 interfaces/min while 13.5k mock FSMs boot | +| Exploration sweep | ~3–5 h | overlaps DHCP; `explorations_per_run=120` per cycle | +| Preingestion | tracks the sweep | ~90% conversion, completes shortly after it | +| Identification + creation | final ~2–3 h | hosts identify in waves; creation drains at ~150–300 machines per explore cycle | +| **End to end** | **~6–9 h, unattended** | 100 hosts ≈ 12 min and 1000 hosts ≈ 25 min, for calibration | + +The pipeline is autonomous once the script completes — it has run through +multi-hour client connectivity outages without intervention. Occasional +nico-api restarts under peak ingestion load are absorbed by the pipeline +(machines resume within a cycle). Re-running the script is always safe +(idempotent) and re-registers any expected machines that arrived late. + +The rest of this document explains what that script does and why, and is the +reference for manual deployment or debugging. The script's header comments +enumerate the non-obvious failure modes it guards against. + +## 1. Prerequisites + +- Docker with `buildx` and a `linux/arm64` builder available (see below) +- A container registry you can push to and the cluster can pull from + (referenced via `NICO_IMAGE_REGISTRY`, same convention as `setup.sh`) +- A cluster where machine-a-tron can reach `nico-api.nico-system.svc.cluster.local:1079` +- The `nico-machine-a-tron` Helm chart (`helm/charts/nico-machine-a-tron`) +- The image pull secret `machine-a-tron-pull` in the `nico-mat` namespace + (created automatically by the setup script from `REGISTRY_PULL_SECRET`) + +## 2. Building the Container Image + +### Why cross-compilation is required + +machine-a-tron must run on **x86_64** cluster nodes. The Rust dependency `aws-lc-sys` +contains hand-written x86_64 assembly (`s2n-bignum`). Compiling under QEMU emulation +causes a SIGSEGV in the assembler (`bignum_madd_n25519.S`). We therefore use true +cross-compilation: a native `linux/arm64` Rust compiler targeting +`x86_64-unknown-linux-gnu`. + +The `carbide-rpc` crate runs a protobuf build script that requires both `protoc` +and the protobuf well-known types (`libprotobuf-dev` on Debian). `libredfish` is a +Git dependency so `git` must also be present in the build stage. + +### Build command + +Run from the **repository root**: + +```bash +# Same convention as setup.sh: your registry/repository prefix, no scheme. +REGISTRY=${NICO_IMAGE_REGISTRY:?export NICO_IMAGE_REGISTRY=/} +COMMIT=$(git rev-parse --short HEAD) +TAG="${COMMIT}-amd64" + +docker buildx build \ + --platform linux/amd64 \ + -f crates/machine-a-tron/Dockerfile \ + --push \ + -t "${REGISTRY}/machine-a-tron:${TAG}" \ + . +``` + +> **Note**: `--load` does not work for cross-platform builds — use `--push` directly. +> The build takes ~4–5 minutes on a cold cache; subsequent builds with warm cache are +> under 30 seconds. + +### Registry authentication + +```bash +docker login "${NICO_IMAGE_REGISTRY%%/*}" \ + -u "${REGISTRY_PULL_USERNAME:-\$oauthtoken}" \ + -p "${REGISTRY_PULL_SECRET}" +``` + +Some registries use a fixed username with API-key auth — set `REGISTRY_PULL_USERNAME` accordingly (default: `$oauthtoken`). + +## 3. Cluster Prerequisites + +### Namespaces and secrets + +```bash +# Create the deployment namespace +kubectl create namespace nico-mat --dry-run=client -o yaml | kubectl apply -f - + +# Image pull secret (replace with your NVIDIA API key) +kubectl create secret docker-registry machine-a-tron-pull \ + -n nico-mat \ + --docker-server="${NICO_IMAGE_REGISTRY%%/*}" \ + --docker-username="${REGISTRY_PULL_USERNAME:-\$oauthtoken}" \ + --docker-password="${REGISTRY_PULL_SECRET}" \ + --dry-run=client -o yaml | kubectl apply -f - + +# Copy nico-roots CA secret from nico-system +kubectl get secret nico-roots -n nico-system -o json \ + | jq 'del(.metadata.namespace,.metadata.resourceVersion,.metadata.uid,.metadata.creationTimestamp)' \ + | kubectl apply -n nico-mat -f - +``` + +### Vault secrets + +machine-a-tron's service account needs the Vault secrets present in its namespace +(even though they are optional — their absence produces confusing log noise): + +```bash +for secret in nico-vault-approle-tokens nico-vault-token vault-cluster-info; do + kubectl get secret "$secret" -n nico-system -o json \ + | jq 'del(.metadata.namespace,.metadata.resourceVersion,.metadata.uid,.metadata.creationTimestamp)' \ + | kubectl apply -n nico-mat -f - +done +``` + +> **Critical after a site reprovision:** re-copy `nico-roots` from `nico-system` +> as well. A reprovision recreates the nico-system CA; a machine-a-tron carried +> over from before will trust the *old* CA (stale `nico-roots`) and present a +> client cert signed by the old CA, so **every** mTLS call to nico-api fails +> with `client error (Connect)`. Re-copy the CA and delete the old cert secret +> (`nico-machine-a-tron-certificate`) so cert-manager reissues from the current +> CA: +> ```bash +> kubectl get secret nico-roots -n nico-system -o json \ +> | jq 'del(.metadata.namespace,.metadata.resourceVersion,.metadata.uid,.metadata.creationTimestamp,.metadata.ownerReferences)' \ +> | kubectl apply -n nico-mat -f - +> kubectl delete secret nico-machine-a-tron-certificate -n nico-mat --ignore-not-found +> ``` + +### BMC credentials in Vault (required for site-explorer) + +site-explorer's `check_preconditions` requires three site-default Vault +credentials before it will explore any endpoint: +`machines/bmc/site/root`, `machines/all_dpus/site_default/uefi-metadata-items/auth`, +and `machines/all_hosts/site_default/uefi-metadata-items/auth`. The two UEFI +paths are created by the nico-prereqs `kvSeeds` but with **empty passwords**, +which fails the check (`vault does not have a valid password entry`) — they +must be re-seeded with any non-empty password. `machines/bmc/site/root` is not +seeded at all; without it every run aborts with `MissingCredentials`. + +Beyond the preconditions, the **credential rotation flow** requires this exact +chain (all handled by `setup-machine-a-tron.sh` Phase 4): + +| Vault path | Value | Why | +|---|---|---| +| `machines/all_hosts/factory_default/bmc-metadata-items/dell` | `root`/`factory_password` | Host BMC factory default (mock's `DUMMY_FACTORY_PASSWORD`). Path segment is **lowercase** `dell` — `BMCVendor`'s `Display` impl lowercases. | +| `machines/all_dpus/factory_default/bmc-metadata-items/root` | `root`/`0penBmc` | DPU BMC factory default (mock's `DUMMY_FACTORY_DPU_PASSWORD`) — note it differs from the host factory password. | +| `machines/bmc/site/root` | `root`/<distinct> | Rotation target. **Must differ from both factory passwords**, or the rotation is a no-op and the mock rejects with `403 Factory-default password must be changed` forever. | + +site-explorer logs into each BMC with its factory default, rotates the password +to the site root value, then proceeds — using the wrong factory password (or a +site root equal to factory) yields `401 Unauthorized`, which latches a +self-perpetuating `AvoidLockout` (NICO-SITEEXPLORER-144) until an operator +clears it (`nico-admin-cli site-explorer refresh `). + +## 4. Deploying the Helm Chart + +### Site values file + +Copy `helm-prereqs/values/machine-a-tron.yaml` and fill in the site-specific values: + +| Field | Description | +|---|---| +| `image.tag` | Tag produced by step 2 (e.g. `8c35783af-amd64`) | +| `machines.dell-hosts.oobDhcpRelayAddress` | Gateway of the OOB/underlay network from nico-core site config | +| `machines.dell-hosts.adminDhcpRelayAddress` | Gateway of the admin network from nico-core site config | +| `machines.dell-hosts.hostCount` | Must not exceed available OOB DHCP addresses (`hostCount + hostCount×dpuPerHostCount`) | + +### Critical: SPIFFE URI override + +The cert-manager `Certificate` resource auto-generates a SPIFFE URI based on the +deployment namespace: `spiffe://nico.local/nico-mat/sa/nico-machine-a-tron`. + +nico-api's `spiffe_service_base_paths` only includes `/nico-system/sa/` (and two +others), so this URI is **not recognized**. The result is that machine-a-tron's +principal is only `TrustedCertificate` — not `SpiffeServiceIdentifier("machine-a-tron")` +— and every gRPC call beyond `Version` returns HTTP 403. + +The values file already includes the fix: + +```yaml +certificate: + uris: + - "spiffe://nico.local/nico-system/sa/machine-a-tron" +``` + +This overrides the auto-generated URI so nico-api can correctly identify and authorize +machine-a-tron as the `Machineatron` RBAC principal. + +### Deploy + +```bash +helm upgrade --install nico-machine-a-tron \ + helm/charts/nico-machine-a-tron \ + -n nico-mat \ + --create-namespace \ + -f helm-prereqs/values/machine-a-tron.yaml +``` + +## 5. Configuring Override Mode + +Configure nico-core's site-explorer to redirect all Redfish traffic to the mock. +Add this to the nico-core site config under `[site_explorer]`: + +```toml +[site_explorer] +bmc_proxy = "nico-machine-a-tron-bmc-mock.nico-mat.svc.cluster.local:1266" +``` + +> **Use the cross-namespace FQDN.** site-explorer runs inside nico-api in +> `nico-system`; a bare service name resolves against that namespace and fails +> ("connection refused" on every Redfish call) because the mock's Service lives +> in `nico-mat`. +> +> **This setting does not survive a nico-core `helm upgrade`** — the ConfigMap +> is chart-owned, so an upgrade silently reverts it. Re-run +> `setup-machine-a-tron.sh` (idempotent) after any nico-core upgrade. + +> **Field name matters.** The config field is `bmc_proxy` — a single +> `"host:port"` string (`crates/site-explorer/src/config.rs`). The older +> `override_target_ip` / `override_target_port` fields are **deprecated**, and +> `override_target_host` was never a valid field at all (earlier revisions of +> this guide were wrong — a value under that key is silently ignored). + +Setting `bmc_proxy` at launch also makes `allow_changing_bmc_proxy` default to +`true`. That is what allows the chart value `machineATron.configureBmcProxyHost` +to work: when set, machine-a-tron calls nico-api's `set_dynamic_config` to set +`bmc_proxy` at runtime, but that call is rejected with `PermissionDenied` unless +`allow_changing_bmc_proxy` is true. The two mechanisms are complementary — the +values file ships `configureBmcProxyHost: +"nico-machine-a-tron-bmc-mock.nico-mat.svc.cluster.local"` (FQDN, same +cross-namespace requirement), and the nico-core `bmc_proxy` setting both +enables that path and covers the case where the runtime call has not happened +yet. + +Apply via `helm upgrade` of the nico-core chart, or patch the configmap and +restart nico-api (site-explorer runs in-process in nico-api): + +```bash +kubectl rollout restart deployment/nico-api -n nico-system +``` + +### Why machines get created (expected_machines) + +site-explorer's `MachineCreator` refuses to create a managed host unless a +matching `expected_machines` row exists (by BMC MAC) — otherwise it logs +`Refusing to create managed host, expected machines entry not found`. machine-a-tron +auto-registers these when `machineATron.registerExpectedMachines: true` (the +default in the values file). DHCP discovery alone is **not** sufficient. + +## 5b. Multi-pod simulation with per-BMC ClusterIP services (`bmcServices`) + +The chart can shard the simulated fleet across several machine-a-tron pods, +each with a dedicated ClusterIP range where every simulated BMC gets its own +Service (`pods..cidr` + `bmcServices.enabled: true`). NICo then dials +each BMC IP directly — no `bmc_proxy`. A validated two-pod example lives at +`helm-prereqs/values/machine-a-tron-multipod.yaml` (2 pods × 5 +`wiwynn_gb200_nvl` hosts × 2 DPUs → 30 machines, 15 per pod). + +Everything single-pod mode needs still applies (namespaces, CA copy, Vault +seeds, SPIFFE URI). Multi-pod adds five requirements, all hit in practice: + +1. **Kubernetes 1.29+** for the `ServiceCIDR` object. On older clusters set + `bmcServices.serviceCIDR.create: false` and pick pod CIDRs **inside** the + apiserver's `--service-cluster-ip-range` — a static `clusterIP` outside + that range is rejected ("the provided IP is not in the valid range"). + Check the chosen sub-ranges are free of existing ClusterIPs first. +2. **Image with the `Host`-header routing fallback (PR #3190).** Without a + `bmc_proxy` the Redfish client sends no `Forwarded` header; older bmc-mock + builds then 404 every request with `no router configured`. Leave + `site_explorer.bmc_proxy` unset in this mode. +3. **Disjoint MAC pools per pod.** All machine-a-tron instances derive MACs + from the same default pools, so the pod that leases second is rejected on + every DHCP with `Network segment mismatch for existing MAC address` and + simulates nothing. Until the chart grows a per-pod MAC knob, supply full + per-pod TOML overrides via `configFiles.matConfigs.` setting distinct + `mac_address_pool` / `hw_mac_address_ranges` bases (see the example + values file). +4. **One NICo network segment per pod CIDR.** `network_prefixes` allows one + IPv4 prefix per segment, so each pod CIDR needs its own cloned underlay + segment (same technique as the scale-mode segment fallback), gateway `.1`, + `num_reserved 1` — BMC Service IPs start at `.2`, matching the DHCP + allocator. +5. **Hardware-type specifics.** Vendors libredfish does not recognize (e.g. + `wiwynn_gb200_nvl` reports `WIWYNN`) resolve to `unknown`, so seed the + host factory credential at + `machines/all_hosts/factory_default/bmc-metadata-items/unknown`. GB-class + hosts (`NvidiaGBx00` flow) additionally require `expected_machines` rows + with a working BMC credential before exploration completes + (`NICO-SITEEXPLORER-141 Missing credential expected_machine`); on nico-api + builds without the Machineatron `AddExpectedMachine` RBAC grant the + auto-registration is 403'd and the rows must be inserted directly with the + **pinned** password (`hostBmcPassword`), not the factory default. DPU BMCs + explore without expected rows. + +## 6. Verifying Startup + +Check that machine-a-tron passes the initial API calls: + +```bash +kubectl logs -n nico-mat deployment/nico-machine-a-tron | grep -E "firmware|Got desired|Error:" +``` + +Expected: `Got desired firmware versions from the server: [...]` + +Check nico-api for denied requests (should be empty after the SPIFFE fix): + +```bash +kubectl logs -n nico-system deployment/nico-api | grep "Request denied.*machine-a-tron" +``` + +## 7. DHCP Address Space + +machine-a-tron allocates one OOB IP per BMC interface (1 per host + 1 per DPU). With +5 hosts and 2 DPUs each that is **15 IPs** (`5 + 5×2`). Ensure the OOB DHCP prefix in +nico-core is large enough. Note the usable count is smaller than the raw CIDR: a +`/28` (16 addresses) yields only ~10–13 usable after subtracting network, +broadcast, gateway, and any reserved addresses — on dev6 a `/28` yielded 10, so +`hostCount: 3 × 2 DPUs = 9` fit but `5 × 2 = 15` did not. Use at least a `/27` +for the default counts, or reduce `hostCount` / `dpuPerHostCount`. Symptom of +overflow: `No IP addresses left in prefix ...` and machines stuck in `BmcInit`. + +If the prefix is exhausted from previous runs, either: + +- Force-delete old machine records via the admin CLI, or +- Reprovision the site (which clears the database) and redeploy from scratch. + +> **Do NOT hand-delete rows** from `machine_interfaces`, `dhcp_entries`, or +> `machine_interface_addresses` to free leases. The `machine_dhcp_records` view +> inner-joins the singleton control row `machine_interfaces_deletion` (id=1); if +> that row is deleted (easy to do by accident when clearing related tables) the +> view returns zero rows and `DiscoverDhcp` fails for **every** BMC with +> `Database Error: no rows returned by a query that expected to return at least +> one row`. If you hit that, restore the row: +> ```sql +> INSERT INTO machine_interfaces_deletion (id) VALUES (1) ON CONFLICT (id) DO NOTHING; +> ``` + +## 8. Summary of Non-Obvious Fixes Discovered + +| Problem | Root cause | Fix | +|---|---|---| +| `exec format error` in pod | Image was built for `arm64`, nodes are `x86_64` | Cross-compile with `--platform linux/amd64` and `x86_64-unknown-linux-gnu` Rust target | +| `SIGSEGV` compiling `aws-lc-sys` | QEMU emulates the `.S` assembler, which crashes | True cross-compilation (native arm64 host → x86_64 target) instead of QEMU | +| `File not found: google/protobuf/timestamp.proto` | `libprotobuf-dev` absent in build image | Add `libprotobuf-dev` to `apt-get install` in builder stage | +| `git fetch ... (exit status: 127)` | `libredfish` is a git dependency, `git` not in slim image | Add `git` to builder stage | +| `--load` fails for cross-platform builds | Docker limitation | Use `--push` directly to registry | +| HTTP 403 on every gRPC call | machine-a-tron cert SPIFFE URI not in nico-api's `service_base_paths` | Set `certificate.uris: ["spiffe://nico.local/nico-system/sa/machine-a-tron"]` in values | +| `client error (Connect)` on every nico-api call after a reprovision | Stale `nico-roots` CA + client cert signed by the old CA | Re-copy `nico-roots` from nico-system; delete `nico-machine-a-tron-certificate` so cert-manager reissues from the current CA | +| site-explorer aborts with `MissingCredentials machines/bmc/site/root` | Site BMC root cred not in default `kvSeeds` | Seed `secrets/machines/bmc/site/root` = `root`/<non-factory password> in Vault | +| site-explorer aborts with `MissingCredentials .../uefi-metadata-items/auth` | kvSeeds create the UEFI creds with **empty** passwords, which fail validation | Re-seed both site_default UEFI creds with any non-empty password | +| Redfish redirect ignored; `endpoint_explorations=0` | Wrong config field (`override_target_host` is not real) | Use `bmc_proxy = "nico-machine-a-tron-bmc-mock.nico-mat.svc.cluster.local:1266"` under `[site_explorer]` | +| Redfish `connection refused` on every endpoint despite bmc_proxy set | Bare service name resolves against nico-system, not nico-mat | Use the cross-namespace FQDN in `bmc_proxy` | +| Host BMCs 401 while DPUs explore fine | Host and DPU factory passwords differ (`factory_password` vs `0penBmc`); host factory cred missing or wrong | Seed `machines/all_hosts/factory_default/bmc-metadata-items/dell` = `root`/`factory_password` (lowercase `dell`) | +| DPU explorations stuck at `403 Factory-default password must be changed` | Site root password equals the factory password → rotation is a no-op | Seed `machines/bmc/site/root` with a password distinct from both factory defaults | +| All endpoints latch `AvoidLockout` (NICO-SITEEXPLORER-144) after a cred fix | A previous Unauthorized is self-perpetuating in the exploration report | `nico-admin-cli site-explorer refresh ` per endpoint (or re-run setup-machine-a-tron.sh, which clears it) | +| `Refusing to create managed host`; machine-a-tron logs `PermissionDenied` on registration | nico-api build lacks the `Machineatron` → `AddExpectedMachine` RBAC grant | Rebuild nico-api with the grant (internal_rbac_rules.rs); the setup script also has a DB fallback | +| Machine creation fails `No IP addresses left in prefix ` | Admin pool too small: creation needs one host-PF IP per DPU | Fit `hostCount×dpuPerHostCount` ≤ usable admin-pool IPs (the script auto-fits) | +| `Refusing to create managed host, expected machines entry not found` | No `expected_machines` row for the discovered BMC MAC | Set `machineATron.registerExpectedMachines: true` (default) so machine-a-tron auto-registers them | +| `No IP addresses left in prefix ...`; machines stuck in `BmcInit` | OOB DHCP pool too small for host×DPU count | Sizing: `hostCount + hostCount×dpuPerHostCount` ≤ usable pool IPs; use ≥ /27 or reduce counts | +| `DiscoverDhcp`: `no rows ... expected to return at least one row` | `machine_interfaces_deletion` singleton (id=1) deleted; breaks `machine_dhcp_records` view | `INSERT INTO machine_interfaces_deletion (id) VALUES (1) ON CONFLICT DO NOTHING;` — never hand-delete lease rows | diff --git a/docs/development/machine-a-tron-scale-testing.md b/docs/development/machine-a-tron-scale-testing.md new file mode 100644 index 0000000000..0088a9c561 --- /dev/null +++ b/docs/development/machine-a-tron-scale-testing.md @@ -0,0 +1,168 @@ +# Scaling NICo with machine-a-tron: 100 → 1000 → 4500 simulated hosts + +> **Status: DRAFT — early feedback wanted.** Stage 1 (100 hosts × 2 DPUs = +> 300 BMCs → 264 machines) is complete on dev6. Stage 2 (1000 hosts) is +> running as this is written. Stage 3 targets 4500 hosts × 2 DPUs = 13,500 +> BMC endpoints, in support of scaling NICo to ~4500 nodes. +> +> **Rebased on #2764**: this work now sits on top of Alexander's ClusterIP +> migration, which removes nginx/MetalLB from the chart entirely (per-BMC +> ClusterIP Services + ServiceCIDR, multi-pod sharding via `pods..cidr`). +> That migration supersedes issues 10, 12 and 13 below (kept for the record — +> they document why the nginx/MetalLB path was abandoned) and independently +> confirms the direction of the proxy-direct pivot. The scripts' scale mode +> (`bmc_proxy` + client-injected Forwarded) remains valid with `bmcServices` +> disabled and is what all stage results below were measured with. +> +> Branch: `machine-a-tron-e2e-on-2764`. Everything below is reproducible +> with two commands: +> +> ```bash +> export KUBECONFIG=/path/to/site/kubeconfig +> helm-prereqs/cleanup-machine-a-tron.sh -y +> MAT_MODE=scale HOST_COUNT=1000 helm-prereqs/setup-machine-a-tron.sh -y +> ``` + +## What this work delivers + +1. **`helm-prereqs/setup-machine-a-tron.sh`** — one idempotent script that + takes a running NICo site from nothing to created machines: namespace, + pull secret, CA/Vault secret refresh, the full BMC/UEFI credential chain, + nico-core site-config changes, DHCP pool sizing with auto-fit, DB safety + checks, helm deploy, and a verification loop that actively shepherds the + ingestion pipeline (details below on why that is necessary). +2. **`helm-prereqs/cleanup-machine-a-tron.sh`** — the full inverse, so + from-scratch runs are reproducible (this caught several + "works-second-time-only" bugs). +3. **`MAT_MODE=scale`** — a scale profile + (`helm-prereqs/values/machine-a-tron-scale.yaml`) using a **proxy-direct** + transport architecture (see next section), simulated network segments + sized for 13.5k endpoints, and raised site-explorer throughput knobs. +4. A one-line RBAC fix in nico-api (`Machineatron` was missing the + `AddExpectedMachine` grant) plus chart fixes to the nginx/MetalLB mode. + +## The architecture decision: proxy-direct + +The chart offers an nginx/MetalLB mode for large scale: one LoadBalancer +Service per simulated BMC (cap 16,384), nginx terminating TLS and routing to +the mock. We started there and hit four independent failure modes at just 300 +endpoints (§ issues 10–13). The pivotal realization: + +**`site_explorer.bmc_proxy` alone already scales.** When it is set, the +Redfish client itself injects `Forwarded: host=` (RFC 7239, +`crates/redfish/src/libredfish/implementation.rs`), and the mock's shared +registry (`use_single_bmc_mock = true`) routes each request to the right +simulated BMC. One ClusterIP Service carries the whole fleet — no nginx, no +MetalLB pool, no per-BMC Services, no `externalTrafficPolicy: Local` +pitfalls. + +The nginx/MetalLB mode remains the right choice when simulated BMCs must +coexist with **real hardware** (each mock needs a real routable IP). For a +simulation-only cluster it only adds moving parts. The chart fixes we made to +that mode are kept for its real users. + +Result at 300 endpoints: exploration went from constant flapping +(Unreachable/ConnectionRefused under load) to rock-stable 300/300. + +## Complete issue log + +Every issue below was found live on dev6 and is fixed on the branch, encoded +in the scripts/charts with explanatory comments. + +### Baseline (override-mode) end-to-end + +| # | Issue | Root cause | Fix | +|---|---|---|---| +| 1 | Every nico-api call fails `client error (Connect)` after a site reprovision | machine-a-tron trusts the old CA (stale `nico-roots` copy) and presents a cert signed by it | Script refreshes `nico-roots` + Vault secrets from nico-system and deletes the client-cert secret so cert-manager reissues from the current CA | +| 2 | Redfish redirect silently ignored | Docs said `override_target_host` — never a valid field; the real field is `bmc_proxy = "host:port"`, and it must be the **cross-namespace FQDN** (site-explorer runs in nico-system; a bare service name doesn't resolve) | Script sets `bmc_proxy` correctly; docs fixed | +| 3 | site-explorer aborts every run: `MissingCredentials` | `machines/bmc/site/root` isn't in default kvSeeds; the seeded UEFI creds ship with **empty** passwords which fail validation | Script seeds the full chain | +| 4 | Host BMCs 401 while DPUs explore fine | Host and DPU mock factory passwords differ (`factory_password` vs `0penBmc`); the host factory Vault path vendor segment is **lowercase** (`…/dell` — `BMCVendor`'s `Display` lowercases; the earlier capital-`Dell` seed was read by nobody) | Script seeds both factory creds on the correct paths | +| 5 | machine-a-tron's expected-machine registration 403s (logged misleadingly as "likely already ingested") | `Machineatron` principal missing from the `AddExpectedMachine` RBAC grant — an oversight; it holds the sibling grants (`DiscoverDhcp`, `CreateNetworkSegment`, `GetExpectedSwitch`) | One-line fix in `internal_rbac_rules.rs`; script includes a DB fallback for nico-api builds without it | +| 6 | Endpoints permanently stuck `AvoidLockout` (NICO-SITEEXPLORER-144) on a fresh deploy | Per-MAC rotated creds (`machines/bmc//root`) survive cleanup; a fresh mock is at factory password but the per-MAC entry makes site-explorer present the old rotated one → 401 latch, self-perpetuating by design | Cleanup purges per-MAC creds; setup self-heals stale ones (only when the machine graph is truly empty — machines AND interfaces at 0) | +| 7 | `DiscoverDhcp` fails for every BMC: "no rows returned…" | The `machine_dhcp_records` VIEW inner-joins a singleton control row (`machine_interfaces_deletion` id=1); manual lease cleanup had deleted it | Script restores the singleton; documented: never hand-delete lease rows | +| 8 | Machines never created: admin pool exhausted | Real demand is OOB = `hosts×(1+dpus)` and admin = `hosts×(dpus+1)` (one host-PF IP per DPU **plus one per host at creation**); usable = `2^(32-mask) − reserve_first − 1` | Sizing check with auto-fit; `reserve_first` parsed from the live site config | + +### Scale mode (100 hosts × 2 DPUs and up) + +| # | Issue | Root cause | Fix | +|---|---|---|---| +| 9 | helm deploy aborts: hundreds of `connection reset by peer` | helm's default burst (100 concurrent API calls) overwhelms SOCKS/ssh tunnels when creating hundreds of Services | `--qps 15 --burst-limit 30` (env-overridable) | +| 10 | nginx bmc-proxy CrashLoopBackOff: `host not found in upstream` | Chart template pointed the upstream at the bare chart name, which is not a Service | Point at the `-bmc-mock` Service (chart fix) | +| 11 | Nothing listens on the mock port; probes kill the pod | `use_single_bmc_mock=false` makes each mock bind its **real BMC IP** on the pod netns (bare-metal mode). `true` is the shared-registry mode K8s needs | `useSingleBmcMock: true` | +| 12 | Every registry lookup 404s: `no router configured for host: 10.233.x.x` | nginx forwarded `host=$server_addr`, but kube-proxy DNATs the LB IP to the nginx **pod IP** before the connection arrives | `Forwarded "host=$host"` — the client-requested host is the LB IP end-to-end (chart fix) | +| 13 | LB IPs intermittently Unreachable in-cluster | Per-BMC Services use `externalTrafficPolicy: Local` and the chart's REQUIRED podAffinity stacked all proxies on the mat node | Required anti-affinity between proxy replicas (+ `maxUnavailable=1/maxSurge=0`; with replicas == nodes a surge pod deadlocks the rollout) — chart fix, kept for nginx-mode users | +| 14 | DHCP fails: `No network segment defined for relay addresses` | Config-driven segment creation is **bootstrap-once** — skipped entirely on multi-domain sites ("Multiple domains, skipping initial network creation") | Script clone-inserts the simulated segments from same-type templates; `allocation_strategy` forced to `dynamic` (templates may be `reserved`, which rejects all dynamic DHCP) | +| 15 | AvoidLockout storm on all DPU endpoints; preingestion pinned at exactly `hostCount` | The rotation dance is racy at scale: preingestion's initial BMC reset reboots the mock, which returns at the **factory** password while its per-MAC Vault entry says "rotated" | Pin mock passwords to the site root (`hostBmcPassword`/`dpuBmcPassword`) — site-explorer's documented fallback ("factory failed → sitewide root, no rotation") logs straight in; resets become harmless | +| 16 | Pipeline stalls at preingestion `initial`; manager idle | `waiting_for_explorer_refresh` (set when errors are cleared) gates endpoints out of preingestion and can linger after a healthy report lands (273/300 were parked) | Verification loop unparks endpoints whose reports are clean | +| 17 | Managed hosts identified but machines never created; cycles never finish | `explorations_per_run` was raised to 400 "for throughput" — but identification and creation only run **at the end of a completed explore cycle**, and 400 deep scans per cycle meant cycles stopped completing | Default lowered to 120: cycles complete in ~1–2 min and creation runs every cycle | +| 18 | `Resource pool lo-ip is empty` on the 3rd machine | Machine creation allocates one loopback IP per machine; pool **definitions are seed-once** ("Declaration has drifted since seed … not re-applying") so config widening is ignored; dev6 ships **3** lo-ip addresses | Script inserts free `resource_pool` rows directly for a simulated range (16k) when the pool is smaller than the machine target | + +### A note on the verification loop + +The script's final phase doesn't just poll — it actively shepherds: +re-clears `AvoidLockout`/`Unauthorized` latches (they are one-way by design; +on real hardware an operator runs `nico-admin-cli site-explorer refresh`) and +unparks healthy endpoints. On a simulation cluster with hundreds of +concurrent resets/explorations, transient races are guaranteed; the loop is +the "operator". Mocks have no lockout threshold, so this is safe here. + +## Where we are today + +| Stage | Scale | Result | +|---|---|---| +| Baseline | 1 host × 1 DPU (override mode) | ✅ end-to-end: machines created, full credential rotation exercised | +| Stage 1 | 100 hosts × 2 DPUs = 300 BMCs (proxy-direct) | ✅ 300/300 endpoints stable, machines created and advancing through `hostinit`/`dpuinit` | +| Stage 2 | 1000 hosts × 2 DPUs = 3000 BMCs | ✅ **END TO END OK — 3000/3000 machines** in a single unattended script run (~25 min total; creation ≈ 240 machines/min) | +| Stage 3 | 4500 hosts × 2 DPUs = 13,500 BMCs | ✅ **13,500/13,500 machines — 100% fleet** (first run: 13,500 endpoints explored, 10k+ machines; consolidated rerun on the #2764 ClusterIP chart + a freshly provisioned cluster: every counter at 100% — 13,500 explored / 13,500 preingestion-complete / 4,500 hosts / 13,500 machines, ~15 h wall clock unattended incl. connectivity outages) | + +Stage-3 observations worth reviewers' attention: + +- **The ingestion pipeline is fully autonomous once configured.** Client + connectivity to the cluster dropped twice for extended periods during + stage 3; ingestion continued unattended both times (e.g. +720 machines + through one outage, +4,000 through another). The shepherd loop's latch + clearing — critical in earlier iterations — was a no-op for the entire + stage-3 run thanks to pinned credentials. +- **Measured stage-3 rates on dev6 (3 nodes, dev-sized postgres):** DHCP + ~110 interfaces/min; exploration ~120–360 endpoints/cycle; creation + 40–240 machines per completed explore cycle, sawtoothing with cycle + phasing (identification rebuilds `explored_managed_hosts` each cycle). +- **Per-MAC Vault credential lifecycle needs batching at scale** (issue 19 + below): site-explorer stores one `machines/bmc//root` entry per BMC — + 13,500 entries; deleting them one API round-trip at a time takes hours, + batched server-side it takes seconds. +- `expected_machines` auto-registration worked at stage 3 (9,890+ registered + by machine-a-tron via the API), confirming the RBAC grant path. + +Additional issue found at stage 3: + +| # | Issue | Root cause | Fix | +|---|---|---|---| +| 19 | Stage-2→3 cleanup ran for over an hour "deleting credentials" | One `kubectl exec` per per-MAC Vault deletion × thousands of entries | Batch the deletion loop server-side on the vault pod — one exec total (both cleanup and setup self-heal) | + +## Open questions — feedback wanted + +1. **RBAC**: is granting `Machineatron` → `AddExpectedMachine` acceptable + (commit `9a9ba072a`)? Until a nico-api image with it is deployed, the + script registers expected machines via direct DB insert — okay as a + documented simulation-only fallback? +2. **Seed-once reconcile semantics**: networks, and resource-pool + definitions are all create-once; config changes on established sites are + silently ignored (or warn-only). The script works around this with direct + DB writes (segment clone-insert, pool row insertion). Should NICo support + declarative updates for these instead? +3. **AvoidLockout at scale**: one-way latches are right for real BMCs, but + simulation fleets guarantee latch storms during resets. Worth a + site-config escape hatch (e.g. `site_explorer.lockout_protection = false`) + instead of the script's DB-level clearing? +4. **Mock fidelity**: the mock returns to its configured password after a + BMC reset. Real BMCs persist a rotated password across resets. Should + bmc-mock persist rotated credentials so the rotation path can be exercised + at scale without pinning? +5. **lo-ip per machine**: is one loopback IP per machine the intended + allocation at 13.5k machines, and is there guidance for sizing this pool + in production site templates (dev templates ship 3)? +6. **Cycle economics**: identification/creation only run at the end of a + completed `explore_site` cycle, so `explorations_per_run` trades sweep + throughput against creation latency in a non-obvious way. Worth + documenting (or decoupling creation from the exploration cycle)? diff --git a/helm-prereqs/cleanup-machine-a-tron.sh b/helm-prereqs/cleanup-machine-a-tron.sh new file mode 100755 index 0000000000..e85f7f82e5 --- /dev/null +++ b/helm-prereqs/cleanup-machine-a-tron.sh @@ -0,0 +1,296 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# ============================================================================= +# cleanup-machine-a-tron.sh — tear down machine-a-tron for a from-scratch redeploy +# +# The inverse of setup-machine-a-tron.sh. By default it removes everything that +# script created EXCEPT the namespace and pull secret (so a re-run of setup does +# not need the registry API key again): +# * uninstalls the nico-machine-a-tron Helm release +# * deletes the client cert secret (so cert-manager reissues from the current CA) +# * resets the machine graph in the NICo DB (TRUNCATE ... CASCADE) while +# PRESERVING network config and the machine_interfaces_deletion singleton +# * reverts nico-core site_explorer.bmc_proxy (and any legacy +# override_target_* lines) and restarts nico-api +# * removes the Vault credential machines/bmc/site/root +# +# ONLY for simulation-only clusters. The DB reset truncates machines, +# machine_interfaces, explored_endpoints, explored_managed_hosts, and +# expected_machines with CASCADE — every machine on the cluster is assumed to +# be a machine-a-tron simulation. NEVER run this against a site with real +# hardware inventory. +# +# The machine_interfaces_deletion singleton (id=1) is preserved and its +# last_deletion watermark bumped — deleting it would break the +# machine_dhcp_records view and every subsequent DiscoverDhcp call. +# +# Required environment: +# KUBECONFIG Path to the target cluster kubeconfig. +# Optional environment: +# MAT_NAMESPACE Default: nico-mat +# NICO_SYSTEM_NS Default: nico-system +# POSTGRES_NS Default: postgres +# VAULT_NS Default: vault +# +# Usage: +# export KUBECONFIG=/path/to/kubeconfig +# ./cleanup-machine-a-tron.sh # prompt before each destructive step +# ./cleanup-machine-a-tron.sh -y # non-interactive +# ./cleanup-machine-a-tron.sh --delete-namespace # full teardown (also removes +# # pull secret + copied secrets) +# ./cleanup-machine-a-tron.sh --keep-db # leave the DB untouched +# ./cleanup-machine-a-tron.sh --keep-nico-core-config # leave bmc_proxy in place +# ./cleanup-machine-a-tron.sh --keep-vault-cred # leave BMC cred in Vault +# ============================================================================= + +set -euo pipefail + +MAT_NAMESPACE="${MAT_NAMESPACE:-nico-mat}" +NICO_SYSTEM_NS="${NICO_SYSTEM_NS:-nico-system}" +POSTGRES_NS="${POSTGRES_NS:-postgres}" +VAULT_NS="${VAULT_NS:-vault}" +RELEASE="nico-machine-a-tron" +NICO_DB="nico_system_nico" + +ASSUME_YES=false +DELETE_NAMESPACE=false +KEEP_DB=false +KEEP_NICO_CORE_CONFIG=false +KEEP_VAULT_CRED=false + +for arg in "$@"; do + case "$arg" in + -y|--yes) ASSUME_YES=true ;; + --delete-namespace) DELETE_NAMESPACE=true ;; + --keep-db) KEEP_DB=true ;; + --keep-nico-core-config) KEEP_NICO_CORE_CONFIG=true ;; + --keep-vault-cred) KEEP_VAULT_CRED=true ;; + -h|--help) grep '^#' "$0" | sed 's/^# \{0,1\}//' | head -70; exit 0 ;; + *) echo "Unknown argument: $arg" >&2; exit 2 ;; + esac +done + +# --- helpers (match setup-machine-a-tron.sh) -------------------------------- +_c() { printf '\033[%sm' "$1"; } +BOLD="$(_c 1)"; RED="$(_c 31)"; GREEN="$(_c 32)"; YEL="$(_c 33)"; BLU="$(_c 34)"; NC="$(_c 0)" +phase() { echo; echo "${BOLD}${BLU}== $* ==${NC}"; } +info() { echo " $*"; } +ok() { echo " ${GREEN}✓${NC} $*"; } +warn() { echo " ${YEL}!${NC} $*" >&2; } +die() { echo "${RED}ERROR:${NC} $*" >&2; exit 1; } +confirm() { $ASSUME_YES && return 0; read -r -p " $* [y/N] " a; [[ "$a" == "y" || "$a" == "Y" ]]; } + +CM_JSON="" +cleanup_tmp() { rm -f "$CM_JSON" 2>/dev/null || true; } +trap cleanup_tmp EXIT + +PG_PRIMARY="" +_pg_primary() { + [[ -n "$PG_PRIMARY" ]] && { echo "$PG_PRIMARY"; return; } + PG_PRIMARY="$(kubectl get pods -n "$POSTGRES_NS" -l application=spilo \ + -o jsonpath='{range .items[*]}{.metadata.name} {.metadata.labels.spilo-role}{"\n"}{end}' 2>/dev/null \ + | awk '$2=="master"{print $1}' | head -1)" + echo "$PG_PRIMARY" +} +psql_run() { # runs SQL, returns raw psql output (not -tAc, so NOTICEs show) + local pg; pg="$(_pg_primary)"; [[ -n "$pg" ]] || die "no Patroni primary in $POSTGRES_NS" + kubectl exec -n "$POSTGRES_NS" "$pg" -- su postgres -c "psql -d $NICO_DB -v ON_ERROR_STOP=1 -c \"$1\"" 2>&1 +} +psql_q() { + local pg; pg="$(_pg_primary)"; [[ -n "$pg" ]] || die "no Patroni primary in $POSTGRES_NS" + kubectl exec -n "$POSTGRES_NS" "$pg" -- su postgres -c "psql -d $NICO_DB -tAc \"$1\"" 2>/dev/null +} +_VAULT_TOKEN="" +vault_cmd() { + if [[ -z "$_VAULT_TOKEN" ]]; then + _VAULT_TOKEN="$(kubectl get secret nico-vault-token -n "$NICO_SYSTEM_NS" -o jsonpath='{.data.token}' | base64 -d)" + [[ -n "$_VAULT_TOKEN" ]] || die "could not read nico-vault-token from $NICO_SYSTEM_NS" + fi + kubectl exec -n "$VAULT_NS" vault-0 -c vault -- sh -c \ + "export VAULT_TOKEN='$_VAULT_TOKEN' VAULT_ADDR=https://127.0.0.1:8200 VAULT_SKIP_VERIFY=true; $1" 2>/dev/null +} + +# ============================================================================= +# Phase 0 — preflight +# ============================================================================= +phase "Phase 0 — preflight" +for t in kubectl helm; do command -v "$t" >/dev/null || die "$t not found in PATH"; done +kubectl cluster-info >/dev/null 2>&1 || die "cannot reach the cluster (check KUBECONFIG)" +ok "cluster reachable" +echo " This will tear down machine-a-tron in namespace ${MAT_NAMESPACE} and reset" +echo " the machine graph in DB ${NICO_DB}. Simulation clusters only." +confirm "Proceed with cleanup?" || die "aborted" + +# ============================================================================= +# Phase 1 — uninstall Helm release +# ============================================================================= +phase "Phase 1 — uninstall Helm release" +if helm status "$RELEASE" -n "$MAT_NAMESPACE" >/dev/null 2>&1; then + # retry — transient "http2: client connection lost" can abort the uninstall + _uninstalled=false + for _i in 1 2 3; do + if helm uninstall "$RELEASE" -n "$MAT_NAMESPACE" >/dev/null 2>&1; then _uninstalled=true; break; fi + helm status "$RELEASE" -n "$MAT_NAMESPACE" >/dev/null 2>&1 || { _uninstalled=true; break; } + warn "uninstall attempt ${_i} failed (transient?) — retrying" + sleep 5 + done + $_uninstalled && ok "release ${RELEASE} uninstalled" || die "could not uninstall ${RELEASE}" +else + ok "release ${RELEASE} not installed" +fi + +# ============================================================================= +# Phase 2 — namespace / cert secret +# ============================================================================= +phase "Phase 2 — namespace / cert secret" +if $DELETE_NAMESPACE; then + if kubectl get ns "$MAT_NAMESPACE" >/dev/null 2>&1; then + confirm "Delete the entire ${MAT_NAMESPACE} namespace (removes pull secret + copied secrets)?" \ + && { kubectl delete namespace "$MAT_NAMESPACE" --wait=false >/dev/null; ok "namespace ${MAT_NAMESPACE} deletion requested"; } \ + || warn "skipped namespace deletion" + else + ok "namespace ${MAT_NAMESPACE} already absent" + fi +else + kubectl delete secret "${RELEASE}-certificate" -n "$MAT_NAMESPACE" --ignore-not-found >/dev/null 2>&1 \ + && ok "client cert secret deleted (cert-manager reissues on next deploy)" + info "namespace + pull secret kept (use --delete-namespace for full teardown)" +fi + +# ============================================================================= +# Phase 3 — reset machine graph in the DB (preserve singleton + network config) +# ============================================================================= +phase "Phase 3 — reset machine graph in DB" +if $KEEP_DB; then + warn "--keep-db set; leaving DB untouched" +else + BEFORE="$(psql_q "SELECT + (SELECT count(*) FROM machines) || '/' || + (SELECT count(*) FROM machine_interfaces) || '/' || + (SELECT count(*) FROM expected_machines) || '/' || + (SELECT count(*) FROM explored_endpoints);")" + info "before (machines/interfaces/expected/endpoints): ${BEFORE:-?}" + if confirm "TRUNCATE machine graph (machines, machine_interfaces, explored_endpoints, explored_managed_hosts, expected_machines) CASCADE?"; then + OUT="$(psql_run "BEGIN; + TRUNCATE machines, machine_interfaces, explored_endpoints, explored_managed_hosts, expected_machines RESTART IDENTITY CASCADE; + UPDATE machine_interfaces_deletion SET last_deletion = now() WHERE id = 1; + INSERT INTO machine_interfaces_deletion (id) VALUES (1) ON CONFLICT (id) DO NOTHING; + COMMIT;")" || die "DB reset failed (rolled back): $OUT" + # surface which tables CASCADE touched + printf '%s\n' "$OUT" | grep -i "truncate cascades" | sed 's/^/ /' || true + AFTER="$(psql_q "SELECT + (SELECT count(*) FROM machines) || '/' || + (SELECT count(*) FROM machine_interfaces) || '/' || + (SELECT count(*) FROM expected_machines) || '/' || + (SELECT count(*) FROM explored_endpoints);")" + SINGLETON="$(psql_q "SELECT count(*) FROM machine_interfaces_deletion WHERE id=1;")" + ok "machine graph reset; after: ${AFTER} singleton_present: ${SINGLETON}" + [[ "$SINGLETON" == "1" ]] || die "singleton row missing after reset — investigate before redeploy" + else + warn "skipped DB reset" + fi +fi + +# ============================================================================= +# Phase 4 — revert nico-core site_explorer bmc_proxy +# ============================================================================= +phase "Phase 4 — revert nico-core site_explorer config" +if $KEEP_NICO_CORE_CONFIG; then + warn "--keep-nico-core-config set; leaving bmc_proxy in place" +else + CM_JSON="$(mktemp)" + if kubectl get cm nico-api-site-config-files -n "$NICO_SYSTEM_NS" -o json > "$CM_JSON" 2>/dev/null; then + if grep -qE "bmc_proxy|override_target_(host|ip|port)" "$CM_JSON"; then + python3 - "$CM_JSON" <<'PY' +import json, sys +path = sys.argv[1] +cm = json.load(open(path)) +drop = ("bmc_proxy", "override_target_host", "override_target_ip", "override_target_port") +for k, v in cm["data"].items(): + lines = [ln for ln in v.splitlines() if not any(t in ln for t in drop)] + cm["data"][k] = "\n".join(lines) + ("\n" if v.endswith("\n") else "") +for f in ("resourceVersion","uid","creationTimestamp","managedFields"): + cm["metadata"].pop(f, None) +json.dump(cm, open(path, "w")) +PY + kubectl apply -f "$CM_JSON" >/dev/null + info "removed bmc_proxy / override_target_* lines; restarting nico-api" + kubectl rollout restart deployment/nico-api -n "$NICO_SYSTEM_NS" >/dev/null + kubectl rollout status deployment/nico-api -n "$NICO_SYSTEM_NS" --timeout=180s >/dev/null \ + || warn "nico-api rollout slow; continuing" + ok "site_explorer config reverted" + else + ok "no bmc_proxy / override_target_* present" + fi + else + warn "nico-api-site-config-files configmap not found; skipping" + fi +fi + +# ============================================================================= +# Phase 5 — remove site BMC root Vault credential +# ============================================================================= +phase "Phase 5 — remove machine-a-tron Vault credentials" +# site root + the host factory cred that setup-machine-a-tron.sh seeds. +# (The DPU factory + UEFI creds belong to nico-prereqs kvSeeds — left alone.) +# NB: vendor segment is lowercase ("dell") — BMCVendor's Display impl +# lowercases; must match what setup-machine-a-tron.sh seeds. +MAT_VAULT_PATHS="machines/bmc/site/root machines/all_hosts/factory_default/bmc-metadata-items/dell" +if $KEEP_VAULT_CRED; then + warn "--keep-vault-cred set; leaving credentials in place" +elif confirm "Delete Vault credentials seeded/rotated for machine-a-tron (site root, host factory, per-MAC)?"; then + for p in $MAT_VAULT_PATHS; do + if vault_cmd "vault kv get secrets/$p" >/dev/null 2>&1; then + vault_cmd "vault kv metadata delete secrets/$p" >/dev/null 2>&1 \ + || vault_cmd "vault kv delete secrets/$p" >/dev/null 2>&1 || true + ok "removed $p" + else + ok "already absent: $p" + fi + done + # Per-MAC rotated creds (machines/bmc//root) — written by site-explorer + # when it rotates each BMC's password to the site root. MUST be purged: a + # fresh mock boots at the factory password, but a surviving per-MAC entry + # tells site-explorer the BMC was already rotated, so it presents the old + # rotated password, gets 401 Unauthorized, and permanently latches + # AvoidLockout (NICO-SITEEXPLORER-144) on the endpoint. + # Batch server-side in ONE kubectl exec: at scale there are thousands of + # per-MAC entries and one exec per deletion takes HOURS (one round-trip + + # exec setup each); a shell loop on the vault pod does them all in seconds. + _n="$(vault_cmd 'count=0 +for m in $(vault kv list -format=yaml secrets/machines/bmc 2>/dev/null | sed "s/^- //" | grep -v "^site/"); do + vault kv metadata delete "secrets/machines/bmc/${m%/}/root" >/dev/null 2>&1 && count=$((count+1)) +done +echo $count' || echo 0)" + ok "removed ${_n:-0} per-MAC rotated creds (batched server-side)" +else + warn "kept credentials" +fi + +# ============================================================================= +# Phase 6 — verify clean state +# ============================================================================= +phase "Phase 6 — verify" +helm status "$RELEASE" -n "$MAT_NAMESPACE" >/dev/null 2>&1 && warn "release still present" || ok "release gone" +if ! $KEEP_DB; then + M="$(psql_q "SELECT count(*) FROM machines;" || echo '?')" + I="$(psql_q "SELECT count(*) FROM machine_interfaces;" || echo '?')" + info "machines=${M} machine_interfaces=${I} (expect 0/0)" +fi +phase "Done" +info "Cluster reset for a from-scratch machine-a-tron deploy." +info "Next: ./setup-machine-a-tron.sh (set REGISTRY_PULL_SECRET if you used --delete-namespace)" diff --git a/helm-prereqs/setup-machine-a-tron.sh b/helm-prereqs/setup-machine-a-tron.sh new file mode 100755 index 0000000000..de42b65f11 --- /dev/null +++ b/helm-prereqs/setup-machine-a-tron.sh @@ -0,0 +1,868 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# ============================================================================= +# setup-machine-a-tron.sh — deploy machine-a-tron end to end on a NICo site +# +# machine-a-tron is a bare-metal simulator: it hosts mock DPUs and servers via a +# Redfish BMC mock so NICo can run full ingestion flows without real hardware. +# This script sets up EVERYTHING needed for a running NICo Core site to discover +# and create the simulated machines, in setup.sh style (phased, idempotent). +# +# It assumes NICo Core (nico-api, postgres/nico-pg-cluster, Vault, ESO, +# cert-manager) is already deployed on the target cluster (i.e. setup.sh has run +# and the site is bootstrapped). It does NOT deploy NICo Core. +# +# --------------------------------------------------------------------------- +# WHY EACH STEP EXISTS — the non-obvious failure modes this script prevents +# (learned the hard way; do not remove without understanding them): +# +# * CA refresh (Phase 3): after a site reprovision the nico-system CA and +# nico-api are recreated. A machine-a-tron left over from before still trusts +# the OLD CA (stale nico-roots) and presents a client cert signed by the OLD +# CA, so mTLS to nico-api fails with "client error (Connect)" on every call. +# We always re-copy nico-roots from nico-system and (Phase 8) delete the old +# cert secret so cert-manager reissues from the CURRENT CA. +# +# * BMC site-root credential (Phase 5): site-explorer's check_preconditions +# requires the Vault credential machines/bmc/site/root. It is NOT in the +# default nico-prereqs kvSeeds, so without it site-explorer aborts every run +# with MissingCredentials and never explores anything. +# +# * bmc_proxy field name (Phase 6): the site_explorer config field is +# `bmc_proxy = "host:port"` (a single string). `override_target_host` is NOT +# a real field (older docs were wrong); override_target_ip/port are +# DEPRECATED. Setting bmc_proxy at launch also makes allow_changing_bmc_proxy +# default true, which is what lets machine-a-tron's configureBmcProxyHost +# runtime call succeed instead of being PermissionDenied. +# +# * expected_machines (chart value registerExpectedMachines: true): without a +# matching expected_machines row (by BMC MAC), MachineCreator refuses to +# create the managed host. machine-a-tron auto-registers them when the value +# is true — the script asserts it. +# +# * DHCP pool sizing (Phase 7): machine-a-tron needs one OOB IP per BMC — +# hostCount + hostCount*dpuPerHostCount. Overflowing the OOB pool yields +# "No IP addresses left in prefix ..." and machines never register. +# +# * machine_interfaces_deletion singleton (Phase 10 check): the +# machine_dhcp_records VIEW inner-joins the singleton row id=1. If it is ever +# deleted (e.g. by manual lease cleanup) the view returns zero rows and +# DiscoverDhcp fails with "no rows ... expected to return at least one row". +# The script restores the row if missing. NEVER delete it. To free stale +# leases, force-delete machine records via the admin CLI or reprovision — +# do NOT hand-delete interface/dhcp rows. +# +# --------------------------------------------------------------------------- +# Tool requirements: kubectl, helm, jq +# +# Required environment: +# KUBECONFIG Path to the target cluster kubeconfig (or current +# kubectl context already points at it). +# +# Optional environment: +# NICO_IMAGE_REGISTRY REQUIRED unless image.repository is set in the +# values file. Registry/repository prefix, without +# http(s):// (same convention as setup.sh). The +# machine-a-tron image is pulled from +# ${NICO_IMAGE_REGISTRY}/machine-a-tron. +# MAT_IMAGE_TAG REQUIRED unless image.tag is set in the values +# file. machine-a-tron image tag. +# REGISTRY_PULL_SECRET Registry password/API key. Only needed if the +# pull secret does not already exist in the +# machine-a-tron namespace. +# REGISTRY_PULL_USERNAME Username for the pull secret. Default: $oauthtoken +# MAT_NAMESPACE Deployment namespace. Default: nico-mat +# NICO_SYSTEM_NS NICo Core namespace. Default: nico-system +# POSTGRES_NS Postgres namespace. Default: postgres +# VAULT_NS Vault namespace. Default: vault +# BMC_USERNAME Site BMC root username. Default: root +# BMC_PASSWORD Site BMC root password (rotation target). MUST +# differ from the mock factory defaults. +# Default: NicoSiteRoot1 +# OOB_DHCP_RELAY OOB/underlay gateway (BMC DHCP relay). Auto-detected +# from nico-core site config if unset. +# ADMIN_DHCP_RELAY Admin network gateway. Auto-detected if unset. +# HOST_COUNT Override machines.dell-hosts.hostCount. +# DPU_PER_HOST Override machines.dell-hosts.dpuPerHostCount. +# CHART_DIR Path to the nico-machine-a-tron chart. +# Default: /helm/charts/nico-machine-a-tron +# VALUES_FILE Base values template. +# Default: /values/machine-a-tron.yaml +# +# Usage: +# export KUBECONFIG=/path/to/kubeconfig +# ./setup-machine-a-tron.sh # prompt before deploy +# ./setup-machine-a-tron.sh -y # non-interactive +# ./setup-machine-a-tron.sh --skip-nico-core-config # don't touch nico-core +# ============================================================================= + +set -euo pipefail + +# --- config / defaults ------------------------------------------------------- +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "${SCRIPT_DIR}/.." && pwd)" + +MAT_NAMESPACE="${MAT_NAMESPACE:-nico-mat}" +NICO_SYSTEM_NS="${NICO_SYSTEM_NS:-nico-system}" +POSTGRES_NS="${POSTGRES_NS:-postgres}" +VAULT_NS="${VAULT_NS:-vault}" +BMC_USERNAME="${BMC_USERNAME:-root}" +# Site-wide BMC root password — the ROTATION TARGET. site-explorer logs into +# each mock BMC with its factory-default password, then rotates it to this +# value. It MUST DIFFER from both factory defaults below, or the rotation is a +# no-op and the mock keeps rejecting with "Factory-default password must be +# changed" (403) forever. +BMC_PASSWORD="${BMC_PASSWORD:-NicoSiteRoot1}" +# Factory defaults HARDCODED in the bmc-mock binary (crates/bmc-mock/src/lib.rs): +# host BMCs: DUMMY_FACTORY_PASSWORD = "factory_password" +# DPU BMCs: DUMMY_FACTORY_DPU_PASSWORD = "0penBmc" +# Do not change unless the mock changes. +FACTORY_HOST_BMC_PASSWORD="factory_password" +FACTORY_DPU_BMC_PASSWORD="0penBmc" +# Vendor path segment for the host factory cred. LOWERCASE is required: the +# credential path is built with format!("{vendor}") and BMCVendor's Display +# impl lowercases the variant name ("Dell Inc." → BMCVendor::Dell → "dell", +# crates/bmc-vendor/src/lib.rs impl Display). to_pascalcase() exists but is +# NOT used for Vault paths. +HOST_BMC_VENDOR="${HOST_BMC_VENDOR:-dell}" +# site-default UEFI passwords — check_preconditions requires them NON-EMPTY. +# The mock BMC does not validate them, so any non-empty value works. +UEFI_DPU_PASSWORD="${UEFI_DPU_PASSWORD:-bluefield}" +UEFI_HOST_PASSWORD="${UEFI_HOST_PASSWORD:-bluefield}" +REGISTRY_PULL_USERNAME="${REGISTRY_PULL_USERNAME:-\$oauthtoken}" +PULL_SECRET_NAME="${PULL_SECRET_NAME:-machine-a-tron-pull}" +RELEASE="nico-machine-a-tron" +BMC_MOCK_SVC="nico-machine-a-tron-bmc-mock" +BMC_MOCK_PORT="1266" +# site-explorer runs in nico-system, so it CANNOT resolve the bare service name +# (which resolves against its own namespace). bmc_proxy MUST use the +# cross-namespace FQDN of the bmc-mock service in the machine-a-tron namespace. +BMC_MOCK_FQDN="${BMC_MOCK_SVC}.${MAT_NAMESPACE}.svc.cluster.local" +NICO_DB="nico_system_nico" + +# --- deployment mode --------------------------------------------------------- +# override (default): all Redfish through site_explorer.bmc_proxy → one mock. +# scale: MetalLB/nginx mode — one LB Service per BMC, bmc_proxy UNSET, mock +# routes per-BMC via the Forwarded header. Uses values/machine-a-tron-scale.yaml +# plus two SIMULATED networks added to the nico-core site config (below) and +# raised site_explorer throughput knobs. See the chart README "METALLB MODE". +MAT_MODE="${MAT_MODE:-override}" +# Simulated networks for scale mode — must match the scale values file +# (relay addresses) and the MetalLB ipRange (inside the OOB prefix). +SCALE_OOB_PREFIX="10.100.0.0/17"; SCALE_OOB_GW="10.100.0.1" +SCALE_ADMIN_PREFIX="10.102.0.0/18"; SCALE_ADMIN_GW="10.102.0.1" +SCALE_RESERVE=1 +# site_explorer throughput knobs applied in scale mode (defaults 30/90/4 make +# 4500-host ingestion take ~9h; these bring it to ~1-2h). +SCALE_CONCURRENT_EXPLORATIONS="${SCALE_CONCURRENT_EXPLORATIONS:-100}" +# NB: keep explorations_per_run MODERATE. Identification and machine creation +# only run at the END of a completed explore_site cycle — a huge per-run value +# makes every cycle deep-scan hundreds of endpoints (dozens of Redfish calls +# each) and cycles stop completing, so machines are never created. ~120 keeps +# cycles under ~2 min while still sweeping the fleet quickly. +SCALE_EXPLORATIONS_PER_RUN="${SCALE_EXPLORATIONS_PER_RUN:-120}" +SCALE_MACHINES_CREATED_PER_RUN="${SCALE_MACHINES_CREATED_PER_RUN:-40}" + +CHART_DIR="${CHART_DIR:-${REPO_ROOT}/helm/charts/nico-machine-a-tron}" + +ASSUME_YES=false +SKIP_NICO_CORE_CONFIG=false +CM_JSON="" +MERGED_VALUES="" +cleanup() { rm -f "$CM_JSON" "$MERGED_VALUES" 2>/dev/null || true; } +trap cleanup EXIT + +for arg in "$@"; do + case "$arg" in + -y|--yes) ASSUME_YES=true ;; + --scale) MAT_MODE="scale" ;; + --skip-nico-core-config) SKIP_NICO_CORE_CONFIG=true ;; + -h|--help) grep '^#' "$0" | sed 's/^# \{0,1\}//' | head -110; exit 0 ;; + *) echo "Unknown argument: $arg" >&2; exit 2 ;; + esac +done + +if [[ "$MAT_MODE" == "scale" ]]; then + VALUES_FILE="${VALUES_FILE:-${SCRIPT_DIR}/values/machine-a-tron-scale.yaml}" +else + VALUES_FILE="${VALUES_FILE:-${SCRIPT_DIR}/values/machine-a-tron.yaml}" +fi + +# --- helpers ----------------------------------------------------------------- +_c() { printf '\033[%sm' "$1"; } +BOLD="$(_c 1)"; RED="$(_c 31)"; GREEN="$(_c 32)"; YEL="$(_c 33)"; BLU="$(_c 34)"; NC="$(_c 0)" +phase() { echo; echo "${BOLD}${BLU}== $* ==${NC}"; } +info() { echo " $*"; } +ok() { echo " ${GREEN}✓${NC} $*"; } +warn() { echo " ${YEL}!${NC} $*" >&2; } +die() { echo "${RED}ERROR:${NC} $*" >&2; exit 1; } +confirm() { + $ASSUME_YES && return 0 + read -r -p " $* [y/N] " ans + [[ "$ans" == "y" || "$ans" == "Y" ]] +} + +# psql against the consolidated NICo DB on the Patroni primary +PG_PRIMARY="" +_pg_primary() { + [[ -n "$PG_PRIMARY" ]] && { echo "$PG_PRIMARY"; return; } + PG_PRIMARY="$(kubectl get pods -n "$POSTGRES_NS" -l application=spilo \ + -o jsonpath='{range .items[*]}{.metadata.name} {.metadata.labels.spilo-role}{"\n"}{end}' 2>/dev/null \ + | awk '$2=="master"{print $1}' | head -1)" + echo "$PG_PRIMARY" +} +psql_q() { + local pg; pg="$(_pg_primary)" + [[ -n "$pg" ]] || die "no Patroni primary found in namespace $POSTGRES_NS" + kubectl exec -n "$POSTGRES_NS" "$pg" -- su postgres -c "psql -d $NICO_DB -v ON_ERROR_STOP=1 -tAc \"$1\"" 2>/dev/null +} +# count query that always yields a number — a transient kubectl/psql failure +# returns "0" instead of an empty string that would blow up (( )) arithmetic. +psql_count() { local r; r="$(psql_q "$1" || true)"; echo "${r:-0}"; } +# vault CLI on vault-0 using the root token stored in nico-system/nico-vault-token. +# Token is cached after the first read (it does not change within a run). +# env vars are exported (not inline-prefixed) so they apply across pipes, e.g. +# `echo ... | vault kv put ... -` — an inline prefix would bind them to echo only. +_VAULT_TOKEN="" +vault_cmd() { + if [[ -z "$_VAULT_TOKEN" ]]; then + _VAULT_TOKEN="$(kubectl get secret nico-vault-token -n "$NICO_SYSTEM_NS" -o jsonpath='{.data.token}' | base64 -d)" + [[ -n "$_VAULT_TOKEN" ]] || die "could not read nico-vault-token from $NICO_SYSTEM_NS" + fi + kubectl exec -n "$VAULT_NS" vault-0 -c vault -- sh -c \ + "export VAULT_TOKEN='$_VAULT_TOKEN' VAULT_ADDR=https://127.0.0.1:8200 VAULT_SKIP_VERIFY=true; $1" 2>/dev/null +} +# copy a secret from nico-system into the machine-a-tron namespace (strip metadata) +copy_secret() { + local name="$1" + kubectl get secret "$name" -n "$NICO_SYSTEM_NS" -o json 2>/dev/null \ + | jq 'del(.metadata.namespace,.metadata.resourceVersion,.metadata.uid,.metadata.creationTimestamp,.metadata.ownerReferences,.metadata.annotations,.metadata.managedFields)' \ + | kubectl apply -n "$MAT_NAMESPACE" -f - >/dev/null +} + +# ============================================================================= +# Phase 0 — preflight +# ============================================================================= +phase "Phase 0 — preflight" +for t in kubectl helm jq; do command -v "$t" >/dev/null || die "$t not found in PATH"; done +kubectl version -o json >/dev/null 2>&1 || kubectl cluster-info >/dev/null 2>&1 || die "cannot reach the cluster (check KUBECONFIG)" +ok "tools present, cluster reachable" +[[ -d "$CHART_DIR" ]] || die "chart dir not found: $CHART_DIR" +[[ -f "$VALUES_FILE" ]] || die "values file not found: $VALUES_FILE" +kubectl get deploy nico-api -n "$NICO_SYSTEM_NS" >/dev/null 2>&1 || die "nico-api not found in $NICO_SYSTEM_NS — deploy NICo Core (setup.sh) first" +[[ -n "$(_pg_primary)" ]] || die "no Postgres primary in $POSTGRES_NS" +kubectl get pod vault-0 -n "$VAULT_NS" >/dev/null 2>&1 || die "vault-0 not found in $VAULT_NS" +ok "NICo Core present: nico-api, postgres primary $(_pg_primary), vault-0" + +# portable extraction (macOS BSD sed/grep lack \s): [[:space:]] + awk on quotes +MAT_IMAGE_TAG="${MAT_IMAGE_TAG:-$(grep -E '^[[:space:]]*tag:' "$VALUES_FILE" | head -1 | awk -F'"' '{print $2}')}" +MAT_IMAGE_REPO="${MAT_IMAGE_REPO:-$(grep -E '^[[:space:]]*repository:' "$VALUES_FILE" | head -1 | awk -F'"' '{print $2}')}" +# Registry-agnostic (mirrors setup.sh): the image location comes from the +# environment, never from committed defaults. +if [[ -z "$MAT_IMAGE_REPO" ]]; then + [[ -n "${NICO_IMAGE_REGISTRY:-}" ]] || die "NICO_IMAGE_REGISTRY is unset and the values file has no image.repository (see setup.sh conventions)" + MAT_IMAGE_REPO="${NICO_IMAGE_REGISTRY}/machine-a-tron" +fi +HOST_COUNT="${HOST_COUNT:-$(grep -E '^[[:space:]]*hostCount:' "$VALUES_FILE" | head -1 | grep -oE '[0-9]+')}" +DPU_PER_HOST="${DPU_PER_HOST:-$(grep -E '^[[:space:]]*dpuPerHostCount:' "$VALUES_FILE" | head -1 | grep -oE '[0-9]+')}" +[[ -n "$MAT_IMAGE_TAG" ]] || die "MAT_IMAGE_TAG is unset and the values file has no image.tag" +[[ "$HOST_COUNT" =~ ^[0-9]+$ && "$DPU_PER_HOST" =~ ^[0-9]+$ ]] \ + || die "could not determine hostCount/dpuPerHostCount from $VALUES_FILE (set HOST_COUNT / DPU_PER_HOST)" +# Passwords are inlined into sh -c JSON heredocs on the vault pod; quotes, +# backslashes, or whitespace would break quoting or corrupt the JSON silently. +for _pw in "$BMC_PASSWORD" "$UEFI_DPU_PASSWORD" "$UEFI_HOST_PASSWORD"; do + case "$_pw" in + *[\'\"\\\ ]*) die "passwords must not contain quotes, backslashes, or spaces (BMC_PASSWORD / UEFI_*_PASSWORD)" ;; + esac +done +info "image: ${MAT_IMAGE_REPO}:${MAT_IMAGE_TAG} hosts: ${HOST_COUNT} dpus/host: ${DPU_PER_HOST}" + +# ============================================================================= +# Phase 1 — namespace +# ============================================================================= +phase "Phase 1 — namespace ${MAT_NAMESPACE}" +kubectl create namespace "$MAT_NAMESPACE" --dry-run=client -o yaml | kubectl apply -f - >/dev/null +# label so ESO / nico-roots sync treats it as a managed namespace +kubectl label namespace "$MAT_NAMESPACE" nico.nvidia.com/managed=true --overwrite >/dev/null +ok "namespace ready" + +# ============================================================================= +# Phase 2 — image pull secret +# ============================================================================= +phase "Phase 2 — image pull secret" +if kubectl get secret "$PULL_SECRET_NAME" -n "$MAT_NAMESPACE" >/dev/null 2>&1; then + ok "pull secret ${PULL_SECRET_NAME} already exists" +elif [[ -n "${REGISTRY_PULL_SECRET:-}" ]]; then + kubectl create secret docker-registry "$PULL_SECRET_NAME" -n "$MAT_NAMESPACE" \ + --docker-server="${MAT_IMAGE_REPO%%/*}" --docker-username="$REGISTRY_PULL_USERNAME" \ + --docker-password="$REGISTRY_PULL_SECRET" \ + --dry-run=client -o yaml | kubectl apply -f - >/dev/null + ok "pull secret ${PULL_SECRET_NAME} created" +else + warn "pull secret ${PULL_SECRET_NAME} missing and REGISTRY_PULL_SECRET unset" + warn " → set REGISTRY_PULL_SECRET, or ensure the image is already cached on nodes" +fi + +# ============================================================================= +# Phase 3 — refresh CA + Vault secrets from nico-system (GOTCHA: stale CA) +# ============================================================================= +phase "Phase 3 — refresh nico-roots CA + Vault secrets" +copy_secret nico-roots +ok "nico-roots synced from ${NICO_SYSTEM_NS} (current CA)" +for s in nico-vault-approle-tokens nico-vault-token; do + if kubectl get secret "$s" -n "$NICO_SYSTEM_NS" >/dev/null 2>&1; then + copy_secret "$s"; ok "$s synced" + else + warn "$s not present in ${NICO_SYSTEM_NS}; skipping (may produce log noise only)" + fi +done + +# ============================================================================= +# Phase 4 — seed site BMC root credential (GOTCHA: not in default kvSeeds) +# ============================================================================= +phase "Phase 4 — seed Vault BMC + UEFI credentials" +_kv_put() { # $1=path $2=username $3=password + vault_cmd "echo '{\"UsernamePassword\":{\"username\":\"$2\",\"password\":\"$3\"}}' | vault kv put secrets/$1 -" >/dev/null \ + || die "failed to write $1 to Vault" +} +_kv_password() { # $1=path → prints current password (empty if absent) + # `|| true` is load-bearing: when the path is absent, vault kv get fails and + # under `set -e` a failing $(...) assignment would kill the whole script. + vault_cmd "vault kv get -format=json secrets/$1" 2>/dev/null | jq -r '.data.data.UsernamePassword.password // empty' 2>/dev/null || true +} +# Factory-default creds — the INITIAL login site-explorer uses before rotating. +# Host and DPU factories DIFFER (see constants above); the wrong one yields 401 +# Unauthorized on every host BMC and a permanent AvoidLockout latch. +_kv_put "machines/all_hosts/factory_default/bmc-metadata-items/${HOST_BMC_VENDOR}" root "$FACTORY_HOST_BMC_PASSWORD" +ok "factory host cred: .../${HOST_BMC_VENDOR} = root/${FACTORY_HOST_BMC_PASSWORD}" +_kv_put "machines/all_dpus/factory_default/bmc-metadata-items/root" root "$FACTORY_DPU_BMC_PASSWORD" +ok "factory DPU cred: .../root = root/${FACTORY_DPU_BMC_PASSWORD}" +# Site-wide root — the rotation target; must differ from both factory passwords. +_cur="$(_kv_password machines/bmc/site/root)" +if [[ -z "$_cur" || "$_cur" == "$FACTORY_HOST_BMC_PASSWORD" || "$_cur" == "$FACTORY_DPU_BMC_PASSWORD" ]]; then + _kv_put "machines/bmc/site/root" "$BMC_USERNAME" "$BMC_PASSWORD" + ok "machines/bmc/site/root seeded (${BMC_USERNAME}/**** — distinct from factory)" +else + ok "machines/bmc/site/root already present with a non-factory password" +fi +# Self-heal stale per-MAC rotated creds (machines/bmc//root). site-explorer +# writes one per BMC after rotating its password to the site root; entries +# surviving from a previous deployment poison a fresh one — the fresh mock is at +# the factory password, but the per-MAC entry makes site-explorer present the +# old rotated password: 401 → permanent AvoidLockout. Only safe to purge when +# the machine graph is empty (live machines' per-MAC creds are real). +# "Fresh deployment" = no machines AND no interfaces. machines==0 alone is +# NOT enough: mid-ingestion (interfaces DHCP'd, endpoints explored, machines +# not yet created) the per-MAC creds are LIVE — purging them forces every +# endpoint back through the credential fallback. +if [[ "$(psql_count "SELECT count(*) FROM machines;")" == "0" \ + && "$(psql_count "SELECT count(*) FROM machine_interfaces;")" == "0" ]]; then + # Batched server-side in ONE kubectl exec — thousands of entries at scale; + # one exec per deletion takes hours. + _n="$(vault_cmd 'count=0 +for m in $(vault kv list -format=yaml secrets/machines/bmc 2>/dev/null | sed "s/^- //" | grep -v "^site/"); do + vault kv metadata delete "secrets/machines/bmc/${m%/}/root" >/dev/null 2>&1 && count=$((count+1)) +done +echo $count' || echo 0)" + if [[ "${_n:-0}" != "0" ]]; then + warn "purged ${_n} stale per-MAC BMC creds from a previous deployment (machine graph was empty)" + fi +fi +# site-explorer's check_preconditions ALSO requires the DPU + Host site_default +# UEFI creds to have a NON-EMPTY password. The nico-prereqs kvSeeds create these +# with empty passwords ("SITE SECRET: populate per site"), which fails the check +# with "vault does not have a valid password entry". Seed a non-empty value if +# the current password is empty/absent (the mock BMC does not validate it). +_seed_uefi() { # $1=vault path $2=password + local path="$1" pw="$2" cur + cur="$(_kv_password "$path")" + if [[ -n "$cur" ]]; then + ok "precondition cred present + valid: $path" + else + vault_cmd "echo '{\"UsernamePassword\":{\"username\":\"admin\",\"password\":\"${pw}\"}}' | vault kv put secrets/$path -" >/dev/null \ + || die "failed to seed UEFI cred $path" + ok "seeded UEFI cred (was empty): $path" + fi +} +_seed_uefi "machines/all_dpus/site_default/uefi-metadata-items/auth" "$UEFI_DPU_PASSWORD" +_seed_uefi "machines/all_hosts/site_default/uefi-metadata-items/auth" "$UEFI_HOST_PASSWORD" + +# ============================================================================= +# Phase 5 — configure nico-core site config for the selected mode +# override: set site_explorer.bmc_proxy (GOTCHA: field name; FQDN required) +# scale: REMOVE bmc_proxy, add simulated networks + throughput knobs +# ============================================================================= +phase "Phase 5 — nico-core site config (${MAT_MODE} mode)" +if $SKIP_NICO_CORE_CONFIG; then + warn "--skip-nico-core-config set; configure [site_explorer]/[networks] manually for ${MAT_MODE} mode" +elif [[ "$MAT_MODE" == "scale" ]]; then + CM_JSON="$(mktemp)" + kubectl get cm nico-api-site-config-files -n "$NICO_SYSTEM_NS" -o json > "$CM_JSON" 2>/dev/null \ + || die "nico-api-site-config-files configmap not found" + _PATCH_RESULT="$(SCALE_OOB_PREFIX="$SCALE_OOB_PREFIX" SCALE_OOB_GW="$SCALE_OOB_GW" \ + SCALE_ADMIN_PREFIX="$SCALE_ADMIN_PREFIX" SCALE_ADMIN_GW="$SCALE_ADMIN_GW" \ + SCALE_RESERVE="$SCALE_RESERVE" \ + BMC_PROXY="${BMC_MOCK_FQDN}:${BMC_MOCK_PORT}" \ + KNOB_CONC="$SCALE_CONCURRENT_EXPLORATIONS" KNOB_EPR="$SCALE_EXPLORATIONS_PER_RUN" \ + KNOB_MCPR="$SCALE_MACHINES_CREATED_PER_RUN" python3 - "$CM_JSON" <<'PY' +import json, os, sys +path = sys.argv[1] +cm = json.load(open(path)) +env = os.environ +# lines managed by this script inside [site_explorer] +drop = ("bmc_proxy", "override_target_host", "override_target_ip", "override_target_port", + "concurrent_explorations", "explorations_per_run", "machines_created_per_run") +knobs = [ + # PROXY-DIRECT: the Redfish client injects "Forwarded: host=" + # whenever bmc_proxy is set; the mock's registry routes on it. One + # ClusterIP service serves the whole simulated fleet. + f' bmc_proxy = "{env["BMC_PROXY"]}"', + f' concurrent_explorations = {env["KNOB_CONC"]}', + f' explorations_per_run = {env["KNOB_EPR"]}', + f' machines_created_per_run = {env["KNOB_MCPR"]}', +] +networks = f''' +# --- simulated networks for machine-a-tron scale testing (managed by +# --- setup-machine-a-tron.sh --scale; safe to leave in place) --- +[networks.simulated-oob] +type = "underlay" +prefix = "{env["SCALE_OOB_PREFIX"]}" +gateway = "{env["SCALE_OOB_GW"]}" +mtu = 9000 +reserve_first = {env["SCALE_RESERVE"]} + +[networks.simulated-admin] +type = "admin" +prefix = "{env["SCALE_ADMIN_PREFIX"]}" +gateway = "{env["SCALE_ADMIN_GW"]}" +mtu = 9000 +reserve_first = {env["SCALE_RESERVE"]} +''' +# Machine creation allocates one loopback IP per machine from pools.lo-ip — +# site templates ship tiny ranges (dev6: 3 addresses) that exhaust instantly +# ("Resource pool lo-ip is empty"). Pools DO reconcile at startup (unlike +# networks), so appending a simulated range takes effect on restart. +SIM_LO = ', { start = "10.103.0.1", end = "10.103.63.254" }]' +changed = False +for k, v in cm["data"].items(): + if "[site_explorer]" not in v: + continue + out, in_lo = [], False + for ln in v.splitlines(): + s = ln.strip() + if any(t in ln for t in drop): + continue + if s.startswith("[pools."): + in_lo = (s == "[pools.lo-ip]") + if in_lo and s.startswith("ranges") and "10.103.0.1" not in ln: + r = ln.rstrip(); idx = r.rfind("]") + ln = r[:idx] + SIM_LO + r[idx+1:] + out.append(ln) + if s == "[site_explorer]": + out.extend(knobs) + new = "\n".join(out) + ("\n" if v.endswith("\n") else "") + if "[networks.simulated-oob]" not in new: + new = new.rstrip("\n") + "\n" + networks + if new != v: + cm["data"][k] = new + changed = True +for f in ("resourceVersion","uid","creationTimestamp","managedFields"): + cm["metadata"].pop(f, None) +json.dump(cm, open(path, "w")) +print("changed" if changed else "nochange") +PY +)" + if [[ "$_PATCH_RESULT" == "changed" ]]; then + kubectl apply -f "$CM_JSON" >/dev/null + info "scale config applied (proxy-direct bmc_proxy, simulated networks, knobs, lo-ip); restarting nico-api" + kubectl rollout restart deployment/nico-api -n "$NICO_SYSTEM_NS" >/dev/null + kubectl rollout status deployment/nico-api -n "$NICO_SYSTEM_NS" --timeout=180s >/dev/null \ + || warn "nico-api rollout did not complete in time; continuing" + ok "scale networks: oob ${SCALE_OOB_PREFIX} (gw ${SCALE_OOB_GW}), admin ${SCALE_ADMIN_PREFIX} (gw ${SCALE_ADMIN_GW})" + ok "site_explorer knobs: concurrent=${SCALE_CONCURRENT_EXPLORATIONS} per_run=${SCALE_EXPLORATIONS_PER_RUN} create/run=${SCALE_MACHINES_CREATED_PER_RUN}" + else + ok "scale config already in place" + fi + + # --- ensure the simulated network SEGMENTS exist ------------------------- + # Config-driven segment creation (create_initial_networks) is bootstrap- + # once: it SKIPS entirely when the DB has multiple DNS domains ("we + # probably created the network much earlier", crates/api-core/src/db_init.rs). + # On an established site the new [networks.*] stanzas therefore never + # materialize and every mat DHCP fails with "No network segment defined + # for relay addresses". Fallback: clone an existing segment of the same + # type (identity fields overridden, vlan/vni cleared) + insert the prefix. + _ensure_segment() { # $1=name $2=type-ilike $3=prefix $4=gateway $5=reserve + local name="$1" typ="$2" pfx="$3" gw="$4" rsv="$5" + if [[ "$(psql_count "SELECT count(*) FROM network_segments WHERE name='${name}';")" != "0" ]]; then + ok "segment ${name} present" + return + fi + warn "segment ${name} missing (config seeding is bootstrap-once on multi-domain sites) — creating from template" + # allocation_strategy is forced to 'dynamic': templates may be + # 'reserved' (static-assignments segments), which rejects every mat + # DHCP with "configured for static DHCP leases only". + psql_q "INSERT INTO network_segments + SELECT (jsonb_populate_record(ns, jsonb_build_object( + 'id', gen_random_uuid()::text, 'name', '${name}', + 'allocation_strategy', 'dynamic', + 'vlan_id', NULL, 'vni_id', NULL))).* + FROM network_segments ns + WHERE ns.network_segment_type::text ILIKE '${typ}' LIMIT 1;" >/dev/null \ + || die "failed to create segment ${name} (no ${typ} template segment?)" + psql_q "INSERT INTO network_prefixes (segment_id, prefix, gateway, num_reserved) + SELECT id, '${pfx}'::cidr, '${gw}'::inet, ${rsv} + FROM network_segments WHERE name='${name}';" >/dev/null \ + || die "failed to add prefix ${pfx} to segment ${name}" + ok "segment ${name} created: ${pfx} (gw ${gw}, reserved ${rsv})" + } + _ensure_segment "simulated-oob" "underlay" "$SCALE_OOB_PREFIX" "$SCALE_OOB_GW" "$SCALE_RESERVE" + _ensure_segment "simulated-admin" "admin" "$SCALE_ADMIN_PREFIX" "$SCALE_ADMIN_GW" "$SCALE_RESERVE" + + # --- widen the lo-ip resource pool --------------------------------------- + # Machine creation allocates one loopback IP per machine from resource_pool + # rows. Pool DEFINITIONS are seed-once ("Declaration has drifted since + # seed ... not re-applying", crates/api-db/src/resource_pool.rs), so config + # changes to [pools.lo-ip] are IGNORED on established sites — rows must be + # inserted directly. Site templates ship tiny ranges (dev6: 3 addresses) + # that exhaust instantly ("Resource pool lo-ip is empty"). + _LO_FREE="$(psql_count "SELECT count(*) FROM resource_pool WHERE name='lo-ip' AND allocated IS NULL;")" + if (( _LO_FREE < HOST_COUNT * (1 + DPU_PER_HOST) )); then + info "widening lo-ip pool (${_LO_FREE} free < needed) with simulated range 10.103.0.1-10.103.63.254" + psql_q "INSERT INTO resource_pool (name, value, value_type, auto_assign, state, state_version, created) + SELECT 'lo-ip', host('10.103.0.0'::inet + g), 'ipv4', true, + '{\\\"state\\\":\\\"free\\\"}'::jsonb, + (SELECT state_version FROM resource_pool WHERE name='lo-ip' LIMIT 1), + now() + FROM generate_series(1, 16382) g + WHERE NOT EXISTS (SELECT 1 FROM resource_pool rp WHERE rp.name='lo-ip' AND rp.value = host('10.103.0.0'::inet + g));" >/dev/null \ + || die "failed to widen lo-ip pool" + ok "lo-ip pool: $(psql_count "SELECT count(*) FROM resource_pool WHERE name='lo-ip' AND allocated IS NULL;") free" + else + ok "lo-ip pool has ${_LO_FREE} free addresses" + fi +else + CM_JSON="$(mktemp)" + kubectl get cm nico-api-site-config-files -n "$NICO_SYSTEM_NS" -o json > "$CM_JSON" 2>/dev/null \ + || die "nico-api-site-config-files configmap not found" + if grep -q "bmc_proxy = \"${BMC_MOCK_FQDN}:${BMC_MOCK_PORT}\"" "$CM_JSON"; then + ok "bmc_proxy already configured" + else + _PATCH_RESULT="$(BMC_PROXY="${BMC_MOCK_FQDN}:${BMC_MOCK_PORT}" python3 - "$CM_JSON" <<'PY' +import json, os, sys +proxy = os.environ["BMC_PROXY"] +path = sys.argv[1] +cm = json.load(open(path)) +line = f' bmc_proxy = "{proxy}"' +drop = ("bmc_proxy", "override_target_host", "override_target_ip", "override_target_port") +changed = False +for k, v in cm["data"].items(): + if "[site_explorer]" not in v: + continue + out = [] + for ln in v.splitlines(): + if any(t in ln for t in drop): # strip any stale/legacy proxy lines + continue + out.append(ln) + if ln.strip() == "[site_explorer]": + out.append(line) # insert the correct FQDN bmc_proxy + cm["data"][k] = "\n".join(out) + ("\n" if v.endswith("\n") else "") + changed = True +for f in ("resourceVersion","uid","creationTimestamp","managedFields"): + cm["metadata"].pop(f, None) +json.dump(cm, open(path, "w")) +print("changed" if changed else "nochange") +PY +)" + if [[ "$_PATCH_RESULT" == "changed" ]]; then + kubectl apply -f "$CM_JSON" >/dev/null + info "configmap patched; restarting nico-api to load bmc_proxy" + kubectl rollout restart deployment/nico-api -n "$NICO_SYSTEM_NS" >/dev/null + kubectl rollout status deployment/nico-api -n "$NICO_SYSTEM_NS" --timeout=180s >/dev/null \ + || warn "nico-api rollout did not complete in time; continuing" + ok "bmc_proxy set to ${BMC_MOCK_FQDN}:${BMC_MOCK_PORT}" + else + ok "no [site_explorer] section found to patch — check the configmap manually" + fi + fi +fi + +# ============================================================================= +# Phase 6 — resolve DHCP relays + sizing check +# ============================================================================= +phase "Phase 6 — DHCP relays + pool sizing (${MAT_MODE})" +if [[ "$MAT_MODE" == "scale" ]]; then + # scale mode uses the SIMULATED networks added in Phase 5 — constants, + # no live-config parsing needed. + OOB_PREFIX="$SCALE_OOB_PREFIX"; OOB_DHCP_RELAY="${OOB_DHCP_RELAY:-$SCALE_OOB_GW}" + ADMIN_PREFIX="$SCALE_ADMIN_PREFIX"; ADMIN_DHCP_RELAY="${ADMIN_DHCP_RELAY:-$SCALE_ADMIN_GW}" + OOB_RESERVE="$SCALE_RESERVE"; ADMIN_RESERVE="$SCALE_RESERVE" +else + SITE_CFG="$(kubectl get cm nico-api-site-config-files -n "$NICO_SYSTEM_NS" \ + -o jsonpath='{.data.nico-api-site-config\.toml}' 2>/dev/null || true)" + # older deployments carry the config under the carbide-* key only + [[ -n "$SITE_CFG" ]] || SITE_CFG="$(kubectl get cm nico-api-site-config-files -n "$NICO_SYSTEM_NS" \ + -o jsonpath='{.data.carbide-api-site-config\.toml}' 2>/dev/null || true)" + # gateway lines appear as: gateway = "10.x.y.z" (admin then underlay in template order). + # Portable parse (no mapfile / negative index — macOS ships bash 3.2). + # Comment lines are stripped first: the config template carries commented + # examples (e.g. "# reserve_first = 5") that would otherwise be picked up. + SITE_CFG_CODE="$(printf '%s\n' "$SITE_CFG" | grep -vE '^[[:space:]]*#' || true)" + GW_LIST="$(printf '%s\n' "$SITE_CFG_CODE" | grep -oE 'gateway = "[0-9.]+"' | grep -oE '[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+' || true)" + PFX_LIST="$(printf '%s\n' "$SITE_CFG_CODE" | grep -oE 'prefix = "[0-9./]+"' | grep -oE '[0-9./]+' || true)" + RSV_LIST="$(printf '%s\n' "$SITE_CFG_CODE" | grep -oE 'reserve_first = [0-9]+' | grep -oE '[0-9]+' || true)" + # admin segment is the FIRST prefix/gateway pair, OOB/underlay the LAST. + ADMIN_PREFIX="$(printf '%s\n' "$PFX_LIST" | head -1)" + OOB_PREFIX="$(printf '%s\n' "$PFX_LIST" | tail -1)" + OOB_DHCP_RELAY="${OOB_DHCP_RELAY:-$(printf '%s\n' "$GW_LIST" | tail -1)}" + ADMIN_DHCP_RELAY="${ADMIN_DHCP_RELAY:-$(printf '%s\n' "$GW_LIST" | head -1)}" + # Usable IPs per pool = 2^(32-mask) − reserve_first − 1 (broadcast). + # reserve_first covers the network address, gateway, and operator-reserved + # leading addresses — verified live: /28 with reserve_first=5 → 10 usable. + ADMIN_RESERVE="$(printf '%s\n' "$RSV_LIST" | head -1)"; ADMIN_RESERVE="${ADMIN_RESERVE:-5}" + OOB_RESERVE="$(printf '%s\n' "$RSV_LIST" | tail -1)"; OOB_RESERVE="${OOB_RESERVE:-5}" +fi +[[ -n "$OOB_DHCP_RELAY" && -n "$ADMIN_DHCP_RELAY" ]] \ + || die "could not resolve DHCP relays; set OOB_DHCP_RELAY and ADMIN_DHCP_RELAY" +ok "OOB: relay ${OOB_DHCP_RELAY} prefix ${OOB_PREFIX:-unknown}" +ok "admin: relay ${ADMIN_DHCP_RELAY} prefix ${ADMIN_PREFIX:-unknown}" +_usable() { local m="${1##*/}" r="$2"; local u=$(( (1 << (32 - m)) - r - 1 )); (( u < 0 )) && u=0; echo "$u"; } +# Demand per pool (measured live): +# OOB = hostCount*(1 + dpuPerHost) — one BMC IP per host and per DPU +# admin = hostCount*(dpuPerHost + 1) — one host-PF IP per DPU at DHCP time, +# PLUS one admin IP per host allocated by machine creation (creation +# fails with "No IP addresses left in prefix " without it) +if [[ "${OOB_PREFIX:-}" == */* && "${ADMIN_PREFIX:-}" == */* ]]; then + OOB_USABLE="$(_usable "$OOB_PREFIX" "$OOB_RESERVE")"; ADMIN_USABLE="$(_usable "$ADMIN_PREFIX" "$ADMIN_RESERVE")" + # max hosts each pool supports, then take the min + FIT_OOB=$(( OOB_USABLE / (1 + DPU_PER_HOST) )) + FIT_ADMIN=$(( ADMIN_USABLE / (DPU_PER_HOST + 1) )) + FIT=$(( FIT_OOB < FIT_ADMIN ? FIT_OOB : FIT_ADMIN )) + info "pool fit: OOB ${OOB_PREFIX} ≈${OOB_USABLE} usable → ≤${FIT_OOB} hosts; admin ${ADMIN_PREFIX} ≈${ADMIN_USABLE} usable → ≤${FIT_ADMIN} hosts" + if (( HOST_COUNT > FIT )); then + (( FIT < 1 )) && die "pools too small for even 1 host × ${DPU_PER_HOST} DPUs — widen the admin/OOB prefixes or lower DPU_PER_HOST" + warn "requested ${HOST_COUNT} hosts exceeds pool capacity (${FIT}) — auto-fitting hostCount=${FIT}" + warn " (override with HOST_COUNT/DPU_PER_HOST env vars, or widen the site's DHCP prefixes)" + confirm "Proceed with hostCount=${FIT} × ${DPU_PER_HOST} DPUs?" || die "aborted on sizing" + HOST_COUNT="$FIT" + fi + NEED=$(( HOST_COUNT + HOST_COUNT * DPU_PER_HOST )) + ok "sizing: ${HOST_COUNT} hosts × ${DPU_PER_HOST} DPUs → ${NEED} OOB + $(( HOST_COUNT * (DPU_PER_HOST + 1) )) admin IPs" +else + NEED=$(( HOST_COUNT + HOST_COUNT * DPU_PER_HOST )) + warn "could not parse both pool prefixes — skipping sizing check (need ${NEED} OOB IPs)" +fi + +# ============================================================================= +# Phase 7 — DB safety: restore machine_interfaces_deletion singleton (GOTCHA) +# ============================================================================= +phase "Phase 7 — DB safety checks" +# "!= 1" (not "== 0") so a transient query failure (empty result) also takes +# the restore path — the INSERT is idempotent (ON CONFLICT DO NOTHING). +SINGLETON="$(psql_count "SELECT count(*) FROM machine_interfaces_deletion WHERE id=1;")" +if [[ "$SINGLETON" != "1" ]]; then + warn "machine_interfaces_deletion singleton (id=1) missing — restoring" + warn " (its absence breaks the machine_dhcp_records view → DiscoverDhcp 'no rows' errors)" + psql_q "INSERT INTO machine_interfaces_deletion (id) VALUES (1) ON CONFLICT (id) DO NOTHING;" >/dev/null + ok "singleton restored" +else + ok "machine_interfaces_deletion singleton present" +fi +ORPHANS="$(psql_count "SELECT count(*) FROM machine_interfaces mi WHERE NOT EXISTS (SELECT 1 FROM machines m WHERE m.id = mi.machine_id);")" +MACHINES_NOW="$(psql_count "SELECT count(*) FROM machines;")" +if [[ "${ORPHANS:-0}" -gt 0 && "${MACHINES_NOW:-0}" == "0" ]]; then + warn "${ORPHANS} orphaned machine_interfaces (no parent machine) may hold OOB leases" + warn " → if DHCP later reports exhaustion, force-delete stale records via the admin CLI" + warn " or reprovision. Do NOT hand-delete interface/dhcp rows (breaks the singleton)." +fi + +# ============================================================================= +# Phase 8 — reissue client cert from current CA (GOTCHA: stale cert) +# ============================================================================= +phase "Phase 8 — reissue machine-a-tron client cert" +# Always delete: a cert issued under a previous CA fails mTLS to nico-api with +# "client error (Connect)" on every call. cert-manager reissues from the +# current CA within seconds of the deploy — there is no reason to keep it. +kubectl delete secret "${RELEASE}-certificate" -n "$MAT_NAMESPACE" --ignore-not-found >/dev/null 2>&1 +ok "client cert cleared; cert-manager reissues from the current CA on deploy" + +# ============================================================================= +# Phase 9 — deploy the chart +# ============================================================================= +phase "Phase 9 — helm upgrade --install ${RELEASE}" +MERGED_VALUES="$(mktemp)" +# Site-specific overrides ONLY (never committed). Passed as a second -f so Helm +# deep-merges it over the base template (last -f wins per key) — avoids +# unreliable duplicate top-level keys within a single YAML file. +cat > "$MERGED_VALUES" <> "$MERGED_VALUES" </dev/null 2>&1 \ + && ok "client certificate Ready" || warn "certificate not Ready yet — check cert-manager" + +# wait windows scale with the deployment size (scale mode: hundreds-thousands) +IFACE_WAIT=$(( 90 + NEED )); (( IFACE_WAIT > 1800 )) && IFACE_WAIT=1800 +info "giving machine-a-tron time to register + DHCP (up to ${IFACE_WAIT}s)..." +_end=$((SECONDS+IFACE_WAIT)) +IFACES=0 +while (( SECONDS < _end )); do + IFACES="$(psql_count "SELECT count(*) FROM machine_interfaces;")" + (( IFACES >= NEED )) && break + sleep 10 +done +IPS="$(psql_count "SELECT count(*) FROM machine_interface_addresses;")" +info "machine_interfaces=${IFACES} (need ${NEED}) ips_allocated=${IPS}" +(( IFACES >= NEED )) && ok "BMC interfaces registered + DHCP allocated" \ + || warn "fewer interfaces than expected — check pool sizing / bmc DHCP" + +# --- expected_machines: required for machine creation (matched by BMC MAC) --- +# machine-a-tron auto-registers them (registerExpectedMachines: true), but on +# nico-api builds without the Machineatron→AddExpectedMachine RBAC grant +# (crates/api-core/src/auth/internal_rbac_rules.rs) the call is 403'd. Fall +# back to direct DB registration mirroring what the API call would create. +info "waiting for expected_machines (auto-registration)..." +_end=$((SECONDS+45)); EXPECTED=0 +while (( SECONDS < _end )); do + EXPECTED="$(psql_count "SELECT count(*) FROM expected_machines;")" + (( EXPECTED > 0 )) && break + sleep 5 +done +if (( EXPECTED > 0 )); then + ok "expected_machines=${EXPECTED} (registerExpectedMachines worked — RBAC grant present)" +else + warn "no expected_machines — this nico-api build lacks the Machineatron" + warn " AddExpectedMachine RBAC grant (403). Falling back to direct DB registration." + # scope strictly to BMC interfaces in the OOB prefix — with an unparsed + # prefix the filter would match admin-segment interfaces too and register + # them with the wrong factory password. + [[ "${OOB_PREFIX:-}" == */* ]] || die "cannot scope the expected_machines fallback: OOB prefix unknown" + psql_q "INSERT INTO expected_machines (id, serial_number, bmc_mac_address, bmc_username, bmc_password) + SELECT gen_random_uuid(), 'MAT-' || replace(mi.mac_address::text, ':', ''), mi.mac_address, 'root', '${FACTORY_HOST_BMC_PASSWORD}' + FROM machine_interfaces mi + JOIN machine_interface_addresses mia ON mia.interface_id = mi.id + WHERE mia.address << '${OOB_PREFIX}'::inet + AND NOT EXISTS (SELECT 1 FROM expected_machines em WHERE em.bmc_mac_address = mi.mac_address);" >/dev/null \ + || die "expected_machines DB fallback INSERT failed" + EXPECTED="$(psql_count "SELECT count(*) FROM expected_machines;")" + (( EXPECTED > 0 )) || die "expected_machines still 0 after fallback — machine creation cannot proceed" + ok "expected_machines=${EXPECTED} registered via DB fallback" +fi + +# --- kick exploration: clear any AvoidLockout latched before creds existed --- +# Exploration cycles that ran before Phase 4/5 completed record Unauthorized, +# which latches a self-perpetuating AvoidLockout in the exploration report. +# This mirrors the API's clear_last_known_error + request_exploration pair +# (crates/api-db/src/explored_endpoints.rs) — including the +# waiting_for_explorer_refresh flag, which gates preingestion until a fresh +# probe lands (skipping it would let preingestion act on the stale report). +psql_q "UPDATE explored_endpoints + SET exploration_report = jsonb_set(exploration_report, '{LastExplorationError}', 'null'::jsonb), + exploration_requested = true, + waiting_for_explorer_refresh = true;" >/dev/null || true +ok "cleared exploration lockouts + requested re-exploration" + +# machine target = one row per host + per DPU; wait scales with host count +MACHINE_TARGET=$(( HOST_COUNT * (1 + DPU_PER_HOST) )) +MACHINE_WAIT=$(( 420 + HOST_COUNT * 3 )); (( MACHINE_WAIT > 5400 )) && MACHINE_WAIT=5400 +info "waiting for explore → rotate → preingest → identify → create (target ${MACHINE_TARGET} machines, up to ${MACHINE_WAIT}s)..." +_end=$((SECONDS+MACHINE_WAIT)); MACHINES=0 +while (( SECONDS < _end )); do + MACHINES="$(psql_count "SELECT count(*) FROM machines;")" + (( MACHINES >= MACHINE_TARGET )) && break + # single round-trip for the progress line + _prog="$(psql_q "SELECT + (SELECT count(*) FILTER (WHERE exploration_report->'LastExplorationError' = 'null'::jsonb) FROM explored_endpoints) || '/' || + (SELECT count(*) FROM explored_managed_hosts);" || echo '?/?')" + info " endpoints_ok=${_prog%%/*}/${IFACES} managed_hosts=${_prog##*/} machines=${MACHINES} ..." + # Re-clear any AvoidLockout that latched during the wait (e.g. an + # exploration racing a mock reboot from preingestion's initial BMC reset). + # Idempotent; scoped to latched endpoints only so successful reports keep + # their state. + psql_q "UPDATE explored_endpoints + SET exploration_report = jsonb_set(exploration_report, '{LastExplorationError}', 'null'::jsonb), + exploration_requested = true, + waiting_for_explorer_refresh = true + WHERE exploration_report->'LastExplorationError'->>'Type' IN ('AvoidLockout','Unauthorized');" >/dev/null || true + # ...and UNPARK endpoints that have since explored clean: a lingering + # waiting_for_explorer_refresh gates them out of preingestion + # (find_preingest_not_waiting) even after a healthy report lands, stalling + # the pipeline at 'initial' indefinitely. + psql_q "UPDATE explored_endpoints + SET waiting_for_explorer_refresh = false, exploration_requested = false + WHERE waiting_for_explorer_refresh + AND exploration_report->'LastExplorationError' = 'null'::jsonb;" >/dev/null || true + sleep 25 +done +ENDPOINTS="$(psql_count "SELECT count(*) FROM explored_endpoints;")" +MHOSTS="$(psql_count "SELECT count(*) FROM explored_managed_hosts;")" +echo +if (( MACHINES >= MACHINE_TARGET )); then + ok "${GREEN}END TO END OK${NC} — endpoints=${ENDPOINTS}, managed_hosts=${MHOSTS}, machines=${MACHINES}/${MACHINE_TARGET}" +elif (( MACHINES > 0 )); then + ok "${GREEN}MACHINES CREATED${NC} (partial) — ${MACHINES}/${MACHINE_TARGET}; ingestion continuing in the background" +else + warn "machines not created yet (endpoints=${ENDPOINTS}, managed_hosts=${MHOSTS})" + warn " check: kubectl logs -n ${NICO_SYSTEM_NS} deploy/nico-api | grep -i 'site.explor\\|MissingCred\\|Refusing\\|Failed to create'" + warn " check: kubectl logs -n ${MAT_NAMESPACE} deploy/${RELEASE} | grep -iE 'No IP addresses|error'" + warn " a common cause: admin/OOB pool exhaustion — see the sizing output of Phase 6" +fi + +phase "Done" +info "machine-a-tron release ${RELEASE} deployed to ${MAT_NAMESPACE}." +info "Redeploy/iterate: re-run this script (idempotent) after 'export KUBECONFIG=...'." diff --git a/helm-prereqs/values/machine-a-tron-multipod.yaml b/helm-prereqs/values/machine-a-tron-multipod.yaml new file mode 100644 index 0000000000..723d24e3a4 --- /dev/null +++ b/helm-prereqs/values/machine-a-tron-multipod.yaml @@ -0,0 +1,194 @@ +# machine-a-tron site values — MULTI-POD bmcServices mode (validated example) +# +# Two machine-a-tron pods, each simulating 5 hosts x 2 DPUs (30 BMCs total), +# each with a dedicated per-BMC ClusterIP Service range. Copy this file and +# adjust CIDRs/counts for your site, then deploy with: +# +# helm upgrade -i nico-machine-a-tron helm/charts/nico-machine-a-tron \ +# -n nico-mat -f .yaml +# +# READ FIRST — requirements this mode adds on top of single-pod mode: +# +# * Kubernetes 1.29+ for the ServiceCIDR object (beta 1.31+). On OLDER +# clusters set bmcServices.serviceCIDR.create=false and pick pod CIDRs +# INSIDE the apiserver's --service-cluster-ip-range — any clusterIP outside +# that range is rejected outright ("the provided IP is not in the valid +# range"). Verify the chosen sub-ranges are free of existing ClusterIPs: +# kubectl get svc -A -o jsonpath='{range .items[*]}{.spec.clusterIP}{"\n"}{end}' +# +# * The machine-a-tron image must include the bmc-mock Host-header routing +# fallback (PR #3190). Without a bmc_proxy the Redfish client sends no +# "Forwarded" header, and older mocks 404 every request with +# "no router configured". Do NOT set site_explorer.bmc_proxy in this mode — +# NICo dials each BMC's ClusterIP directly. +# +# * Every pod MUST get disjoint MAC pools (the per-pod TOML overrides below). +# All machine-a-tron instances otherwise derive identical MACs from the +# default pools (02:00:00:00:00:00 / 06:00:00:00:00:00); the pod that +# leases second is rejected on every DHCP with "Network segment mismatch +# for existing MAC address" and simulates nothing. The chart has no +# per-pod MAC knob yet, so the configFiles.matConfigs overrides are the +# only way to set mac_address_pool / hw_mac_address_ranges per pod. +# +# * One NICo network segment per pod CIDR: network_prefixes has a +# one-IPv4-prefix-per-segment constraint, so the pod CIDRs cannot be added +# as two prefixes of one OOB segment. Clone one underlay segment per pod +# (same jsonb_populate_record technique as setup-machine-a-tron.sh scale +# mode), prefix = the pod CIDR, gateway = .1, num_reserved = 1 — the chart +# allocates BMC Service IPs starting at .2, which then lines up with the +# DHCP allocator. +# +# * Vendor-specific factory credential path: hw types whose Redfish vendor +# string is not recognized by libredfish (e.g. wiwynn_gb200_nvl reports +# "WIWYNN") resolve to BMCVendor::Unknown, so the host factory credential +# must be seeded at machines/all_hosts/factory_default/bmc-metadata-items/unknown. +# +# * GB-class hosts (NvidiaGBx00 flow, e.g. wiwynn_gb200_nvl) require +# expected_machines rows with a WORKING BMC credential before exploration +# completes (error NICO-SITEEXPLORER-141 "Missing credential +# expected_machine"). machine-a-tron registers them itself when +# registerExpectedMachines=true, but only on nico-api builds with the +# Machineatron AddExpectedMachine RBAC grant — on older builds the call is +# 403'd and the rows must be inserted directly (see the fallback in +# setup-machine-a-tron.sh; the row password must be the PINNED password +# below, not the factory default). DPU BMCs explore without expected rows. + +certificate: + uris: + - spiffe://nico.local/nico-system/sa/machine-a-tron + +extraEnv: + - name: RUST_LOG + value: info + +global: + imagePullSecrets: + - name: machine-a-tron-pull + +# Build and push the image to the registry of your choice (see the deployment +# guide); the setup script injects these from NICO_IMAGE_REGISTRY/MAT_IMAGE_TAG +# when deploying via setup-machine-a-tron.sh. +image: + pullPolicy: IfNotPresent + repository: "" + tag: "" + +machineATron: + configureBmcProxyHost: "" + hostBmcPassword: NicoSiteRoot1 + dpuBmcPassword: NicoSiteRoot1 + nicoApiUrl: https://nico-api.nico-system.svc.cluster.local:1079 + registerExpectedMachines: true + useSingleBmcMock: true + +bmcServices: + enabled: true + servicePort: 443 + serviceCIDR: + # Requires Kubernetes 1.29+. On older clusters leave false and carve the + # pod CIDRs from the cluster's existing service range (see header). + create: false + +pods: + mat-0: + cidr: "10.233.58.0/23" + machines: + compute: + hwType: wiwynn_gb200_nvl + hostCount: 5 + dpuPerHostCount: 2 + oobDhcpRelayAddress: "10.233.58.1" + adminDhcpRelayAddress: "10.102.0.1" + mat-1: + cidr: "10.233.60.0/23" + machines: + compute: + hwType: wiwynn_gb200_nvl + hostCount: 5 + dpuPerHostCount: 2 + oobDhcpRelayAddress: "10.233.60.1" + adminDhcpRelayAddress: "10.102.0.1" + +resources: + limits: + cpu: "2" + memory: 2Gi + requests: + cpu: 500m + memory: 512Mi + +# Full per-pod TOML overrides. The chart uses these VERBATIM instead of +# generating mat.toml from pods..machines, so they must repeat the +# machine sections — keep them in sync with the pods block above. They exist +# solely to give each pod disjoint MAC pools (see header). +configFiles: + matConfigs: + mat-0: | + # Pod: mat-0 CIDR: 10.233.58.0/23 + carbide_api_url = "https://nico-api.nico-system.svc.cluster.local:1079" + interface = "NOTUSED" + tui_enabled = false + bmc_mock_port = 1266 + use_single_bmc_mock = true + mock_bmc_ssh_server = true + mock_bmc_ssh_port = 2222 + cleanup_on_quit = false + register_expected_machines = true + host_bmc_password = "NicoSiteRoot1" + dpu_bmc_password = "NicoSiteRoot1" + api_refresh_interval = "2s" + + mac_address_pool = { base = "02:00:00:00:00:00", host_bits = 24 } + hw_mac_address_ranges = { base = "06:00:00:00:00:00", host_bits = 32, range_host_bits = 8 } + + [machines.compute] + hw_type = "wiwynn_gb200_nvl" + host_count = 5 + vpc_count = 0 + subnets_per_vpc = 0 + dpu_per_host_count = 2 + dpu_reboot_delay = 1 + host_reboot_delay = 1 + scout_run_interval = "60s" + run_interval_working = "1s" + run_interval_idle = "10s" + network_status_run_interval = "20s" + template_dir = "/opt/machine-a-tron/templates" + oob_dhcp_relay_address = "10.233.58.1" + admin_dhcp_relay_address = "10.102.0.1" + dpus_in_nic_mode = false + mat-1: | + # Pod: mat-1 CIDR: 10.233.60.0/23 + carbide_api_url = "https://nico-api.nico-system.svc.cluster.local:1079" + interface = "NOTUSED" + tui_enabled = false + bmc_mock_port = 1266 + use_single_bmc_mock = true + mock_bmc_ssh_server = true + mock_bmc_ssh_port = 2222 + cleanup_on_quit = false + register_expected_machines = true + host_bmc_password = "NicoSiteRoot1" + dpu_bmc_password = "NicoSiteRoot1" + api_refresh_interval = "2s" + + # Disjoint MAC spaces from mat-0 (outside its 2^24 / 2^32 host spans) + mac_address_pool = { base = "02:00:01:00:00:00", host_bits = 24 } + hw_mac_address_ranges = { base = "06:01:00:00:00:00", host_bits = 32, range_host_bits = 8 } + + [machines.compute] + hw_type = "wiwynn_gb200_nvl" + host_count = 5 + vpc_count = 0 + subnets_per_vpc = 0 + dpu_per_host_count = 2 + dpu_reboot_delay = 1 + host_reboot_delay = 1 + scout_run_interval = "60s" + run_interval_working = "1s" + run_interval_idle = "10s" + network_status_run_interval = "20s" + template_dir = "/opt/machine-a-tron/templates" + oob_dhcp_relay_address = "10.233.60.1" + admin_dhcp_relay_address = "10.102.0.1" + dpus_in_nic_mode = false diff --git a/helm-prereqs/values/machine-a-tron-scale.yaml b/helm-prereqs/values/machine-a-tron-scale.yaml new file mode 100644 index 0000000000..856097927c --- /dev/null +++ b/helm-prereqs/values/machine-a-tron-scale.yaml @@ -0,0 +1,98 @@ +# ============================================================================= +# machine-a-tron-scale.yaml — LARGE-SCALE site values (MetalLB/nginx BMC mode) +# +# For scale testing NICo ingestion with hundreds to thousands of simulated +# hosts. Used by: MAT_MODE=scale setup-machine-a-tron.sh +# See docs/development/machine-a-tron-deployment.md and the chart README's +# "METALLB MODE" section. +# +# How this differs from Override Mode (values/machine-a-tron.yaml): +# - nginxBmcProxy.enabled: true — every simulated BMC gets its own +# LoadBalancer IP (one Service per BMC, chart cap 16384). nginx routes to +# the right mock via "Forwarded: host=". +# - machineATron.useSingleBmcMock: false — the mock registers one router per +# BMC keyed by its DHCP'd IP instead of one combined server. +# - site_explorer.bmc_proxy MUST BE UNSET in nico-core — site-explorer dials +# each BMC's dedicated IP directly (setup script removes it in scale mode). +# - Requires the simulated-oob / simulated-admin networks in the nico-core +# site config (added by the setup script's scale mode): +# [networks.simulated-oob] prefix 10.100.0.0/17 gateway 10.100.0.1 +# [networks.simulated-admin] prefix 10.102.0.0/18 gateway 10.102.0.1 +# The MetalLB ipRange below MUST sit inside the simulated-oob prefix so +# NICo-allocated BMC IPs match the MetalLB Service IPs (allocation is +# sequential on both sides). +# ============================================================================= + +## Registry-agnostic (like the NICo REST values): setup-machine-a-tron.sh +## injects image.repository = ${NICO_IMAGE_REGISTRY}/machine-a-tron and +## image.tag = ${MAT_IMAGE_TAG} at install time. Set them here only for +## direct helm installs without the script. +image: + repository: "" + tag: "" + pullPolicy: IfNotPresent + +global: + imagePullSecrets: + - name: machine-a-tron-pull + +# Same SPIFFE identity requirement as Override Mode (see machine-a-tron.yaml). +certificate: + uris: + - "spiffe://nico.local/nico-system/sa/machine-a-tron" + +machineATron: + nicoApiUrl: "https://nico-api.nico-system.svc.cluster.local:1079" + # Scale mode: no bmc_proxy anywhere — leave unset. + configureBmcProxyHost: "" + registerExpectedMachines: true + # true = SHARED REGISTRY mode (crates/machine-a-tron/src/main.rs): one + # server on bmc_mock_port routes to per-BMC mock routers via the + # "Forwarded: host=" header nginx injects. This is REQUIRED in K8s. + # false would make each mock bind its real BMC IP on the pod netns (bare- + # metal style) — nothing listens on 1266 and the pod crash-loops on probes. + useSingleBmcMock: true + +# Larger resources: one tokio task per host + per DPU; at thousands of +# machines the FSM/API chatter is CPU-bound. +resources: + limits: + cpu: "8" + memory: 8Gi + requests: + cpu: "2" + memory: 2Gi + +pods: + default: + machines: + # disable the chart-default example group (deep-merge would keep it) + rack-machines: null + dell-hosts: + hwType: dell_poweredge_r750 + # STAGE 1 default: 100 hosts × 2 DPUs = 300 BMC endpoints / LB Services. + # Raise per stage (1000 → 4500). Chart caps total BMCs at 16384. + # Override per run: HOST_COUNT / DPU_PER_HOST env to the setup script. + hostCount: 100 + dpuPerHostCount: 2 + vpcCount: 0 + subnetsPerVpc: 0 + dpuRebootDelay: 1 + hostRebootDelay: 1 + # Slower cadences than the dev profile — at thousands of machines the + # aggregate API/CPU load scales linearly with these. + scoutRunInterval: "300s" + runIntervalWorking: "5s" + runIntervalIdle: "60s" + networkStatusRunInterval: "120s" + templateDir: "/opt/machine-a-tron/templates" + # Gateways of the SIMULATED networks (added to nico-core by scale mode). + oobDhcpRelayAddress: "10.100.0.1" + adminDhcpRelayAddress: "10.102.0.1" + networkVirtualizationType: "" + dpusInNicMode: false + +extraEnv: + - name: RUST_LOG + value: "info" + diff --git a/helm-prereqs/values/machine-a-tron.yaml b/helm-prereqs/values/machine-a-tron.yaml new file mode 100644 index 0000000000..d25d4c385b --- /dev/null +++ b/helm-prereqs/values/machine-a-tron.yaml @@ -0,0 +1,105 @@ +# ============================================================================= +# machine-a-tron.yaml — site values template +# +# Copy this file and fill in site-specific values before deploying, OR let +# setup-machine-a-tron.sh consume it directly (it fills the site-specific +# fields from the running nico-core site config). +# See docs/development/machine-a-tron-deployment.md for the full guide. +# +# Deployment mode: Override Mode (nginxBmcProxy.enabled: false) +# site-explorer's Redfish calls are redirected to machine-a-tron's mock BMC. +# Not compatible with real hardware — use only on simulation-only clusters. +# +# IMPORTANT — the redirect is driven by nico-core, not this chart. nico-core's +# site_explorer config must contain (note the CROSS-NAMESPACE FQDN): +# [site_explorer] +# bmc_proxy = "nico-machine-a-tron-bmc-mock.nico-mat.svc.cluster.local:1266" +# (Field is `bmc_proxy`, a single "host:port" string. The older +# override_target_ip/override_target_port fields are DEPRECATED, and +# override_target_host was never a valid field. setup-machine-a-tron.sh sets +# bmc_proxy for you.) The FQDN is REQUIRED: site-explorer runs in nico-system +# and cannot resolve the bare service name (which resolves against its own +# namespace) — a bare name yields a Redfish "connection refused". Setting +# bmc_proxy at launch also makes allow_changing_bmc_proxy default to true, +# which is what lets configureBmcProxyHost (below) work at runtime. +# ============================================================================= + +## Registry-agnostic (like the NICo REST values): setup-machine-a-tron.sh +## injects image.repository = ${NICO_IMAGE_REGISTRY}/machine-a-tron and +## image.tag = ${MAT_IMAGE_TAG} at install time. Set them here only for +## direct helm installs without the script. +image: + repository: "" + tag: "" + pullPolicy: IfNotPresent + +global: + imagePullSecrets: + - name: machine-a-tron-pull + +# Override the SPIFFE URI so nico-api can authorize machine-a-tron calls. +# nico-api recognizes paths under /nico-system/sa/ and the RBAC rule for +# Machineatron expects SpiffeServiceIdentifier("machine-a-tron"). Without +# this override the cert-manager-issued cert uses the deployment namespace +# (/nico-mat/sa/nico-machine-a-tron) which is not in nico-api's trusted +# service_base_paths list and results in a 403 on every API call. +certificate: + uris: + - "spiffe://nico.local/nico-system/sa/machine-a-tron" + +machineATron: + nicoApiUrl: "https://nico-api.nico-system.svc.cluster.local:1079" + # When set, machine-a-tron calls nico-api set_dynamic_config to point + # site_explorer.bmc_proxy at ":" at runtime. + # This ONLY succeeds if nico-core has allow_changing_bmc_proxy=true (which is + # the default once nico-core launches with bmc_proxy set). It is a + # belt-and-suspenders companion to the nico-core bmc_proxy setting — harmless + # when both are set to the same target. + # MUST be the cross-namespace FQDN (site-explorer is in nico-system and cannot + # resolve the bare name). Update the ".nico-mat." segment if you deploy to a + # different namespace. + configureBmcProxyHost: "nico-machine-a-tron-bmc-mock.nico-mat.svc.cluster.local" + # Auto-register each mock host as an expected_machine (matched by BMC MAC). + # REQUIRED for end-to-end: site-explorer's MachineCreator refuses to create a + # managed host when no expected_machines row matches the discovered BMC MAC + # ("Refusing to create managed host, expected machines entry not found"). + registerExpectedMachines: true + +pods: + default: + machines: + # disable the chart-default example group (deep-merge would keep it) + rack-machines: null + dell-hosts: + hwType: dell_poweredge_r750 + # SIZING: machine-a-tron allocates one OOB DHCP IP per BMC interface — + # 1 per host + 1 per DPU. Total = hostCount + hostCount*dpuPerHostCount. + # This MUST fit the usable addresses in the OOB/underlay DHCP pool of your + # nico-core site config, or discovery reports "No IP addresses left in + # prefix ..." and machines never register. + # e.g. a /28 pool with a gateway and a few reserved addresses yields + # ~10 usable → hostCount:3 * dpuPerHostCount:2 = 9 fits; 5*2=15 does not. + # setup-machine-a-tron.sh estimates the pool size and warns if you overflow. + hostCount: 3 + dpuPerHostCount: 2 + vpcCount: 0 + subnetsPerVpc: 0 + dpuRebootDelay: 1 + hostRebootDelay: 1 + scoutRunInterval: "60s" + runIntervalWorking: "1s" + runIntervalIdle: "10s" + networkStatusRunInterval: "20s" + templateDir: "/opt/machine-a-tron/templates" + # Set to the gateway of the OOB/underlay network in nico-core site config. + # (setup-machine-a-tron.sh fills this from the running nico-core config.) + oobDhcpRelayAddress: "FILL_IN" + # Set to the gateway of the admin network in nico-core site config. + adminDhcpRelayAddress: "FILL_IN" + networkVirtualizationType: "" + dpusInNicMode: false + +extraEnv: + - name: RUST_LOG + value: "info" + diff --git a/helm/Chart.yaml b/helm/Chart.yaml index 12a69869c4..e5e07b5738 100644 --- a/helm/Chart.yaml +++ b/helm/Chart.yaml @@ -44,3 +44,6 @@ dependencies: - name: unbound version: "0.1.0" condition: unbound.enabled + - name: nico-machine-a-tron + version: "0.1.0" + condition: nico-machine-a-tron.enabled diff --git a/helm/charts/nico-machine-a-tron/Chart.yaml b/helm/charts/nico-machine-a-tron/Chart.yaml new file mode 100644 index 0000000000..5f6ceac8d6 --- /dev/null +++ b/helm/charts/nico-machine-a-tron/Chart.yaml @@ -0,0 +1,12 @@ +apiVersion: v2 +name: nico-machine-a-tron +description: Helm chart for Machine-A-Tron - a mock machine simulator for NICo development +type: application +version: 0.1.0 +appVersion: "latest" +keywords: + - nico + - machine-a-tron + - mock + - testing + - development diff --git a/helm/charts/nico-machine-a-tron/README.md b/helm/charts/nico-machine-a-tron/README.md new file mode 100644 index 0000000000..cd9397f2e0 --- /dev/null +++ b/helm/charts/nico-machine-a-tron/README.md @@ -0,0 +1,314 @@ +# Machine-A-Tron Helm Chart + +Helm chart for deploying Machine-A-Tron - a mock machine simulator for NICo testing. + +## Overview + +Machine-A-Tron creates simulated bare-metal machines that behave like real hosts, allowing you to: +- Test NICo without physical hardware +- Simulate multiple hosts, DPUs, switches and power shelves +- Perform load testing at scale (multiple pods, thousands of BMCs) +- Run simulations alongside real hardware + +## Deployment Modes + +| Mode | Use Case | Real HW Compatible | Network Setup | +|------|----------|-------------------|---------------| +| **Override Mode** | Development | No | Simple - single endpoint | +| **ClusterIP Mode** | Scale testing | Yes | Per-BMC ClusterIP services | + +--- + +## Mode 1: Override Mode (Development) + +**Use for development environments where only simulated machines are needed.** + +NICo's Site-Explorer is configured to redirect ALL Redfish calls to machine-a-tron. +Simple but **incompatible with real hardware**. + +### Setup + +```bash +helm upgrade --install nico ./helm \ + --namespace nico-mat \ + --set nico-machine-a-tron.enabled=true \ + --set nico-machine-a-tron.pods.default.machines.rack-machines.hostCount=10 \ + --set nico-machine-a-tron.pods.default.machines.rack-machines.dpuPerHostCount=2 +``` + +**NICo Site Config:** +```toml +[site_explorer] +override_target_host = "nico-machine-a-tron-bmc-mock" +override_target_port = 1266 +``` + +--- + +## Mode 2: ClusterIP Mode (Scale Testing) + +**Use for load testing environments where simulated machines run alongside real hardware.** + +Each simulated BMC gets a dedicated ClusterIP service. Supports multi-pod deployments. + +### Architecture + +```mermaid +flowchart TB + subgraph CIDR["ServiceCIDR: 10.100.0.0/20 (reserved)"] + direction TB + subgraph pod0["pod-0 (~14 racks)"] + cidr0["10.100.0.0/22
1021 IPs"] + end + subgraph pod1["pod-1 (~14 racks)"] + cidr1["10.100.4.0/22
1021 IPs"] + end + subgraph pod2["pod-2 (~14 racks)"] + cidr2["10.100.8.0/22
1021 IPs"] + end + end + + SE[Site-Explorer] --> cidr0 + SE --> cidr1 + SE --> cidr2 +``` + +### Prerequisites + +- **Kubernetes 1.29+** (ServiceCIDR API support) +- IP range within cluster's default service CIDR (e.g., `10.96.0.0/12`) + +### Single Pod Setup + +```yaml +# values.yaml +bmcServices: + enabled: true + serviceCIDR: + create: true + +pods: + default: + cidr: "10.100.0.0/22" # 1021 IPs for ~14 racks + machines: + compute: + hwType: wiwynn_gb200_nvl + hostCount: 252 # 18 trays × 14 racks + dpuPerHostCount: 2 # 2 BF3 per tray → 504 DPU BMCs + oobDhcpRelayAddress: "10.100.0.1" + switches: + hwType: nvidia_switch_nd5200_ld + hostCount: 126 # 9 switches × 14 racks + dpuPerHostCount: 0 + oobDhcpRelayAddress: "10.100.0.1" + power: + hwType: liteon_power_shelf + hostCount: 112 # 8 shelves × 14 racks + dpuPerHostCount: 0 + oobDhcpRelayAddress: "10.100.0.1" +``` + +**Total BMCs:** 252 + 504 + 126 + 112 = **994 BMCs** (fits in /22) + +### Multi-Pod Setup (Large Scale) + +For deployments exceeding 1021 BMCs, use multiple pods with separate CIDRs: + +```yaml +# values.yaml +bmcServices: + enabled: true + serviceCIDR: + create: true + cidr: "10.100.0.0/20" # Covers all pods + +pods: + pod-0: + cidr: "10.100.0.0/22" + machines: + compute: + hwType: wiwynn_gb200_nvl + hostCount: 252 + dpuPerHostCount: 2 + oobDhcpRelayAddress: "10.100.0.1" + switches: + hwType: nvidia_switch_nd5200_ld + hostCount: 126 + oobDhcpRelayAddress: "10.100.0.1" + power: + hwType: liteon_power_shelf + hostCount: 112 + oobDhcpRelayAddress: "10.100.0.1" + + pod-1: + cidr: "10.100.4.0/22" + machines: + compute: + hwType: wiwynn_gb200_nvl + hostCount: 252 + dpuPerHostCount: 2 + oobDhcpRelayAddress: "10.100.4.1" + switches: + hwType: nvidia_switch_nd5200_ld + hostCount: 126 + oobDhcpRelayAddress: "10.100.4.1" + power: + hwType: liteon_power_shelf + hostCount: 112 + oobDhcpRelayAddress: "10.100.4.1" + + pod-2: + cidr: "10.100.8.0/22" + machines: + # ... same pattern +``` + +### What Gets Created + +**Per pod:** +| Resource | Name Pattern | +|----------|--------------| +| Deployment | `nico-machine-a-tron-pod-0` | +| ConfigMap | `nico-machine-a-tron-pod-0-config-files` | +| Certificate | `nico-machine-a-tron-pod-0-certificate` | +| Service | `nico-machine-a-tron-pod-0-bmc-mock` | + +**Per BMC:** +| Resource | Name Pattern | +|----------|--------------| +| ClusterIP Service | `nico-machine-a-tron-bmc-10-100-0-2` | + +**Cluster-scoped:** +| Resource | Name | +|----------|------| +| ServiceCIDR | `nico-machine-a-tron-bmc-cidr` | + +### Scale Guidelines + +**BMC count per GB200 NVL72 rack:** +| Component | Count | BMCs per Unit | Total | +|-----------|-------|---------------|-------| +| Compute trays | 18 | 3 (1 tray + 2 BF3) | 54 | +| NVLink switches | 9 | 1 | 9 | +| Power shelves | 8 | 1 | 8 | +| **Total per rack** | | | **71** | + +**CIDR sizing:** +| CIDR | Usable IPs | Racks per Pod | +|------|------------|---------------| +| /24 | 253 | ~3 | +| /23 | 509 | ~7 | +| /22 | 1021 | ~14 | +| /21 | 2045 | ~28 | + +### NICo Configuration + +**DO NOT set `override_target_host`** - let NICo connect to actual BMC IPs: + +```toml +[site_explorer] +enabled = true +create_machines = true +# override_target_host = ... # DO NOT SET +``` + +**Network config per pod:** +```toml +[networks.pod-0-oob] +type = "underlay" +prefix = "10.100.0.0/22" +gateway = "10.100.0.1" + +[networks.pod-1-oob] +type = "underlay" +prefix = "10.100.4.0/22" +gateway = "10.100.4.1" +``` + +--- + +## Configuration Reference + +### Pod Configuration + +```yaml +pods: + : + cidr: "" # Required for bmcServices mode + machines: + : + hwType: wiwynn_gb200_nvl + hostCount: 10 + dpuPerHostCount: 2 + oobDhcpRelayAddress: "10.100.0.1" + adminDhcpRelayAddress: "192.168.176.1" + # ... other machine settings +``` + +### BMC Services Configuration + +```yaml +bmcServices: + enabled: false + servicePort: 443 # External HTTPS port + serviceCIDR: + create: true + name: "" # Defaults to -bmc-cidr + cidr: "" # Auto-detected for single pod, required for multi-pod +``` + +### Supported Hardware Types + +| Type | Description | +|------|-------------| +| `supermicro_gb300_nvl` | Supermicro GB300 NVL | +| `nvidia_dgx_gb300` | NVIDIA DGX GB300 | +| `nvidia_dgx_h100` | NVIDIA DGX H100 | +| `wiwynn_gb200_nvl` | Wiwynn GB200 NVL | +| `lenovo_gb300_nvl` | Lenovo GB300 NVL | +| `dell_poweredge_r750` | Dell PowerEdge R750 | +| `liteon_power_shelf` | Liteon Power Shelf | +| `nvidia_switch_nd5200_ld` | NVIDIA ND5200 Switch | +| `generic_ami` | Generic AMI BMC | +| `generic_supermicro` | Generic Supermicro BMC | + +--- + +## Troubleshooting + +### ServiceCIDR Not Ready + +```bash +kubectl get servicecidr -o wide +``` + +Causes: +- CIDR outside cluster's default service CIDR +- Kubernetes version < 1.29 + +### Service IP Allocation Failed + +```bash +kubectl -n nico-mat describe svc nico-machine-a-tron-bmc-10-100-0-2 +``` + +Causes: +- IP already in use +- ServiceCIDR not ready + +### Pod Not Receiving Traffic + +Verify selector labels match: +```bash +# Check service selector +kubectl -n nico-mat get svc nico-machine-a-tron-bmc-10-100-0-2 -o jsonpath='{.spec.selector}' + +# Check pod labels +kubectl -n nico-mat get pods -l nvidia-infra-controller/pod-name=pod-0 --show-labels +``` + +### View Generated Config + +```bash +kubectl -n nico-mat get cm nico-machine-a-tron-pod-0-config-files -o yaml +``` diff --git a/helm/charts/nico-machine-a-tron/templates/_helpers.tpl b/helm/charts/nico-machine-a-tron/templates/_helpers.tpl new file mode 100644 index 0000000000..b27a1fdf47 --- /dev/null +++ b/helm/charts/nico-machine-a-tron/templates/_helpers.tpl @@ -0,0 +1,99 @@ +{{/* +Allow the release namespace to be overridden for multi-namespace deployments. +*/}} +{{- define "nico-machine-a-tron.namespace" -}} +{{- default .Release.Namespace .Values.namespaceOverride | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Resource name prefix. Defaults to the chart name; override with nameOverride. +*/}} +{{- define "nico-machine-a-tron.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{- define "nico-machine-a-tron.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +What image to use: Use subchart-local image if defined, fall back on global +image. In devspace deployments, {{ include "nico-machine-a-tron.name" . }} gets its own image. +In other deployments, the main nico image contains all binaries. +*/}} +{{- define "nico-machine-a-tron.image" -}} +{{- if not (eq (toString (.Values.image.repository | default "")) "") }} +{{- .Values.image.repository }}:{{ .Values.image.tag | default "latest" }} +{{- else if and .Values.global.image (not (eq (toString (.Values.global.image.repository | default "")) "")) (not (eq (toString (.Values.global.image.tag | default "")) "")) }} +{{- .Values.global.image.repository }}:{{ .Values.global.image.tag }} +{{- else }} +{{- "nico:latest" }} +{{- end }} +{{- end }} + +{{- define "nico-machine-a-tron.labels" -}} +helm.sh/chart: {{ include "nico-machine-a-tron.chart" . }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +app.kubernetes.io/part-of: site-controller +app.kubernetes.io/name: {{ include "nico-machine-a-tron.name" . }} +app.kubernetes.io/component: machine-a-tron +{{- end }} + +{{- define "nico-machine-a-tron.selectorLabels" -}} +app.kubernetes.io/name: {{ include "nico-machine-a-tron.name" . }} +app.kubernetes.io/component: machine-a-tron +{{- end }} + +{{- define "nico-machine-a-tron.certificateSpec" -}} +duration: {{ .global.certificate.duration }} +renewBefore: {{ .global.certificate.renewBefore }} +commonName: {{ printf "%s.%s.svc.cluster.local" (.cert.serviceName | default .svcName) (.cert.identityNamespace | default .namespace) }} +dnsNames: +{{- if .cert.dnsNames }} +{{- range .cert.dnsNames }} + - {{ . }} +{{- end }} +{{- else }} + - {{ printf "%s.%s.svc.cluster.local" (.cert.serviceName | default .svcName) (.cert.identityNamespace | default .namespace) }} +{{- if ne (toString .cert.includeShortDnsName) "false" }} + - {{ printf "%s.%s" (.cert.serviceName | default .svcName) (.cert.identityNamespace | default .namespace) }} +{{- end }} +{{- range .cert.extraDnsNames | default list }} + - {{ . }} +{{- end }} +{{- end }} +uris: +{{- if .cert.uris }} +{{- range .cert.uris }} + - {{ . }} +{{- end }} +{{- else }} + - {{ printf "spiffe://%s/%s/sa/%s" .global.spiffe.trustDomain (.cert.identityNamespace | default .namespace) (.cert.spiffeServiceName | default .cert.serviceName | default .svcName) }} +{{- range .cert.extraUris | default list }} + - {{ . }} +{{- end }} +{{- end }} +privateKey: + algorithm: {{ .global.certificate.privateKey.algorithm }} + size: {{ .global.certificate.privateKey.size }} +issuerRef: + kind: {{ .global.certificate.issuerRef.kind }} + name: {{ .global.certificate.issuerRef.name }} + group: {{ .global.certificate.issuerRef.group }} +secretName: {{ .name }} +{{- end }} + +{{- define "nico-machine-a-tron.serviceMonitorSpec" -}} +endpoints: + - honorLabels: false + interval: {{ .monitor.interval }} + port: {{ .port }} + scheme: http + scrapeTimeout: {{ .monitor.scrapeTimeout }} +namespaceSelector: + matchNames: + - {{ .namespace }} +selector: + matchLabels: + app.kubernetes.io/metrics: {{ .name }} +{{- end }} diff --git a/helm/charts/nico-machine-a-tron/templates/bmc-service-cidr.yaml b/helm/charts/nico-machine-a-tron/templates/bmc-service-cidr.yaml new file mode 100644 index 0000000000..ea24422efd --- /dev/null +++ b/helm/charts/nico-machine-a-tron/templates/bmc-service-cidr.yaml @@ -0,0 +1,55 @@ +{{- if .Values.bmcServices.enabled }} +{{- if .Values.bmcServices.serviceCIDR.create }} +{{- $root := . }} +{{- $baseName := include "nico-machine-a-tron.name" . }} + +{{/* +Determine the ServiceCIDR to use: +1. If explicitly set in bmcServices.serviceCIDR.cidr, use that +2. Otherwise, use the CIDR from the first pod (single-pod case) +3. For multi-pod with different CIDRs, user must set serviceCIDR.cidr explicitly +*/}} +{{- $serviceCIDR := "" }} +{{- if .Values.bmcServices.serviceCIDR.cidr }} +{{- $serviceCIDR = .Values.bmcServices.serviceCIDR.cidr }} +{{- else }} +{{/* Collect all pod CIDRs */}} +{{- $cidrs := list }} +{{- range $podName, $podConfig := .Values.pods }} +{{- if $podConfig.cidr }} +{{- $cidrs = append $cidrs $podConfig.cidr }} +{{- end }} +{{- end }} +{{- if eq (len $cidrs) 1 }} +{{- $serviceCIDR = index $cidrs 0 }} +{{- else if gt (len $cidrs) 1 }} +{{- fail "Multiple pods with different CIDRs detected. Set bmcServices.serviceCIDR.cidr to a CIDR that covers all pod CIDRs." }} +{{- end }} +{{- end }} + +{{- if $serviceCIDR }} +--- +## ServiceCIDR reserves a dedicated IP range for BMC services. +## This prevents Kubernetes from allocating these IPs to other services. +## Requires Kubernetes 1.29+ (beta in 1.31+). +## +## The ServiceCIDR is cluster-scoped and persists independently of this release. +## Deleting the helm release will NOT delete the ServiceCIDR to prevent +## disruption to other releases using the same CIDR. +apiVersion: networking.k8s.io/v1beta1 +kind: ServiceCIDR +metadata: + name: {{ .Values.bmcServices.serviceCIDR.name | default (printf "%s-bmc-cidr" $baseName) }} + labels: + {{- include "nico-machine-a-tron.labels" . | nindent 4 }} + app.kubernetes.io/component: bmc-service-cidr + annotations: + helm.sh/resource-policy: keep + nvidia-infra-controller/purpose: "Reserved IP range for machine-a-tron BMC services" + nvidia-infra-controller/cidr: "{{ $serviceCIDR }}" +spec: + cidrs: + - {{ $serviceCIDR }} +{{- end }} +{{- end }} +{{- end }} diff --git a/helm/charts/nico-machine-a-tron/templates/bmc-services.yaml b/helm/charts/nico-machine-a-tron/templates/bmc-services.yaml new file mode 100644 index 0000000000..54f2353f2a --- /dev/null +++ b/helm/charts/nico-machine-a-tron/templates/bmc-services.yaml @@ -0,0 +1,131 @@ +{{- if .Values.bmcServices.enabled }} +{{- $root := . }} +{{- $namespace := include "nico-machine-a-tron.namespace" . }} +{{- $baseName := include "nico-machine-a-tron.name" . }} +{{- $servicePort := .Values.bmcServices.servicePort | default 443 }} +{{- $targetPort := .Values.service.bmcMock.port | default 1266 }} + +{{/* +Count pods that have CIDR configured (for naming and selector logic) +*/}} +{{- $podsWithCIDR := 0 }} +{{- range $podName, $podConfig := .Values.pods }} +{{- if $podConfig.cidr }} +{{- $podsWithCIDR = add $podsWithCIDR 1 }} +{{- end }} +{{- end }} + +{{- range $podName, $podConfig := .Values.pods }} +{{/* +Skip pods without CIDR (e.g., the default pod when custom pods are defined) +*/}} +{{- if not $podConfig.cidr }} +{{- continue }} +{{- end }} + +{{- $name := $baseName }} +{{- if gt $podsWithCIDR 1 }} +{{- $name = printf "%s-%s" $baseName $podName }} +{{- end }} + +{{/* +Parse CIDR and calculate available IPs +*/}} +{{- $cidrParts := split "/" $podConfig.cidr }} +{{- $networkIP := $cidrParts._0 }} +{{- $prefixLen := $cidrParts._1 | int }} +{{- $ipParts := split "." $networkIP }} +{{- $octet1 := $ipParts._0 | int }} +{{- $octet2 := $ipParts._1 | int }} +{{- $octet3 := $ipParts._2 | int }} +{{- $octet4 := $ipParts._3 | int }} + +{{/* +Calculate total IPs and usable IPs: 2^(32-prefix) - 3 (network, gateway, broadcast) +*/}} +{{- $hostBits := sub 32 $prefixLen }} +{{- $totalIPs := 1 }} +{{- range $i := until (int $hostBits) }} +{{- $totalIPs = mul $totalIPs 2 }} +{{- end }} +{{- $usableIPs := sub $totalIPs 3 }} + +{{/* +Calculate total BMCs needed for this pod +*/}} +{{- $totalBMCs := 0 }} +{{- range $sectionName, $section := $podConfig.machines }} +{{- if $section }} +{{- $hostCount := $section.hostCount | default 1 }} +{{- $dpuCount := $section.dpuPerHostCount | default 0 }} +{{- $sectionBMCs := add $hostCount (mul $hostCount $dpuCount) }} +{{- $totalBMCs = add $totalBMCs $sectionBMCs }} +{{- end }} +{{- end }} + +{{/* +Validate we have enough IPs +*/}} +{{- if gt (int $totalBMCs) (int $usableIPs) }} +{{- fail (printf "INSUFFICIENT IP CAPACITY for pod '%s': %d BMCs requested but CIDR %s only has %d usable IPs" $podName $totalBMCs $podConfig.cidr $usableIPs) }} +{{- end }} + +{{/* +Generate ClusterIP services for each BMC +Services start at .2 (skipping .0 network and .1 gateway) +*/}} +{{- $startOctet4 := add $octet4 2 }} +{{- range $bmcIndex := until (int $totalBMCs) }} +{{- $offset := $bmcIndex }} +{{- $o4 := add $startOctet4 $offset }} +{{- $o3 := $octet3 }} +{{- $o2 := $octet2 }} +{{- $o1 := $octet1 }} + +{{/* Handle octet overflows */}} +{{- if gt $o4 255 }} +{{- $o3 = add $o3 (div $o4 256) }} +{{- $o4 = mod $o4 256 }} +{{- end }} +{{- if gt $o3 255 }} +{{- $o2 = add $o2 (div $o3 256) }} +{{- $o3 = mod $o3 256 }} +{{- end }} +{{- if gt $o2 255 }} +{{- $o1 = add $o1 (div $o2 256) }} +{{- $o2 = mod $o2 256 }} +{{- end }} + +{{- $currentIP := printf "%d.%d.%d.%d" (int $o1) (int $o2) (int $o3) (int $o4) }} +{{- $safeName := $currentIP | replace "." "-" }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ $baseName }}-bmc-{{ $safeName }} + namespace: {{ $namespace }} + labels: + {{- include "nico-machine-a-tron.labels" $root | nindent 4 }} + app.kubernetes.io/component: bmc-service + nvidia-infra-controller/bmc-ip: "{{ $currentIP }}" + nvidia-infra-controller/bmc-index: "{{ $bmcIndex }}" + nvidia-infra-controller/pod-name: {{ $podName | quote }} + annotations: + nvidia-infra-controller/purpose: "ClusterIP service for simulated BMC" + nvidia-infra-controller/pod-cidr: "{{ $podConfig.cidr }}" +spec: + type: ClusterIP + clusterIP: {{ $currentIP }} + selector: + {{- include "nico-machine-a-tron.selectorLabels" $root | nindent 4 }} + {{- if gt $podsWithCIDR 1 }} + nvidia-infra-controller/pod-name: {{ $podName | quote }} + {{- end }} + ports: + - name: https + port: {{ $servicePort }} + targetPort: {{ $targetPort }} + protocol: TCP +{{- end }} +{{- end }} +{{- end }} diff --git a/helm/charts/nico-machine-a-tron/templates/certificate.yaml b/helm/charts/nico-machine-a-tron/templates/certificate.yaml new file mode 100644 index 0000000000..17d9401239 --- /dev/null +++ b/helm/charts/nico-machine-a-tron/templates/certificate.yaml @@ -0,0 +1,55 @@ +{{- $root := . }} +{{- $namespace := include "nico-machine-a-tron.namespace" . }} +{{- $baseName := include "nico-machine-a-tron.name" . }} + +{{/* +Count active pods (same logic as deployment.yaml) +*/}} +{{- $activePods := 0 }} +{{- range $podName, $podConfig := .Values.pods }} +{{- if $root.Values.bmcServices.enabled }} +{{- if $podConfig.cidr }} +{{- $activePods = add $activePods 1 }} +{{- end }} +{{- else }} +{{- if $podConfig.machines }} +{{- $activePods = add $activePods 1 }} +{{- end }} +{{- end }} +{{- end }} + +{{- range $podName, $podConfig := .Values.pods }} +{{/* +Skip conditions (same as deployment.yaml) +*/}} +{{- if $root.Values.bmcServices.enabled }} +{{- if not $podConfig.cidr }} +{{- continue }} +{{- end }} +{{- else }} +{{- if not $podConfig.machines }} +{{- continue }} +{{- end }} +{{- end }} + +{{- $name := $baseName }} +{{- if gt $activePods 1 }} +{{- $name = printf "%s-%s" $baseName $podName }} +{{- end }} +--- +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: {{ $name }}-certificate + namespace: {{ $namespace }} + labels: + {{- include "nico-machine-a-tron.labels" $root | nindent 4 }} + {{- if gt $activePods 1 }} + nvidia-infra-controller/pod-name: {{ $podName | quote }} + {{- end }} +spec: + {{- $bmcMockDns := printf "%s-bmc-mock.%s.svc.cluster.local" $name $namespace }} + {{- $extraDns := prepend ($root.Values.certificate.extraDnsNames | default list) $bmcMockDns }} + {{- $certWithExtra := merge (dict "extraDnsNames" $extraDns) $root.Values.certificate }} + {{- include "nico-machine-a-tron.certificateSpec" (dict "name" (printf "%s-certificate" $name) "svcName" $name "namespace" $namespace "cert" $certWithExtra "global" $root.Values.global) | nindent 2 }} +{{- end }} diff --git a/helm/charts/nico-machine-a-tron/templates/configmap.yaml b/helm/charts/nico-machine-a-tron/templates/configmap.yaml new file mode 100644 index 0000000000..79e4f48ae8 --- /dev/null +++ b/helm/charts/nico-machine-a-tron/templates/configmap.yaml @@ -0,0 +1,119 @@ +{{- $root := . }} +{{- $namespace := include "nico-machine-a-tron.namespace" . }} +{{- $baseName := include "nico-machine-a-tron.name" . }} + +{{/* +Count active pods (same logic as deployment.yaml) +*/}} +{{- $activePods := 0 }} +{{- range $podName, $podConfig := .Values.pods }} +{{- if $root.Values.bmcServices.enabled }} +{{- if $podConfig.cidr }} +{{- $activePods = add $activePods 1 }} +{{- end }} +{{- else }} +{{- if $podConfig.machines }} +{{- $activePods = add $activePods 1 }} +{{- end }} +{{- end }} +{{- end }} + +{{/* Machine defaults */}} +{{- $defaults := .Values.machineDefaults | default dict }} + +{{- range $podName, $podConfig := .Values.pods }} +{{/* +Skip conditions (same as deployment.yaml) +*/}} +{{- if $root.Values.bmcServices.enabled }} +{{- if not $podConfig.cidr }} +{{- continue }} +{{- end }} +{{- else }} +{{- if not $podConfig.machines }} +{{- continue }} +{{- end }} +{{- end }} + +{{- $name := $baseName }} +{{- if gt $activePods 1 }} +{{- $name = printf "%s-%s" $baseName $podName }} +{{- end }} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ $name }}-config-files + namespace: {{ $namespace }} + labels: + {{- include "nico-machine-a-tron.labels" $root | nindent 4 }} + {{- if gt $activePods 1 }} + nvidia-infra-controller/pod-name: {{ $podName | quote }} + {{- end }} +data: + mat.toml: | + {{- if and $root.Values.configFiles.matConfigs (index $root.Values.configFiles.matConfigs $podName) }} + {{- index $root.Values.configFiles.matConfigs $podName | nindent 4 }} + {{- else }} + # Machine-A-Tron configuration - generated by Helm + # Pod: {{ $podName }} + {{- if $podConfig.cidr }} + # CIDR: {{ $podConfig.cidr }} + {{- end }} + carbide_api_url = {{ $root.Values.machineATron.nicoApiUrl | quote }} + interface = {{ $root.Values.machineATron.interface | quote }} + tui_enabled = {{ $root.Values.machineATron.tuiEnabled }} + {{- if $root.Values.machineATron.logFile }} + log_file = {{ $root.Values.machineATron.logFile | quote }} + {{- end }} + bmc_mock_port = {{ $root.Values.service.bmcMock.port }} + use_single_bmc_mock = {{ $root.Values.machineATron.useSingleBmcMock }} + mock_bmc_ssh_server = {{ $root.Values.machineATron.mockBmcSshServer }} + {{- if $root.Values.machineATron.mockBmcSshPort }} + mock_bmc_ssh_port = {{ $root.Values.machineATron.mockBmcSshPort }} + {{- end }} + {{- if $root.Values.machineATron.configureBmcProxyHost }} + configure_carbide_bmc_proxy_host = {{ $root.Values.machineATron.configureBmcProxyHost | quote }} + {{- end }} + {{- if $root.Values.persistence.enabled }} + persist_dir = {{ $root.Values.machineATron.persistDir | quote }} + {{- end }} + cleanup_on_quit = {{ $root.Values.machineATron.cleanupOnQuit }} + register_expected_machines = {{ $root.Values.machineATron.registerExpectedMachines }} + {{- if $root.Values.machineATron.hostBmcPassword }} + host_bmc_password = {{ $root.Values.machineATron.hostBmcPassword | quote }} + {{- end }} + {{- if $root.Values.machineATron.dpuBmcPassword }} + dpu_bmc_password = {{ $root.Values.machineATron.dpuBmcPassword | quote }} + {{- end }} + api_refresh_interval = {{ $root.Values.machineATron.apiRefreshInterval | quote }} + + {{- range $sectionName, $section := $podConfig.machines }} + {{- if $section }} + + [machines.{{ $sectionName }}] + {{- if $section.hwType }} + hw_type = {{ $section.hwType | quote }} + {{- end }} + host_count = {{ $section.hostCount | default 1 }} + vpc_count = {{ $section.vpcCount | default $defaults.vpcCount | default 0 }} + subnets_per_vpc = {{ $section.subnetsPerVpc | default $defaults.subnetsPerVpc | default 0 }} + dpu_per_host_count = {{ $section.dpuPerHostCount | default 0 }} + dpu_reboot_delay = {{ $section.dpuRebootDelay | default $defaults.dpuRebootDelay | default 1 }} + host_reboot_delay = {{ $section.hostRebootDelay | default $defaults.hostRebootDelay | default 1 }} + scout_run_interval = {{ $section.scoutRunInterval | default $defaults.scoutRunInterval | default "60s" | quote }} + run_interval_working = {{ $section.runIntervalWorking | default $defaults.runIntervalWorking | default "1s" | quote }} + run_interval_idle = {{ $section.runIntervalIdle | default $defaults.runIntervalIdle | default "10s" | quote }} + network_status_run_interval = {{ $section.networkStatusRunInterval | default $defaults.networkStatusRunInterval | default "20s" | quote }} + template_dir = {{ $section.templateDir | default $defaults.templateDir | default "/opt/machine-a-tron/templates" | quote }} + oob_dhcp_relay_address = {{ $section.oobDhcpRelayAddress | default "192.168.192.1" | quote }} + admin_dhcp_relay_address = {{ $section.adminDhcpRelayAddress | default "192.168.176.1" | quote }} + {{- $nvType := $section.networkVirtualizationType | default $defaults.networkVirtualizationType }} + {{- if $nvType }} + network_virtualization_type = {{ $nvType | quote }} + {{- end }} + dpus_in_nic_mode = {{ $section.dpusInNicMode | default $defaults.dpusInNicMode | default false }} + {{- end }} + {{- end }} + {{- end }} +{{- end }} diff --git a/helm/charts/nico-machine-a-tron/templates/deployment.yaml b/helm/charts/nico-machine-a-tron/templates/deployment.yaml new file mode 100644 index 0000000000..05074aa06f --- /dev/null +++ b/helm/charts/nico-machine-a-tron/templates/deployment.yaml @@ -0,0 +1,178 @@ +{{- $root := . }} +{{- $namespace := include "nico-machine-a-tron.namespace" . }} +{{- $baseName := include "nico-machine-a-tron.name" . }} + +{{/* +Count pods that should be deployed: +- When bmcServices.enabled: only pods with CIDR +- Otherwise: all pods with machines +*/}} +{{- $activePods := 0 }} +{{- range $podName, $podConfig := .Values.pods }} +{{- if $root.Values.bmcServices.enabled }} +{{- if $podConfig.cidr }} +{{- $activePods = add $activePods 1 }} +{{- end }} +{{- else }} +{{- if $podConfig.machines }} +{{- $activePods = add $activePods 1 }} +{{- end }} +{{- end }} +{{- end }} + +{{- range $podName, $podConfig := .Values.pods }} +{{/* +Skip conditions: +- When bmcServices.enabled: skip pods without CIDR +- Otherwise: skip pods without machines +*/}} +{{- if $root.Values.bmcServices.enabled }} +{{- if not $podConfig.cidr }} +{{- continue }} +{{- end }} +{{- else }} +{{- if not $podConfig.machines }} +{{- continue }} +{{- end }} +{{- end }} + +{{- $name := $baseName }} +{{- if gt $activePods 1 }} +{{- $name = printf "%s-%s" $baseName $podName }} +{{- end }} +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ $name }} + namespace: {{ $namespace }} + labels: + {{- include "nico-machine-a-tron.labels" $root | nindent 4 }} + {{- if gt $activePods 1 }} + nvidia-infra-controller/pod-name: {{ $podName | quote }} + {{- end }} + annotations: + {{- tpl (toYaml $root.Values.annotations) $root | nindent 4 }} +spec: + replicas: 1 + strategy: + type: Recreate + selector: + matchLabels: + {{- include "nico-machine-a-tron.selectorLabels" $root | nindent 6 }} + {{- if gt $activePods 1 }} + nvidia-infra-controller/pod-name: {{ $podName | quote }} + {{- end }} + template: + metadata: + labels: + {{- include "nico-machine-a-tron.labels" $root | nindent 8 }} + {{- if gt $activePods 1 }} + nvidia-infra-controller/pod-name: {{ $podName | quote }} + {{- end }} + annotations: + kubectl.kubernetes.io/default-container: {{ $baseName }} + spec: + automountServiceAccountToken: {{ $root.Values.automountServiceAccountToken }} + serviceAccountName: {{ $baseName }} + terminationGracePeriodSeconds: {{ $root.Values.terminationGracePeriodSeconds }} + {{- with $root.Values.global.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + containers: + - name: {{ $baseName }} + image: "{{ include "nico-machine-a-tron.image" $root }}" + imagePullPolicy: {{ ($root.Values.global.image).pullPolicy | default $root.Values.image.pullPolicy | default "IfNotPresent" }} + command: + {{- toYaml $root.Values.command | nindent 12 }} + env: + - name: VAULT_ROLE_ID + valueFrom: + secretKeyRef: + name: {{ $root.Values.envFrom.vaultApproleTokens.secretName }} + key: VAULT_ROLE_ID + optional: true + - name: VAULT_SECRET_ID + valueFrom: + secretKeyRef: + name: {{ $root.Values.envFrom.vaultApproleTokens.secretName }} + key: VAULT_SECRET_ID + optional: true + - name: VAULT_TOKEN + valueFrom: + secretKeyRef: + name: {{ $root.Values.envFrom.vaultToken.secretName }} + key: token + optional: true + - name: VAULT_ADDR + valueFrom: + configMapKeyRef: + name: {{ $root.Values.envFrom.vaultClusterInfo.configMapName }} + key: VAULT_SERVICE + optional: true + - name: MACHINE_A_TRON_CONFIG_PATH + value: "/etc/nvidia-infra-controller/machine-a-tron/mat.toml" + - name: FORGE_ROOT_CA_PATH + value: "/var/run/secrets/spiffe.io/ca.crt" + - name: CLIENT_CERT_PATH + value: "/var/run/secrets/spiffe.io/tls.crt" + - name: CLIENT_KEY_PATH + value: "/var/run/secrets/spiffe.io/tls.key" + - name: RUST_BACKTRACE + value: {{ $root.Values.env.RUST_BACKTRACE | quote }} + - name: RUST_LIB_BACKTRACE + value: {{ $root.Values.env.RUST_LIB_BACKTRACE | quote }} + {{- with $root.Values.extraEnv }} + {{- toYaml . | nindent 12 }} + {{- end }} + ports: + - name: redfish + containerPort: {{ $root.Values.service.bmcMock.port }} + - name: ssh + containerPort: {{ $root.Values.machineATron.mockBmcSshPort }} + - name: metrics + containerPort: {{ $root.Values.service.metrics.port }} + livenessProbe: + {{- toYaml $root.Values.livenessProbe | nindent 12 }} + readinessProbe: + {{- toYaml $root.Values.readinessProbe | nindent 12 }} + volumeMounts: + - name: config-files + mountPath: /etc/forge/machine-a-tron + - name: config-files + mountPath: /etc/nvidia-infra-controller/machine-a-tron + - name: spiffe + mountPath: "/var/run/secrets/spiffe.io" + readOnly: true + - name: nico-roots + mountPath: "/var/run/secrets/forge-roots" + readOnly: true + - name: nico-roots + mountPath: "/var/run/secrets/nico-roots" + readOnly: true + {{- if $root.Values.persistence.enabled }} + - name: data + mountPath: {{ $root.Values.machineATron.persistDir }} + {{- if gt $activePods 1 }} + subPath: {{ $podName }} + {{- end }} + {{- end }} + resources: + {{- toYaml $root.Values.resources | nindent 12 }} + volumes: + - name: config-files + configMap: + name: {{ $name }}-config-files + - name: spiffe + secret: + secretName: {{ $name }}-certificate + - name: nico-roots + secret: + secretName: nico-roots + {{- if $root.Values.persistence.enabled }} + - name: data + persistentVolumeClaim: + claimName: {{ $baseName }}-data + {{- end }} +{{- end }} diff --git a/helm/charts/nico-machine-a-tron/templates/external-service.yaml b/helm/charts/nico-machine-a-tron/templates/external-service.yaml new file mode 100644 index 0000000000..33f262c97f --- /dev/null +++ b/helm/charts/nico-machine-a-tron/templates/external-service.yaml @@ -0,0 +1,28 @@ +{{/* +External LoadBalancer service for Override Mode (development). +Only used when bmcServices is disabled - provides a single external endpoint. +When bmcServices.enabled, use ClusterIP services instead for per-BMC routing. +*/}} +{{- if and .Values.externalService.enabled (not .Values.bmcServices.enabled) }} +apiVersion: v1 +kind: Service +metadata: + name: {{ include "nico-machine-a-tron.name" . }}-external + namespace: {{ include "nico-machine-a-tron.namespace" . }} + labels: + {{- include "nico-machine-a-tron.labels" . | nindent 4 }} + {{- with .Values.externalService.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + type: {{ .Values.externalService.type }} + externalTrafficPolicy: {{ .Values.externalService.externalTrafficPolicy }} + selector: + {{- include "nico-machine-a-tron.selectorLabels" . | nindent 4 }} + ports: + - port: {{ .Values.service.bmcMock.port }} + name: bmc-mock + targetPort: redfish + protocol: TCP +{{- end }} diff --git a/helm/charts/nico-machine-a-tron/templates/metrics-service.yaml b/helm/charts/nico-machine-a-tron/templates/metrics-service.yaml new file mode 100644 index 0000000000..f7dafc029a --- /dev/null +++ b/helm/charts/nico-machine-a-tron/templates/metrics-service.yaml @@ -0,0 +1,15 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ include "nico-machine-a-tron.name" . }}-metrics + namespace: {{ include "nico-machine-a-tron.namespace" . }} + labels: + {{- include "nico-machine-a-tron.labels" . | nindent 4 }} + app.kubernetes.io/metrics: {{ include "nico-machine-a-tron.name" . }} +spec: + selector: + {{- include "nico-machine-a-tron.selectorLabels" . | nindent 4 }} + ports: + - port: {{ .Values.service.metrics.port }} + name: metrics + protocol: TCP diff --git a/helm/charts/nico-machine-a-tron/templates/namespace.yaml b/helm/charts/nico-machine-a-tron/templates/namespace.yaml new file mode 100644 index 0000000000..bc0d92cdfd --- /dev/null +++ b/helm/charts/nico-machine-a-tron/templates/namespace.yaml @@ -0,0 +1,10 @@ +{{- if and .Values.namespaceOverride .Values.createNamespace }} +apiVersion: v1 +kind: Namespace +metadata: + name: {{ include "nico-machine-a-tron.namespace" . }} + annotations: + helm.sh/resource-policy: keep + labels: + {{- include "nico-machine-a-tron.labels" . | nindent 4 }} +{{- end }} diff --git a/helm/charts/nico-machine-a-tron/templates/pvc.yaml b/helm/charts/nico-machine-a-tron/templates/pvc.yaml new file mode 100644 index 0000000000..7ac4d8de03 --- /dev/null +++ b/helm/charts/nico-machine-a-tron/templates/pvc.yaml @@ -0,0 +1,18 @@ +{{- if .Values.persistence.enabled }} +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: {{ include "nico-machine-a-tron.name" . }}-data + namespace: {{ include "nico-machine-a-tron.namespace" . }} + labels: + {{- include "nico-machine-a-tron.labels" . | nindent 4 }} +spec: + accessModes: + {{- toYaml .Values.persistence.accessModes | nindent 4 }} + {{- if .Values.persistence.storageClass }} + storageClassName: {{ .Values.persistence.storageClass | quote }} + {{- end }} + resources: + requests: + storage: {{ .Values.persistence.size }} +{{- end }} diff --git a/helm/charts/nico-machine-a-tron/templates/rbac.yaml b/helm/charts/nico-machine-a-tron/templates/rbac.yaml new file mode 100644 index 0000000000..5cc13d0f4b --- /dev/null +++ b/helm/charts/nico-machine-a-tron/templates/rbac.yaml @@ -0,0 +1,27 @@ +kind: Role +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: {{ include "nico-machine-a-tron.name" . }} + namespace: {{ include "nico-machine-a-tron.namespace" . }} + labels: + {{- include "nico-machine-a-tron.labels" . | nindent 4 }} +rules: + - apiGroups: ["cert-manager.io"] + resources: ["certificaterequests"] + verbs: ["create"] +--- +kind: RoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: {{ include "nico-machine-a-tron.name" . }} + namespace: {{ include "nico-machine-a-tron.namespace" . }} + labels: + {{- include "nico-machine-a-tron.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: {{ include "nico-machine-a-tron.name" . }} +subjects: + - kind: ServiceAccount + name: {{ include "nico-machine-a-tron.name" . }} + namespace: {{ include "nico-machine-a-tron.namespace" . }} diff --git a/helm/charts/nico-machine-a-tron/templates/service-account.yaml b/helm/charts/nico-machine-a-tron/templates/service-account.yaml new file mode 100644 index 0000000000..d7a1fe8a92 --- /dev/null +++ b/helm/charts/nico-machine-a-tron/templates/service-account.yaml @@ -0,0 +1,7 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "nico-machine-a-tron.name" . }} + namespace: {{ include "nico-machine-a-tron.namespace" . }} + labels: + {{- include "nico-machine-a-tron.labels" . | nindent 4 }} diff --git a/helm/charts/nico-machine-a-tron/templates/service-monitor.yaml b/helm/charts/nico-machine-a-tron/templates/service-monitor.yaml new file mode 100644 index 0000000000..36c039fc0b --- /dev/null +++ b/helm/charts/nico-machine-a-tron/templates/service-monitor.yaml @@ -0,0 +1,11 @@ +{{- if .Values.serviceMonitor.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ include "nico-machine-a-tron.name" . }} + namespace: {{ include "nico-machine-a-tron.namespace" . }} + labels: + {{- include "nico-machine-a-tron.labels" . | nindent 4 }} +spec: + {{- include "nico-machine-a-tron.serviceMonitorSpec" (dict "name" (include "nico-machine-a-tron.name" .) "monitor" .Values.serviceMonitor "port" "metrics" "namespace" (include "nico-machine-a-tron.namespace" .)) | nindent 2 }} +{{- end }} diff --git a/helm/charts/nico-machine-a-tron/templates/service.yaml b/helm/charts/nico-machine-a-tron/templates/service.yaml new file mode 100644 index 0000000000..a8faab7fc6 --- /dev/null +++ b/helm/charts/nico-machine-a-tron/templates/service.yaml @@ -0,0 +1,68 @@ +{{- $root := . }} +{{- $namespace := include "nico-machine-a-tron.namespace" . }} +{{- $baseName := include "nico-machine-a-tron.name" . }} + +{{/* +Count active pods (same logic as deployment.yaml) +*/}} +{{- $activePods := 0 }} +{{- range $podName, $podConfig := .Values.pods }} +{{- if $root.Values.bmcServices.enabled }} +{{- if $podConfig.cidr }} +{{- $activePods = add $activePods 1 }} +{{- end }} +{{- else }} +{{- if $podConfig.machines }} +{{- $activePods = add $activePods 1 }} +{{- end }} +{{- end }} +{{- end }} + +{{- range $podName, $podConfig := .Values.pods }} +{{/* +Skip conditions (same as deployment.yaml) +*/}} +{{- if $root.Values.bmcServices.enabled }} +{{- if not $podConfig.cidr }} +{{- continue }} +{{- end }} +{{- else }} +{{- if not $podConfig.machines }} +{{- continue }} +{{- end }} +{{- end }} + +{{- $name := $baseName }} +{{- if gt $activePods 1 }} +{{- $name = printf "%s-%s" $baseName $podName }} +{{- end }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ $name }}-bmc-mock + namespace: {{ $namespace }} + labels: + {{- include "nico-machine-a-tron.labels" $root | nindent 4 }} + app: {{ $baseName }} + {{- if gt $activePods 1 }} + nvidia-infra-controller/pod-name: {{ $podName | quote }} + {{- end }} +spec: + selector: + {{- include "nico-machine-a-tron.selectorLabels" $root | nindent 4 }} + {{- if gt $activePods 1 }} + nvidia-infra-controller/pod-name: {{ $podName | quote }} + {{- end }} + ports: + - port: {{ $root.Values.service.bmcMock.port }} + name: redfish + targetPort: redfish + protocol: TCP + {{- if $root.Values.machineATron.mockBmcSshServer }} + - port: {{ $root.Values.service.ssh.port }} + name: ssh + targetPort: ssh + protocol: TCP + {{- end }} +{{- end }} diff --git a/helm/charts/nico-machine-a-tron/tests/certificate_test.yaml b/helm/charts/nico-machine-a-tron/tests/certificate_test.yaml new file mode 100644 index 0000000000..20d7dd901c --- /dev/null +++ b/helm/charts/nico-machine-a-tron/tests/certificate_test.yaml @@ -0,0 +1,42 @@ +suite: certificate tests +templates: + - certificate.yaml +tests: + - it: should create a certificate with default values + asserts: + - isKind: + of: Certificate + - equal: + path: metadata.name + value: nico-machine-a-tron-certificate + + - it: should use custom certificate settings + set: + certificate: + serviceName: custom-service + extraDnsNames: + - extra.example.com + asserts: + - isKind: + of: Certificate + + - it: should create per-pod certificates in multi-pod mode + set: + bmcServices: + enabled: true + pods: + mat-0: + cidr: "10.100.0.0/22" + machines: + compute: + hwType: supermicro_gb300_nvl + hostCount: 5 + mat-1: + cidr: "10.100.4.0/22" + machines: + compute: + hwType: supermicro_gb300_nvl + hostCount: 5 + asserts: + - hasDocuments: + count: 2 diff --git a/helm/charts/nico-machine-a-tron/tests/configmap_test.yaml b/helm/charts/nico-machine-a-tron/tests/configmap_test.yaml new file mode 100644 index 0000000000..41f46ac80c --- /dev/null +++ b/helm/charts/nico-machine-a-tron/tests/configmap_test.yaml @@ -0,0 +1,90 @@ +suite: configmap tests +templates: + - configmap.yaml +tests: + - it: should create configmap with default machine config + asserts: + - isKind: + of: ConfigMap + - equal: + path: metadata.name + value: nico-machine-a-tron-config-files + - matchRegex: + path: data["mat.toml"] + pattern: "host_count = 10" + - matchRegex: + path: data["mat.toml"] + pattern: "use_single_bmc_mock = true" + + - it: should configure custom host count via pods structure + set: + pods: + default: + machines: + dell-hosts: + hwType: dell_poweredge_r750 + hostCount: 20 + dpuPerHostCount: 2 + asserts: + - matchRegex: + path: data["mat.toml"] + pattern: "host_count = 20" + - matchRegex: + path: data["mat.toml"] + pattern: "dpu_per_host_count = 2" + - matchRegex: + path: data["mat.toml"] + pattern: 'hw_type = "dell_poweredge_r750"' + + - it: should allow full config override via matConfigs + set: + configFiles: + matConfigs: + default: | + carbide_api_url = "https://custom-api:443" + [machines.custom] + host_count = 5 + asserts: + - matchRegex: + path: data["mat.toml"] + pattern: "custom-api" + + - it: should create per-pod configmaps in multi-pod mode + set: + bmcServices: + enabled: true + pods: + mat-0: + cidr: "10.100.0.0/22" + machines: + compute: + hwType: supermicro_gb300_nvl + hostCount: 10 + mat-1: + cidr: "10.100.4.0/22" + machines: + compute: + hwType: supermicro_gb300_nvl + hostCount: 20 + asserts: + - hasDocuments: + count: 2 + + - it: should use machineDefaults for all machine groups + set: + machineDefaults: + runIntervalIdle: "30s" + scoutRunInterval: "120s" + pods: + default: + machines: + compute: + hwType: supermicro_gb300_nvl + hostCount: 5 + asserts: + - matchRegex: + path: data["mat.toml"] + pattern: 'run_interval_idle = "30s"' + - matchRegex: + path: data["mat.toml"] + pattern: 'scout_run_interval = "120s"' diff --git a/helm/charts/nico-machine-a-tron/values.yaml b/helm/charts/nico-machine-a-tron/values.yaml new file mode 100644 index 0000000000..5d3fa1f308 --- /dev/null +++ b/helm/charts/nico-machine-a-tron/values.yaml @@ -0,0 +1,307 @@ +## ============================================================================= +## nico-machine-a-tron - Mock machine simulator for NICo testing +## ============================================================================= +## +## DEPLOYMENT MODES: +## +## 1. OVERRIDE MODE (Development) - bmcServices.enabled: false +## - NICo Site-Explorer redirects ALL Redfish calls to machine-a-tron +## - Simple setup, single pod +## - INCOMPATIBLE with real hardware +## +## 2. CLUSTERIP MODE (Large-Scale Testing) - bmcServices.enabled: true +## - Each simulated BMC gets dedicated ClusterIP service +## - Supports multi-pod deployments via `pods` configuration +## - Compatible with real hardware +## +## See README.md for detailed setup instructions. +## +## ============================================================================= + +global: + certificate: + duration: 720h0m0s + renewBefore: 360h0m0s + privateKey: + algorithm: ECDSA + size: 384 + issuerRef: + kind: ClusterIssuer + name: vault-nico-issuer + group: cert-manager.io + spiffe: + trustDomain: nico.local + labels: + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/part-of: site-controller + +image: + ## Container image repository. When empty, uses global.image.repository or defaults to nico:latest + repository: "" + ## Container image tag. Defaults to chart appVersion + tag: "" + pullPolicy: IfNotPresent + +namespaceOverride: "" + +## Create namespace when namespaceOverride is set +createNamespace: true + +automountServiceAccountToken: true + +## Graceful shutdown timeout (seconds) +terminationGracePeriodSeconds: 60 + +annotations: + configmap.reloader.stakater.com/reload: '{{ include "nico-machine-a-tron.name" . }}-config-files' + +resources: + limits: + cpu: 4 + memory: 4Gi + requests: + cpu: 500m + memory: 1Gi + +service: + bmcMock: + port: 1266 + ssh: + port: 22 + targetPort: 2222 + metrics: + port: 9090 + +certificate: + serviceName: "" + spiffeServiceName: "" + identityNamespace: "" + dnsNames: [] + uris: [] + extraDnsNames: [] + extraUris: [] + +command: + - /bin/sh + - -c + - /opt/machine-a-tron/bin/machine-a-tron + +env: + RUST_BACKTRACE: "full" + RUST_LIB_BACKTRACE: "0" + +extraEnv: [] + +envFrom: + vaultApproleTokens: + secretName: nico-vault-approle-tokens + vaultToken: + secretName: nico-vault-token + vaultClusterInfo: + configMapName: vault-cluster-info + +livenessProbe: + tcpSocket: + port: redfish + initialDelaySeconds: 30 + periodSeconds: 30 + failureThreshold: 3 + successThreshold: 1 + timeoutSeconds: 10 + +readinessProbe: + tcpSocket: + port: redfish + initialDelaySeconds: 5 + periodSeconds: 10 + failureThreshold: 3 + successThreshold: 1 + timeoutSeconds: 5 + +serviceMonitor: + enabled: false + interval: 30s + scrapeTimeout: 25s + +## ============================================================================= +## Machine-a-tron Core Settings (shared across all pods) +## ============================================================================= +machineATron: + nicoApiUrl: "https://nico-api.nico-system.svc.cluster.local:1079" + interface: "NOTUSED" + tuiEnabled: false + logFile: "" + useSingleBmcMock: true + mockBmcSshServer: true + mockBmcSshPort: 2222 + configureBmcProxyHost: "" + persistDir: "/tmp/machine-a-tron-data" + cleanupOnQuit: false + registerExpectedMachines: true + hostBmcPassword: "" + dpuBmcPassword: "" + apiRefreshInterval: "2s" + +## ============================================================================= +## Machine Defaults - Shared settings for all machine groups +## ============================================================================= +## These defaults apply to all machine groups in all pods. +## Override per-group by specifying the value in pods.*.machines.*. +## +machineDefaults: + vpcCount: 0 + subnetsPerVpc: 0 + dpuRebootDelay: 1 + hostRebootDelay: 1 + scoutRunInterval: "60s" + runIntervalWorking: "1s" + runIntervalIdle: "10s" + networkStatusRunInterval: "20s" + templateDir: "/opt/machine-a-tron/templates" + networkVirtualizationType: "" + dpusInNicMode: false + +## ============================================================================= +## Pod Configurations - One pod per config entry +## ============================================================================= +## Each entry creates: +## - 1 Deployment (pod) +## - 1 ConfigMap with mat.toml +## - 1 Certificate +## - 1 internal Service +## - N ClusterIP BMC services (when bmcServices.enabled) +## +## SCALE: Use /22 CIDR per pod = 1021 usable IPs = ~14 racks worth of BMCs +## +## BMC count per GB200 NVL72 rack: 71 +## - Compute trays: 18 × 3 = 54 +## - NVLink switches: 9 × 1 = 9 +## - Power shelves: 8 × 1 = 8 +## +## Per-machine group, only these fields are typically needed: +## - hwType: Hardware type (required) +## - hostCount: Number of hosts (required) +## - dpuPerHostCount: DPUs per host (default: 0) +## - oobDhcpRelayAddress: OOB network gateway +## - adminDhcpRelayAddress: Admin network gateway +## +## All other settings inherit from machineDefaults above. +## +pods: + ## Single-pod configuration is used when bmcServices.enabled=false (by default) + ## For multi-pod deployments, add more entries below and enable bmcServices + default: + cidr: "" + + ## Machine groups for this pod + machines: + rack-machines: + hwType: wiwynn_gb200_nvl + hostCount: 10 + dpuPerHostCount: 2 + oobDhcpRelayAddress: "192.168.192.1" + adminDhcpRelayAddress: "192.168.176.1" + +## ============================================================================= +## Multi-Pod Example (bmcServices.enabled is required) +## ============================================================================= +## Uncomment and customize for large-scale deployments: +## +# mat-0: +# cidr: "10.100.0.0/22" # 1021 IPs for ~14 racks +# machines: +# compute: +# hwType: wiwynn_gb200_nvl +# hostCount: 252 # 18 trays × 14 racks +# dpuPerHostCount: 2 # 2 BF3 per tray +# oobDhcpRelayAddress: "10.100.0.1" +# adminDhcpRelayAddress: "192.168.176.1" +# switches: +# hwType: nvidia_switch_nd5200_ld +# hostCount: 126 # 9 switches × 14 racks +# oobDhcpRelayAddress: "10.100.0.1" +# adminDhcpRelayAddress: "192.168.176.1" +# power: +# hwType: liteon_power_shelf +# hostCount: 112 # 8 shelves × 14 racks +# oobDhcpRelayAddress: "10.100.0.1" +# adminDhcpRelayAddress: "192.168.176.1" +# +# mat-1: +# cidr: "10.100.4.0/22" # Next /22 block +# machines: +# compute: +# hwType: wiwynn_gb200_nvl +# hostCount: 252 +# dpuPerHostCount: 2 +# oobDhcpRelayAddress: "10.100.4.1" +# adminDhcpRelayAddress: "192.168.176.1" +# # ... same pattern + +## ============================================================================= +## BMC Services Configuration +## ============================================================================= +bmcServices: + enabled: false + + ## External port for BMC services (matches real BMC HTTPS port) + servicePort: 443 + + ## ServiceCIDR reserves the IP range (Kubernetes 1.29+) + serviceCIDR: + create: true + ## Name for the ServiceCIDR (cluster-scoped) + name: "" + ## Combined CIDR covering all pods (auto-calculated if empty for single pod) + ## Set manually if pods have non-contiguous CIDRs + cidr: "" + +## Persistence volume for machine state +persistence: + enabled: false + storageClass: "" + size: 1Gi + accessModes: + - ReadWriteOnce + +externalService: + enabled: false + type: LoadBalancer + externalTrafficPolicy: Local + annotations: {} + +configFiles: + ## Override mat.toml per pod (optional) + ## Key = pod name, value = full config content + ## When set, this completely replaces the auto-generated config for that pod. + ## + ## Example - simple single-pod config: + # matConfigs: + # default: | + # carbide_api_url = "https://nico-api.nico-system.svc.cluster.local:1079" + # interface = "NOTUSED" + # tui_enabled = false + # bmc_mock_port = 1266 + # use_single_bmc_mock = true + # mock_bmc_ssh_server = true + # mock_bmc_ssh_port = 2222 + # cleanup_on_quit = false + # register_expected_machines = true + # api_refresh_interval = "2s" + # + # [machines.compute] + # hw_type = "supermicro_gb300_nvl" + # host_count = 10 + # dpu_per_host_count = 2 + # oob_dhcp_relay_address = "10.100.0.1" + # admin_dhcp_relay_address = "192.168.176.1" + # template_dir = "/opt/machine-a-tron/templates" + # + # [machines.switches] + # hw_type = "nvidia_switch_nd5200_ld" + # host_count = 5 + # dpu_per_host_count = 0 + # oob_dhcp_relay_address = "10.100.0.1" + # admin_dhcp_relay_address = "192.168.176.1" + # template_dir = "/opt/machine-a-tron/templates" + matConfigs: {} diff --git a/helm/templates/NOTES.txt b/helm/templates/NOTES.txt index bfb2ae32c4..685e307a3b 100644 --- a/helm/templates/NOTES.txt +++ b/helm/templates/NOTES.txt @@ -30,6 +30,9 @@ Deployed sub-charts: {{- if index .Values "nico-ssh-console-rs" "enabled" | default true }} ✓ nico-ssh-console-rs {{- end }} +{{- if index .Values "nico-machine-a-tron" "enabled" | default false }} + ✓ nico-machine-a-tron (mock machines for dev/test) +{{- end }} {{- if index .Values "unbound" "enabled" | default false }} ✓ unbound {{- end }} diff --git a/helm/values.yaml b/helm/values.yaml index 92e7a81bdc..1ea49ea43f 100644 --- a/helm/values.yaml +++ b/helm/values.yaml @@ -165,3 +165,13 @@ nico-ssh-console-rs: ## --------------------------------------------------------------------------- unbound: enabled: false + +## --------------------------------------------------------------------------- +## nico-machine-a-tron - Mock machine simulator (development/testing only) +## Simulates bare-metal machines, DPUs, switches and power shelves for +## testing NICo without physical hardware. NOT for production use. +## Deploy standalone for dev environments: +## helm install mat ./helm/charts/nico-machine-a-tron -n nico-system ... +## --------------------------------------------------------------------------- +nico-machine-a-tron: + enabled: false