From 43ae91be62b3dce6ad53d9562aeafeebcf6be612 Mon Sep 17 00:00:00 2001 From: Baichao He Date: Fri, 19 Jun 2026 20:46:39 +0000 Subject: [PATCH 1/5] Add Azure Linux nspawn image support --- docs/usages/configuration.md | 1 + go.mod | 28 +++++------ go.sum | 70 +++++++++++++-------------- hack/e2e/README.md | 24 +++++++++- hack/e2e/infra/main.bicep | 32 ++++++++++++- hack/e2e/infra/modules/vm.bicep | 49 +++++++++++++++++-- hack/e2e/lib/cleanup.sh | 16 ++++++- hack/e2e/lib/common.sh | 6 +++ hack/e2e/lib/infra.sh | 32 +++++++++++++ hack/e2e/lib/node-join-token.sh | 83 +++++++++++++++++++++++++++++++++ hack/e2e/lib/node-join.sh | 36 +++++++++++--- hack/e2e/lib/validate.sh | 24 ++++++++-- hack/e2e/run.sh | 14 +++++- pkg/config/adapter.go | 4 +- pkg/config/adapter_test.go | 5 +- pkg/config/config.go | 3 ++ pkg/config/config_test.go | 4 +- 17 files changed, 357 insertions(+), 74 deletions(-) diff --git a/docs/usages/configuration.md b/docs/usages/configuration.md index d24a94d7..263dd358 100644 --- a/docs/usages/configuration.md +++ b/docs/usages/configuration.md @@ -83,6 +83,7 @@ Exactly one authentication mode must be configured. | `agent.logLevel` | string | Agent log verbosity. | `info` | | `agent.logDir` | string | Host directory for agent logs. | `/var/log/aks-flex-node` | | `agent.nodeName` | string | Optional Kubernetes node name override. Defaults to the host hostname. | `edge-node-01` | +| `agent.ociImage` | string | Optional nspawn rootfs OCI image. Set an Azure Linux image such as `ghcr.io/azure/agent-azlinux3:` to use Azure Linux in the nspawn machine. | `ghcr.io/azure/agent-azlinux3:` | | `agent.machineReconcileInterval` | duration string | Daemon interval for re-reading machine state. Uses Go duration syntax. | `10m` | | `agent.e2eMode` | boolean | Uses the local file-backed machine client for E2E tests. | `false` | | `agent.requireMachineRegistration` | boolean | Fails bootstrap when the AKS machine resource cannot be read or created. When false, registration is best-effort. | `false` | diff --git a/go.mod b/go.mod index 1a7aa7e6..6fe58b5a 100644 --- a/go.mod +++ b/go.mod @@ -9,14 +9,14 @@ require ( github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerservice/armcontainerservice/v8 v8.3.0-beta.2 github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/hybridcompute/armhybridcompute v1.2.0 github.com/Azure/kubelogin v0.2.15 - github.com/Azure/unbounded v0.1.11 + github.com/Azure/unbounded v0.1.18-0.20260619173346-9a7b731ce879 github.com/google/renameio/v2 v2.0.2 github.com/google/uuid v1.6.0 github.com/spf13/cobra v1.10.2 k8s.io/api v0.35.4 k8s.io/apimachinery v0.35.4 k8s.io/client-go v0.35.4 - k8s.io/utils v0.0.0-20251002143259-bc988d571ff4 + k8s.io/utils v0.0.0-20260319190234-28399d86e0b5 sigs.k8s.io/controller-runtime v0.23.3 ) @@ -38,9 +38,9 @@ require ( github.com/blang/semver/v4 v4.0.0 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/containerd/log v0.1.0 // indirect - github.com/containerd/platforms v0.2.1 // indirect + github.com/containerd/platforms v1.0.0-rc.4 // indirect github.com/cpuguy83/go-md2man/v2 v2.0.7 // indirect - github.com/cyphar/filepath-securejoin v0.5.0 // indirect + github.com/cyphar/filepath-securejoin v0.6.0 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/docker/go-units v0.5.0 // indirect github.com/emicklei/go-restful/v3 v3.13.0 // indirect @@ -72,23 +72,23 @@ require ( github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/opencontainers/go-digest v1.0.0 // indirect github.com/opencontainers/image-spec v1.1.1 // indirect - github.com/opencontainers/runtime-spec v1.2.1 // indirect + github.com/opencontainers/runtime-spec v1.3.0 // indirect github.com/opencontainers/umoci v0.6.0 // indirect github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c // indirect github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/prometheus/client_golang v1.23.2 // indirect github.com/prometheus/client_model v0.6.2 // indirect - github.com/prometheus/common v0.66.1 // indirect - github.com/prometheus/procfs v0.16.1 // indirect + github.com/prometheus/common v0.67.5 // indirect + github.com/prometheus/procfs v0.20.1 // indirect github.com/rootless-containers/proto/go-proto v0.0.0-20230421021042-4cd87ebadd67 // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect github.com/sirupsen/logrus v1.9.4 // indirect github.com/spf13/pflag v1.0.10 // indirect - github.com/urfave/cli v1.22.12 // indirect + github.com/urfave/cli v1.22.16 // indirect github.com/vbatts/go-mtree v0.6.1-0.20250911112631-8307d76bc1b9 // indirect github.com/x448/float16 v0.8.4 // indirect - go.yaml.in/yaml/v2 v2.4.3 // indirect + go.yaml.in/yaml/v2 v2.4.4 // indirect go.yaml.in/yaml/v3 v3.0.4 // indirect golang.org/x/crypto v0.52.0 // indirect golang.org/x/net v0.55.0 // indirect @@ -97,18 +97,18 @@ require ( golang.org/x/sys v0.45.0 // indirect golang.org/x/term v0.43.0 // indirect golang.org/x/text v0.37.0 // indirect - golang.org/x/time v0.11.0 // indirect + golang.org/x/time v0.15.0 // indirect gomodules.xyz/jsonpatch/v2 v2.4.0 // indirect - google.golang.org/protobuf v1.36.11 // indirect + google.golang.org/protobuf v1.36.12-0.20260120151049-f2248ac996af // indirect gopkg.in/evanphx/json-patch.v4 v4.13.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect k8s.io/apiextensions-apiserver v0.35.0 // indirect - k8s.io/klog/v2 v2.130.1 // indirect - k8s.io/kube-openapi v0.0.0-20250910181357-589584f1c912 // indirect + k8s.io/klog/v2 v2.140.0 // indirect + k8s.io/kube-openapi v0.0.0-20260319004828-5883c5ee87b9 // indirect oras.land/oras-go/v2 v2.6.0 // indirect sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730 // indirect sigs.k8s.io/randfill v1.0.0 // indirect - sigs.k8s.io/structured-merge-diff/v6 v6.3.2-0.20260122202528-d9cc6641c482 // indirect + sigs.k8s.io/structured-merge-diff/v6 v6.3.2 // indirect sigs.k8s.io/yaml v1.6.0 // indirect ) diff --git a/go.sum b/go.sum index a7ef61c0..8b7373ae 100644 --- a/go.sum +++ b/go.sum @@ -36,13 +36,13 @@ github.com/Azure/go-autorest/tracing v0.6.0 h1:TYi4+3m5t6K48TGI9AUdb+IzbnSxvnvUM github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU= github.com/Azure/kubelogin v0.2.15 h1:oJqD8Dvput3rO/xZgMTU+hBrcgg0BfQGPCNHJ2dEmys= github.com/Azure/kubelogin v0.2.15/go.mod h1:RwJS8TzSHTVQhfIZA4HLS79QGfvIp0ocIVLT5oHS/ls= -github.com/Azure/unbounded v0.1.11 h1:U0tQa/K2F1WVKbMFf7A1KD+RzpjiMvCdY2cMpBtpiJM= -github.com/Azure/unbounded v0.1.11/go.mod h1:8/ekWflNUvo8KTVPSYBX+7w/JpcoaeE7DTsdMfVfI6c= +github.com/Azure/unbounded v0.1.18-0.20260619173346-9a7b731ce879 h1:Puq+/5KWe86I5eTIPQAiAubCh8Wj2U0Nbn6qc/jmBQY= +github.com/Azure/unbounded v0.1.18-0.20260619173346-9a7b731ce879/go.mod h1:nTkSAO2q0NLMbdNDSDqd9j32wvoZUtheAQ5zsIRvRwg= github.com/AzureAD/microsoft-authentication-extensions-for-go/cache v0.1.1 h1:WJTmL004Abzc5wDB5VtZG2PJk5ndYDgVacGqfirKxjM= github.com/AzureAD/microsoft-authentication-extensions-for-go/cache v0.1.1/go.mod h1:tCcJZ0uHAmvjsVYzEFivsRTN00oz5BEsRgQHu5JZ9WE= github.com/AzureAD/microsoft-authentication-library-for-go v1.7.2 h1:RHK7bS+HQMslb1sZpAokUt+zTVmue0hKSs2C791hhzU= github.com/AzureAD/microsoft-authentication-library-for-go v1.7.2/go.mod h1:HKpQxkWaGLJ+D/5H8QRpyQXA1eKjxkFlOMwck5+33Jk= -github.com/BurntSushi/toml v1.2.1/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= +github.com/BurntSushi/toml v1.4.0/go.mod h1:ukJfTF/6rtPPRCnwkur4qwRxa8vTRFBF0uk2lLoLwho= github.com/Masterminds/semver/v3 v3.5.0 h1:kQceYJfbupGfZOKZQg0kou0DgAKhzDg2NZPAwZ/2OOE= github.com/Masterminds/semver/v3 v3.5.0/go.mod h1:4V+yj/TJE1HU9XfppCwVMZq3I84lprf4nC11bSS5beM= github.com/apex/log v1.9.0 h1:FHtw/xuaM8AgmvDDTI9fiwoAL25Sq2cxojnZICUU8l0= @@ -60,15 +60,15 @@ github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UF github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/containerd/log v0.1.0 h1:TCJt7ioM2cr/tfR8GPbGf9/VRAX8D2B4PjzCpfX540I= github.com/containerd/log v0.1.0/go.mod h1:VRRf09a7mHDIRezVKTRCrOq78v577GXq3bSa3EhrzVo= -github.com/containerd/platforms v0.2.1 h1:zvwtM3rz2YHPQsF2CHYM8+KtB5dvhISiXh5ZpSBQv6A= -github.com/containerd/platforms v0.2.1/go.mod h1:XHCb+2/hzowdiut9rkudds9bE5yJ7npe7dG/wG+uFPw= -github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= +github.com/containerd/platforms v1.0.0-rc.4 h1:M42JrUT4zfZTqtkUwkr0GzmUWbfyO5VO0Q5b3op97T4= +github.com/containerd/platforms v1.0.0-rc.4/go.mod h1:lKlMXyLybmBedS/JJm11uDofzI8L2v0J2ZbYvNsbq1A= +github.com/cpuguy83/go-md2man/v2 v2.0.5/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g= github.com/cpuguy83/go-md2man/v2 v2.0.7 h1:zbFlGlXEAKlwXpmvle3d8Oe3YnkKIK4xSRTd3sHPnBo= github.com/cpuguy83/go-md2man/v2 v2.0.7/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= -github.com/cyphar/filepath-securejoin v0.5.0 h1:hIAhkRBMQ8nIeuVwcAoymp7MY4oherZdAxD+m0u9zaw= -github.com/cyphar/filepath-securejoin v0.5.0/go.mod h1:Sdj7gXlvMcPZsbhwhQ33GguGLDGQL7h7bg04C/+u9jI= +github.com/cyphar/filepath-securejoin v0.6.0 h1:BtGB77njd6SVO6VztOHfPxKitJvd/VPT+OFBFMOi1Is= +github.com/cyphar/filepath-securejoin v0.6.0/go.mod h1:A8hd4EnAeyujCJRrICiOWqjS1AX0a9kM5XL+NwKoYSc= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= @@ -186,8 +186,8 @@ github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8 github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= github.com/opencontainers/image-spec v1.1.1 h1:y0fUlFfIZhPF1W537XOLg0/fcx6zcHCJwooC2xJA040= github.com/opencontainers/image-spec v1.1.1/go.mod h1:qpqAh3Dmcf36wStyyWU+kCeDgrGnAve2nCC8+7h8Q0M= -github.com/opencontainers/runtime-spec v1.2.1 h1:S4k4ryNgEpxW1dzyqffOmhI1BHYcjzU8lpJfSlR0xww= -github.com/opencontainers/runtime-spec v1.2.1/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= +github.com/opencontainers/runtime-spec v1.3.0 h1:YZupQUdctfhpZy3TM39nN9Ika5CBWT5diQ8ibYCRkxg= +github.com/opencontainers/runtime-spec v1.3.0/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= github.com/opencontainers/umoci v0.6.0 h1:Dsm4beJpglN5y2E2EUSZZcNey4Ml4+nKepvwLQwgIec= github.com/opencontainers/umoci v0.6.0/go.mod h1:2DS3cxVN9pRJGYaCK5mnmmwVKV5vd9r6HIYAV0IvdbI= github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c h1:+mdjkGKdHQG3305AYmdv1U2eRNDiU2ErMBj1gwrq8eQ= @@ -202,10 +202,10 @@ github.com/prometheus/client_golang v1.23.2 h1:Je96obch5RDVy3FDMndoUsjAhG5Edi49h github.com/prometheus/client_golang v1.23.2/go.mod h1:Tb1a6LWHB3/SPIzCoaDXI4I8UHKeFTEQ1YCr+0Gyqmg= github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNwqPLxwZyk= github.com/prometheus/client_model v0.6.2/go.mod h1:y3m2F6Gdpfy6Ut/GBsUqTWZqCUvMVzSfMLjcu6wAwpE= -github.com/prometheus/common v0.66.1 h1:h5E0h5/Y8niHc5DlaLlWLArTQI7tMrsfQjHV+d9ZoGs= -github.com/prometheus/common v0.66.1/go.mod h1:gcaUsgf3KfRSwHY4dIMXLPV0K/Wg1oZ8+SbZk/HH/dA= -github.com/prometheus/procfs v0.16.1 h1:hZ15bTNuirocR6u0JZ6BAHHmwS1p8B4P6MRqxtzMyRg= -github.com/prometheus/procfs v0.16.1/go.mod h1:teAbpZRB1iIAJYREa1LsoWUXykVXA1KlTmWl8x/U+Is= +github.com/prometheus/common v0.67.5 h1:pIgK94WWlQt1WLwAC5j2ynLaBRDiinoAb86HZHTUGI4= +github.com/prometheus/common v0.67.5/go.mod h1:SjE/0MzDEEAyrdr5Gqc6G+sXI67maCxzaT3A2+HqjUw= +github.com/prometheus/procfs v0.20.1 h1:XwbrGOIplXW/AU3YhIhLODXMJYyC1isLFfYCsTEycfc= +github.com/prometheus/procfs v0.20.1/go.mod h1:o9EMBZGRyvDrSPH1RqdxhojkuXstoe4UlK79eF5TGGo= github.com/rogpeppe/fastuuid v1.1.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= @@ -235,6 +235,8 @@ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= github.com/tj/assert v0.0.0-20171129193455-018094318fb0/go.mod h1:mZ9/Rh9oLWpLLDRpvE+3b7gP/C2YyLFYxNmcLnPTMe0= @@ -244,8 +246,8 @@ github.com/tj/go-buffer v1.1.0/go.mod h1:iyiJpfFcR2B9sXu7KvjbT9fpM4mOelRSDTbntVj github.com/tj/go-elastic v0.0.0-20171221160941-36157cbbebc2/go.mod h1:WjeM0Oo1eNAjXGDx2yma7uG2XoyRZTq1uv3M/o7imD0= github.com/tj/go-kinesis v0.0.0-20171128231115-08b17f58cb1b/go.mod h1:/yhzCV0xPfx6jb1bBgRFjl5lytqVqZXEaeqWP8lTEao= github.com/tj/go-spin v1.1.0/go.mod h1:Mg1mzmePZm4dva8Qz60H2lHwmJ2loum4VIrLgVnKwh4= -github.com/urfave/cli v1.22.12 h1:igJgVw1JdKH+trcLWLeLwZjU9fEfPesQ+9/e4MQ44S8= -github.com/urfave/cli v1.22.12/go.mod h1:sSBEIC79qR6OvcmsD4U3KABeOTxDqQtdDnaFuUN30b8= +github.com/urfave/cli v1.22.16 h1:MH0k6uJxdwdeWQTwhSO42Pwr4YLrNLwBtg1MRgTqPdQ= +github.com/urfave/cli v1.22.16/go.mod h1:EeJR6BKodywf4zciqrdw6hpCPk68JO9z5LazXZMn5Po= github.com/vbatts/go-mtree v0.6.1-0.20250911112631-8307d76bc1b9 h1:R6l9BtUe83abUGu1YKGkfa17wMMFLt6mhHVQ8MxpfRE= github.com/vbatts/go-mtree v0.6.1-0.20250911112631-8307d76bc1b9/go.mod h1:W7bcG9PCn6lFY+ljGlZxx9DONkxL3v8a7HyN+PrSrjA= github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= @@ -253,14 +255,14 @@ github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcY github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= -go.uber.org/mock v0.5.0 h1:KAMbZvZPyBPWgD14IrIQ38QCyjwpvVVV6K/bHl1IwQU= -go.uber.org/mock v0.5.0/go.mod h1:ge71pBPLYDk7QIi1LupWxdAykm7KIEFchiOqd6z7qMM= +go.uber.org/mock v0.6.0 h1:hyF9dfmbgIX5EfOdasqLsWD6xqpNZlXblLB/Dbnwv3Y= +go.uber.org/mock v0.6.0/go.mod h1:KiVJ4BqZJaMj4svdfmHM0AUx4NJYO8ZNpPnZn1Z+BBU= go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= -go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8= -go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= -go.yaml.in/yaml/v2 v2.4.3 h1:6gvOSjQoTB3vt1l+CU+tSyi/HOjfOjRLJ4YwYZGwRO0= -go.yaml.in/yaml/v2 v2.4.3/go.mod h1:zSxWcmIDjOzPXpjlTTbAsKokqkDNAVtZO0WOMiT90s8= +go.uber.org/zap v1.27.1 h1:08RqriUEv8+ArZRYSTXy1LeBScaMpVSTBhCeaZYfMYc= +go.uber.org/zap v1.27.1/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= +go.yaml.in/yaml/v2 v2.4.4 h1:tuyd0P+2Ont/d6e2rl3be67goVK4R6deVxCUX5vyPaQ= +go.yaml.in/yaml/v2 v2.4.4/go.mod h1:gMZqIpDtDqOfM0uNfy0SkpRhvUryYH0Z6wdMYcacYXQ= go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc= go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= @@ -315,8 +317,8 @@ golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.37.0 h1:Cqjiwd9eSg8e0QAkyCaQTNHFIIzWtidPahFWR83rTrc= golang.org/x/text v0.37.0/go.mod h1:a5sjxXGs9hsn/AJVwuElvCAo9v8QYLzvavO5z2PiM38= -golang.org/x/time v0.11.0 h1:/bpjEDfN9tkoN/ryeYHnv5hcMlc8ncjMcM4XBk5NWV0= -golang.org/x/time v0.11.0/go.mod h1:CDIdPxbZBQxdj6cxyCIdrNogrJKMJ7pr37NYpMcMDSg= +golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U= +golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= @@ -328,8 +330,8 @@ gomodules.xyz/jsonpatch/v2 v2.4.0 h1:Ci3iUJyx9UeRx7CeFN8ARgGbkESwJK+KB9lLcWxY/Zw gomodules.xyz/jsonpatch/v2 v2.4.0/go.mod h1:AH3dM2RI6uoBZxn3LVrfvJ3E0/9dG4cSrbuBJT4moAY= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE= -google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= +google.golang.org/protobuf v1.36.12-0.20260120151049-f2248ac996af h1:+5/Sw3GsDNlEmu7TfklWKPdQ0Ykja5VEmq2i817+jbI= +google.golang.org/protobuf v1.36.12-0.20260120151049-f2248ac996af/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= @@ -357,12 +359,12 @@ k8s.io/apimachinery v0.35.4 h1:xtdom9RG7e+yDp71uoXoJDWEE2eOiHgeO4GdBzwWpds= k8s.io/apimachinery v0.35.4/go.mod h1:NNi1taPOpep0jOj+oRha3mBJPqvi0hGdaV8TCqGQ+cc= k8s.io/client-go v0.35.4 h1:DN6fyaGuzK64UvnKO5fOA6ymSjvfGAnCAHAR0C66kD8= k8s.io/client-go v0.35.4/go.mod h1:2Pg9WpsS4NeOpoYTfHHfMxBG8zFMSAUi4O/qoiJC3nY= -k8s.io/klog/v2 v2.130.1 h1:n9Xl7H1Xvksem4KFG4PYbdQCQxqc/tTUyrgXaOhHSzk= -k8s.io/klog/v2 v2.130.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE= -k8s.io/kube-openapi v0.0.0-20250910181357-589584f1c912 h1:Y3gxNAuB0OBLImH611+UDZcmKS3g6CthxToOb37KgwE= -k8s.io/kube-openapi v0.0.0-20250910181357-589584f1c912/go.mod h1:kdmbQkyfwUagLfXIad1y2TdrjPFWp2Q89B3qkRwf/pQ= -k8s.io/utils v0.0.0-20251002143259-bc988d571ff4 h1:SjGebBtkBqHFOli+05xYbK8YF1Dzkbzn+gDM4X9T4Ck= -k8s.io/utils v0.0.0-20251002143259-bc988d571ff4/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= +k8s.io/klog/v2 v2.140.0 h1:Tf+J3AH7xnUzZyVVXhTgGhEKnFqye14aadWv7bzXdzc= +k8s.io/klog/v2 v2.140.0/go.mod h1:o+/RWfJ6PwpnFn7OyAG3QnO47BFsymfEfrz6XyYSSp0= +k8s.io/kube-openapi v0.0.0-20260319004828-5883c5ee87b9 h1:Sztf7ESG9tAXRW/ACJZjrj5jhdOUqS2KFRQT+CTvu78= +k8s.io/kube-openapi v0.0.0-20260319004828-5883c5ee87b9/go.mod h1:uGBT7iTA6c6MvqUvSXIaYZo9ukscABYi2btjhvgKGZ0= +k8s.io/utils v0.0.0-20260319190234-28399d86e0b5 h1:kBawHLSnx/mYHmRnNUf9d4CpjREbeZuxoSGOX/J+aYM= +k8s.io/utils v0.0.0-20260319190234-28399d86e0b5/go.mod h1:xDxuJ0whA3d0I4mf/C4ppKHxXynQ+fxnkmQH0vTHnuk= oras.land/oras-go/v2 v2.6.0 h1:X4ELRsiGkrbeox69+9tzTu492FMUu7zJQW6eJU+I2oc= oras.land/oras-go/v2 v2.6.0/go.mod h1:magiQDfG6H1O9APp+rOsvCPcW1GD2MM7vgnKY0Y+u1o= sigs.k8s.io/controller-runtime v0.23.3 h1:VjB/vhoPoA9l1kEKZHBMnQF33tdCLQKJtydy4iqwZ80= @@ -371,7 +373,7 @@ sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730 h1:IpInykpT6ceI+QxKBbEflcR5E sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730/go.mod h1:mdzfpAEoE6DHQEN0uh9ZbOCuHbLK5wOm7dK4ctXE9Tg= sigs.k8s.io/randfill v1.0.0 h1:JfjMILfT8A6RbawdsK2JXGBR5AQVfd+9TbzrlneTyrU= sigs.k8s.io/randfill v1.0.0/go.mod h1:XeLlZ/jmk4i1HRopwe7/aU3H5n1zNUcX6TM94b3QxOY= -sigs.k8s.io/structured-merge-diff/v6 v6.3.2-0.20260122202528-d9cc6641c482 h1:2WOzJpHUBVrrkDjU4KBT8n5LDcj824eX0I5UKcgeRUs= -sigs.k8s.io/structured-merge-diff/v6 v6.3.2-0.20260122202528-d9cc6641c482/go.mod h1:M3W8sfWvn2HhQDIbGWj3S099YozAsymCo/wrT5ohRUE= +sigs.k8s.io/structured-merge-diff/v6 v6.3.2 h1:kwVWMx5yS1CrnFWA/2QHyRVJ8jM6dBA80uLmm0wJkk8= +sigs.k8s.io/structured-merge-diff/v6 v6.3.2/go.mod h1:M3W8sfWvn2HhQDIbGWj3S099YozAsymCo/wrT5ohRUE= sigs.k8s.io/yaml v1.6.0 h1:G8fkbMSAFqgEFgh4b1wmtzDnioxFCUgTZhlbj5P9QYs= sigs.k8s.io/yaml v1.6.0/go.mod h1:796bPqUfzR/0jLAl6XjHl3Ck7MiyVv8dbTdyT3/pMf4= diff --git a/hack/e2e/README.md b/hack/e2e/README.md index c328a0cb..0c6a3509 100644 --- a/hack/e2e/README.md +++ b/hack/e2e/README.md @@ -1,6 +1,6 @@ # AKS Flex Node E2E Tests -The E2E suite provisions an AKS cluster and three Ubuntu VMs in Azure, joins the VMs as Flex Nodes, validates workloads, exercises unjoin/rejoin behavior, validates repave, collects logs, and tears down the resources. +The E2E suite provisions an AKS cluster and three Ubuntu VMs in Azure, joins the VMs as Flex Nodes, validates workloads, exercises unjoin/rejoin behavior, validates repave, collects logs, and tears down the resources. It can optionally add an Azure Linux 3 host VM from a generalized VHD and join it with an Azure Linux 3 nspawn image. ## Prerequisites @@ -47,10 +47,12 @@ The default `all` command runs: | `join-msi` | Join only the managed-identity node. | | `join-token` | Join only the bootstrap-token node. | | `join-kubeadm` | Join only the kubeadm-style bootstrap-token node. | +| `join-azlinux3` | Join only the optional Azure Linux 3 bootstrap-token node. | | `unjoin` | Unjoin all Flex Node VMs. | | `unjoin-msi` | Unjoin only the managed-identity node. | | `unjoin-token` | Unjoin only the bootstrap-token node. | | `unjoin-kubeadm` | Unjoin only the kubeadm-style node. | +| `unjoin-azlinux3` | Unjoin only the optional Azure Linux 3 bootstrap-token node. | | `validate` | Verify joined nodes and run smoke tests. | | `validate-absent` | Verify Flex Node objects are absent after unjoin. | | `smoke` | Run smoke workloads only. | @@ -80,6 +82,9 @@ Additional environment variables: | `E2E_KUBERNETES_VERSION` | `1.35.0` | Kubernetes version used in generated node configs. | | `E2E_CONTAINERD_VERSION` | `2.0.4` | Containerd version used in generated node configs. | | `E2E_RUNC_VERSION` | `1.1.12` | Runc version used in generated node configs. | +| `E2E_ENABLE_AZLINUX3` | `0` | Set to `1` to deploy and test the optional Azure Linux 3 host VM. | +| `E2E_AZLINUX3_VHD_URI` | empty | Generalized Azure Linux 3 VHD URI used to create the optional host VM image. Required when `E2E_ENABLE_AZLINUX3=1`. | +| `E2E_AZLINUX3_OCI_IMAGE` | `ghcr.io/azure/agent-azlinux3:v20260619` | Azure Linux 3 nspawn rootfs OCI image set as `agent.ociImage`. | | `E2E_SSH_WAIT_TIMEOUT` | `300` | Timeout in seconds while waiting for SSH. | | `E2E_NODE_JOIN_TIMEOUT` | `300` | Timeout in seconds while waiting for node bootstrap. | | `E2E_POD_READY_TIMEOUT` | `120` | Timeout in seconds while waiting for smoke pods. | @@ -96,6 +101,7 @@ The suite validates three join paths: | `vm-e2e-msi-*` | Managed Identity | Generated managed-identity config and `aks-flex-node start` flow. | | `vm-e2e-token-*` | Bootstrap Token | Kubernetes bootstrap token, RBAC, generated config, and `aks-flex-node start` flow. | | `vm-e2e-kubeadm-*` | Bootstrap Token | Kubeadm-style bootstrap resources plus generated config and `aks-flex-node start` flow. | +| `vm-e2e-azlinux3-*` | Bootstrap Token | Optional Azure Linux 3 host VM from VHD plus generated config with `agent.ociImage` pointing at the Azure Linux 3 nspawn image. | The bootstrap-token VM is provisioned with an uppercase guest OS hostname while its Azure resource name remains lowercase. This verifies that an omitted @@ -104,6 +110,22 @@ cluster under the lowercase VM name. Each join path uploads the locally built binary, renders a config file, installs the binary through `scripts/install.sh` with `AKS_FLEX_NODE_LOCAL_BINARY`, and starts the node through a transient systemd unit. The installed agent service is then validated with systemd checks. +## Azure Linux 3 Scenario + +The optional Azure Linux 3 scenario validates both host and nspawn Azure Linux support. Provide a generalized Azure Linux 3 VHD URI that Azure Compute can read, usually a blob URI with SAS, then enable the scenario: + +```bash +export E2E_ENABLE_AZLINUX3=1 +export E2E_AZLINUX3_VHD_URI='https://.blob.core.windows.net//.vhd?' +export E2E_AZLINUX3_OCI_IMAGE='ghcr.io/azure/agent-azlinux3:v20260619' + +./hack/e2e/run.sh infra +./hack/e2e/run.sh join-azlinux3 +./hack/e2e/run.sh validate +``` + +The VHD must already include the host dependencies needed by Flex Node bootstrap, including systemd-nspawn support. The test creates a managed Compute image from the VHD and uses that image for the VM. + ## Repave Validation The `upgrade-drift` command validates the local-machine-driven repave path: diff --git a/hack/e2e/infra/main.bicep b/hack/e2e/infra/main.bicep index 20f19db2..0d74208b 100644 --- a/hack/e2e/infra/main.bicep +++ b/hack/e2e/infra/main.bicep @@ -6,8 +6,9 @@ // - VM with system-assigned managed identity (MSI auth mode) // - VM without managed identity (bootstrap token auth mode) // - VM without managed identity (kubeadm apply -f auth mode) +// - Optional Azure Linux 3 VM from VHD (bootstrap token auth mode) // -// All flex-node VMs run Ubuntu 22.04 LTS, have public IPs, and allow SSH +// Flex-node VMs have public IPs and allow SSH // ingress. VM creation is delegated to the reusable modules/vm.bicep module. // ============================================================================= @@ -33,6 +34,13 @@ param sshPublicKey string @description('Tags applied to every resource.') param tags object = {} +@description('Whether to deploy the optional Azure Linux 3 host VM scenario.') +param enableAzLinux3Scenario bool = false + +@secure() +@description('Generalized Azure Linux 3 VHD URI for the optional Azure Linux 3 host VM scenario.') +param azLinux3VhdUri string = '' + // --------------------------------------------------------------------------- // Variables // --------------------------------------------------------------------------- @@ -40,6 +48,7 @@ var clusterName = 'aks-e2e-${nameSuffix}' var msiVmName = 'vm-e2e-msi-${nameSuffix}' var tokenVmName = 'vm-e2e-token-${nameSuffix}' var kubeadmVmName = 'vm-e2e-kubeadm-${nameSuffix}' +var azLinux3VmName = 'vm-e2e-azlinux3-${nameSuffix}' var vnetName = 'vnet-e2e-${nameSuffix}' var nsgName = 'nsg-e2e-${nameSuffix}' @@ -185,6 +194,23 @@ module vmKubeadm 'modules/vm.bicep' = { } } +module vmAzLinux3 'modules/vm.bicep' = if (enableAzLinux3Scenario) { + name: 'deploy-vm-azlinux3' + params: { + location: location + vmName: azLinux3VmName + vmSize: vmSize + adminUsername: adminUsername + sshPublicKey: sshPublicKey + subnetId: vnet.properties.subnets[1].id + assignManagedIdentity: false + imageSourceType: 'vhd' + imageVhdUri: azLinux3VhdUri + imageHyperVGeneration: 'V2' + tags: tags + } +} + // --------------------------------------------------------------------------- // Role assignments: grant MSI VM permissions on the AKS cluster // --------------------------------------------------------------------------- @@ -228,4 +254,8 @@ output tokenVmPrivateIp string = vmToken.outputs.privateIpAddress output kubeadmVmName string = vmKubeadm.outputs.vmName output kubeadmVmIp string = vmKubeadm.outputs.publicIpAddress +output azLinux3VmName string = enableAzLinux3Scenario ? vmAzLinux3!.outputs.vmName : '' +output azLinux3VmIp string = enableAzLinux3Scenario ? vmAzLinux3!.outputs.publicIpAddress : '' +output azLinux3VmPrivateIp string = enableAzLinux3Scenario ? vmAzLinux3!.outputs.privateIpAddress : '' + output adminUsername string = adminUsername diff --git a/hack/e2e/infra/modules/vm.bicep b/hack/e2e/infra/modules/vm.bicep index 6c90df49..0c40e592 100644 --- a/hack/e2e/infra/modules/vm.bicep +++ b/hack/e2e/infra/modules/vm.bicep @@ -1,8 +1,10 @@ // ============================================================================= -// modules/vm.bicep - Reusable Ubuntu flex-node VM module +// modules/vm.bicep - Reusable flex-node VM module // -// Creates a public IP, NIC, and Ubuntu VM in the given subnet. -// The VHD image defaults to Ubuntu 24.04 LTS (Noble) but can be overridden. +// Creates a public IP, NIC, and Linux VM in the given subnet. The marketplace +// image defaults to Ubuntu 24.04 LTS (Noble) but can be overridden. A +// generalized VHD URI can also be imported as a managed image and used as the +// VM source image. // ============================================================================= @description('Azure region for all resources.') @@ -30,6 +32,13 @@ param subnetId string @description('Whether to assign a system-assigned managed identity to the VM.') param assignManagedIdentity bool = false +@allowed([ + 'marketplace' + 'vhd' +]) +@description('Image source type. Use marketplace for imageReference fields, or vhd to create a managed image from imageVhdUri.') +param imageSourceType string = 'marketplace' + @description('Marketplace image publisher.') param imagePublisher string = 'Canonical' @@ -42,9 +51,39 @@ param imageSku string = 'server' @description('Marketplace image version.') param imageVersion string = 'latest' +@secure() +@description('Generalized Linux VHD URI used when imageSourceType is vhd. The URI must be readable by Azure Compute, for example via SAS.') +param imageVhdUri string = '' + +@allowed([ + 'V1' + 'V2' +]) +@description('Hyper-V generation for a managed image created from imageVhdUri.') +param imageHyperVGeneration string = 'V2' + @description('Tags applied to all resources in this module.') param tags object = {} +var useVhdImage = imageSourceType == 'vhd' + +resource vhdImage 'Microsoft.Compute/images@2024-03-01' = if (useVhdImage) { + name: '${vmName}-image' + location: location + tags: tags + properties: { + hyperVGeneration: imageHyperVGeneration + storageProfile: { + osDisk: { + osType: 'Linux' + osState: 'Generalized' + blobUri: imageVhdUri + storageAccountType: 'StandardSSD_LRS' + } + } + } +} + // --------------------------------------------------------------------------- // Public IP // --------------------------------------------------------------------------- @@ -113,7 +152,9 @@ resource vm 'Microsoft.Compute/virtualMachines@2024-03-01' = { } } storageProfile: { - imageReference: { + imageReference: useVhdImage ? { + id: vhdImage.id + } : { publisher: imagePublisher offer: imageOffer sku: imageSku diff --git a/hack/e2e/lib/cleanup.sh b/hack/e2e/lib/cleanup.sh index fcc98f9a..109f92b7 100755 --- a/hack/e2e/lib/cleanup.sh +++ b/hack/e2e/lib/cleanup.sh @@ -76,10 +76,11 @@ collect_logs() { mkdir -p "${E2E_LOG_DIR}" - local msi_vm_ip token_vm_ip kubeadm_vm_ip + local msi_vm_ip token_vm_ip kubeadm_vm_ip azlinux3_vm_ip msi_vm_ip="$(state_get msi_vm_ip)" token_vm_ip="$(state_get token_vm_ip)" kubeadm_vm_ip="$(state_get kubeadm_vm_ip)" + azlinux3_vm_ip="$(state_get azlinux3_vm_ip)" if [[ -n "${msi_vm_ip}" ]]; then _collect_vm_logs "${msi_vm_ip}" "msi" || true @@ -93,6 +94,10 @@ collect_logs() { _collect_vm_logs "${kubeadm_vm_ip}" "kubeadm" || true fi + if [[ -n "${azlinux3_vm_ip}" ]]; then + _collect_vm_logs "${azlinux3_vm_ip}" "azlinux3" || true + fi + # Also capture cluster-side info { echo "=== Nodes ===" @@ -134,12 +139,13 @@ cleanup() { stop_daemon_csr_approver - local resource_group cluster_name msi_vm_name token_vm_name kubeadm_vm_name + local resource_group cluster_name msi_vm_name token_vm_name kubeadm_vm_name azlinux3_vm_name resource_group="$(state_get resource_group)" cluster_name="$(state_get cluster_name)" msi_vm_name="$(state_get msi_vm_name)" token_vm_name="$(state_get token_vm_name)" kubeadm_vm_name="$(state_get kubeadm_vm_name)" + azlinux3_vm_name="$(state_get azlinux3_vm_name)" local deployment_name deployment_name="$(state_get deployment_name)" @@ -161,6 +167,12 @@ cleanup() { az vm delete --resource-group "${resource_group}" --name "${kubeadm_vm_name}" \ --force-deletion yes --yes --no-wait 2>/dev/null || true + if [[ -n "${azlinux3_vm_name}" ]]; then + log_info "[3/5] Deleting Azure Linux 3 VM: ${azlinux3_vm_name}..." + az vm delete --resource-group "${resource_group}" --name "${azlinux3_vm_name}" \ + --force-deletion yes --yes --no-wait 2>/dev/null || true + fi + # Clean up leftover networking resources tied to our deployment log_info "[4/5] Cleaning up networking resources..." local run_id="${GITHUB_RUN_ID:-}" diff --git a/hack/e2e/lib/common.sh b/hack/e2e/lib/common.sh index 0096e991..65a33df3 100755 --- a/hack/e2e/lib/common.sh +++ b/hack/e2e/lib/common.sh @@ -185,6 +185,11 @@ load_config() { E2E_CONTAINERD_VERSION="${E2E_CONTAINERD_VERSION:-2.0.4}" E2E_RUNC_VERSION="${E2E_RUNC_VERSION:-1.1.12}" + # Optional Azure Linux 3 host and nspawn image scenario. + E2E_ENABLE_AZLINUX3="${E2E_ENABLE_AZLINUX3:-0}" + E2E_AZLINUX3_VHD_URI="${E2E_AZLINUX3_VHD_URI:-}" + E2E_AZLINUX3_OCI_IMAGE="${E2E_AZLINUX3_OCI_IMAGE:-ghcr.io/azure/agent-azlinux3:v20260619}" + # Timeouts (seconds) E2E_SSH_WAIT_TIMEOUT="${E2E_SSH_WAIT_TIMEOUT:-300}" E2E_NODE_JOIN_TIMEOUT="${E2E_NODE_JOIN_TIMEOUT:-300}" @@ -197,6 +202,7 @@ load_config() { log_info " Subscription: ${AZURE_SUBSCRIPTION_ID}" log_info " Name Suffix: ${E2E_NAME_SUFFIX}" log_info " Skip Cleanup: ${E2E_SKIP_CLEANUP}" + log_info " Azure Linux 3: ${E2E_ENABLE_AZLINUX3}" } # --------------------------------------------------------------------------- diff --git a/hack/e2e/lib/infra.sh b/hack/e2e/lib/infra.sh index d0307984..0d2d0e74 100755 --- a/hack/e2e/lib/infra.sh +++ b/hack/e2e/lib/infra.sh @@ -57,6 +57,11 @@ infra_deploy() { return 1 fi + if [[ "${E2E_ENABLE_AZLINUX3}" == "1" && -z "${E2E_AZLINUX3_VHD_URI}" ]]; then + log_error "E2E_ENABLE_AZLINUX3=1 requires E2E_AZLINUX3_VHD_URI to point at a generalized Azure Linux 3 VHD" + return 1 + fi + # Ensure resource group exists if ! az group show --name "${E2E_RESOURCE_GROUP}" --output none 2>/dev/null; then log_info "Creating resource group: ${E2E_RESOURCE_GROUP} in ${E2E_LOCATION}" @@ -93,6 +98,8 @@ infra_deploy() { nameSuffix="${E2E_NAME_SUFFIX}" \ sshPublicKey="${ssh_key}" \ tags="${tags_json}" \ + enableAzLinux3Scenario="$([[ "${E2E_ENABLE_AZLINUX3}" == "1" ]] && echo true || echo false)" \ + azLinux3VhdUri="${E2E_AZLINUX3_VHD_URI}" \ --output none # Extract outputs @@ -106,6 +113,7 @@ infra_deploy() { local cluster_name cluster_id msi_vm_name msi_vm_ip msi_vm_principal_id local token_vm_name token_vm_ip token_vm_private_ip kubeadm_vm_name kubeadm_vm_ip admin_username + local azlinux3_vm_name azlinux3_vm_ip azlinux3_vm_private_ip cluster_name=$(echo "${outputs}" | jq -r '.clusterName.value') cluster_id=$(echo "${outputs}" | jq -r '.clusterId.value') @@ -117,6 +125,9 @@ infra_deploy() { token_vm_private_ip=$(echo "${outputs}" | jq -r '.tokenVmPrivateIp.value // ""') kubeadm_vm_name=$(echo "${outputs}" | jq -r '.kubeadmVmName.value') kubeadm_vm_ip=$(echo "${outputs}" | jq -r '.kubeadmVmIp.value') + azlinux3_vm_name=$(echo "${outputs}" | jq -r '.azLinux3VmName.value // ""') + azlinux3_vm_ip=$(echo "${outputs}" | jq -r '.azLinux3VmIp.value // ""') + azlinux3_vm_private_ip=$(echo "${outputs}" | jq -r '.azLinux3VmPrivateIp.value // ""') admin_username=$(echo "${outputs}" | jq -r '.adminUsername.value') if [[ -z "${token_vm_private_ip}" ]] || ! is_valid_ipv4 "${token_vm_private_ip}"; then @@ -124,6 +135,11 @@ infra_deploy() { return 1 fi + if [[ "${E2E_ENABLE_AZLINUX3}" == "1" && ( -z "${azlinux3_vm_private_ip}" || ! is_valid_ipv4 "${azlinux3_vm_private_ip}" ) ]]; then + log_error "Missing or invalid Azure Linux 3 VM private IP from deployment outputs: '${azlinux3_vm_private_ip}'" + return 1 + fi + # Persist to state state_set "cluster_name" "${cluster_name}" state_set "cluster_id" "${cluster_id}" @@ -135,6 +151,11 @@ infra_deploy() { state_set "token_vm_private_ip" "${token_vm_private_ip}" state_set "kubeadm_vm_name" "${kubeadm_vm_name}" state_set "kubeadm_vm_ip" "${kubeadm_vm_ip}" + state_set "azlinux3_enabled" "${E2E_ENABLE_AZLINUX3}" + state_set "azlinux3_vm_name" "${azlinux3_vm_name}" + state_set "azlinux3_vm_ip" "${azlinux3_vm_ip}" + state_set "azlinux3_vm_private_ip" "${azlinux3_vm_private_ip}" + state_set "azlinux3_oci_image" "${E2E_AZLINUX3_OCI_IMAGE}" state_set "admin_username" "${admin_username}" state_set "resource_group" "${E2E_RESOURCE_GROUP}" state_set "location" "${E2E_LOCATION}" @@ -146,6 +167,9 @@ infra_deploy() { log_info "MSI VM: ${msi_vm_name} @ ${msi_vm_ip}" log_info "Token VM: ${token_vm_name} @ ${token_vm_ip}" log_info "Kubeadm VM: ${kubeadm_vm_name} @ ${kubeadm_vm_ip}" + if [[ "${E2E_ENABLE_AZLINUX3}" == "1" ]]; then + log_info "AzLinux3 VM: ${azlinux3_vm_name} @ ${azlinux3_vm_ip}" + fi # Get kubeconfig and extract cluster info infra_get_kubeconfig @@ -158,11 +182,19 @@ infra_deploy() { local pid_token=$! wait_for_ssh "${kubeadm_vm_ip}" & local pid_kubeadm=$! + local pid_azlinux3="" + if [[ "${E2E_ENABLE_AZLINUX3}" == "1" ]]; then + wait_for_ssh "${azlinux3_vm_ip}" & + pid_azlinux3=$! + fi local ssh_failed=0 wait "${pid_msi}" || ssh_failed=1 wait "${pid_token}" || ssh_failed=1 wait "${pid_kubeadm}" || ssh_failed=1 + if [[ -n "${pid_azlinux3}" ]]; then + wait "${pid_azlinux3}" || ssh_failed=1 + fi if [[ "${ssh_failed}" -eq 1 ]]; then log_error "One or more VMs not reachable via SSH" diff --git a/hack/e2e/lib/node-join-token.sh b/hack/e2e/lib/node-join-token.sh index ffefc359..640e7dcd 100644 --- a/hack/e2e/lib/node-join-token.sh +++ b/hack/e2e/lib/node-join-token.sh @@ -76,6 +76,70 @@ node_join_token() { log_success "Token node joined in $(timer_elapsed "${start}")s" } +# --------------------------------------------------------------------------- +# node_join_azlinux3 - Join the optional Azure Linux 3 host VM using the +# Azure Linux 3 nspawn OCI image. +# --------------------------------------------------------------------------- +node_join_azlinux3() { + if [[ "$(state_get azlinux3_enabled 0)" != "1" ]]; then + log_info "Azure Linux 3 scenario disabled; skipping join" + return 0 + fi + + log_section "Joining Azure Linux 3 Node" + local start + start=$(timer_start) + + local vm_ip vm_private_ip cluster_name resource_group subscription_id + vm_ip="$(state_get azlinux3_vm_ip)" + vm_private_ip="$(state_get azlinux3_vm_private_ip)" + cluster_name="$(state_get cluster_name)" + resource_group="$(state_get resource_group)" + subscription_id="$(state_get subscription_id)" + + if [[ -z "${vm_private_ip}" ]] || ! is_valid_ipv4 "${vm_private_ip}"; then + log_error "Invalid Azure Linux 3 VM private IP in state: '${vm_private_ip}'" + return 1 + fi + + log_info "Setting up bootstrap token RBAC resources..." + "${REPO_ROOT}/scripts/aks-flex-config" setup-node-rbac \ + --resource-group "${resource_group}" \ + --cluster-name "${cluster_name}" \ + --subscription "${subscription_id}" + + ensure_daemon_csr_approver + + log_info "Generating Azure Linux 3 token config..." + local config_file="${E2E_WORK_DIR}/config-azlinux3.json" + "${REPO_ROOT}/scripts/aks-flex-config" generate-node-config \ + --resource-group "${resource_group}" \ + --cluster-name "${cluster_name}" \ + --subscription "${subscription_id}" \ + --bootstrap-token \ + --output "${config_file}" + + jq \ + --arg nodeIP "${vm_private_ip}" \ + --arg kubernetesVersion "${E2E_KUBERNETES_VERSION}" \ + --arg containerdVersion "${E2E_CONTAINERD_VERSION}" \ + --arg runcVersion "${E2E_RUNC_VERSION}" \ + --arg ociImage "$(state_get azlinux3_oci_image "${E2E_AZLINUX3_OCI_IMAGE}")" \ + '.agent.logLevel = "debug" + | .agent.e2eMode = true + | .agent.ociImage = $ociImage + | .node.kubelet.nodeIP = $nodeIP + | .kubernetes.version = $kubernetesVersion + | .containerd.version = $containerdVersion + | .runc.version = $runcVersion' \ + "${config_file}" > "${config_file}.tmp" + mv "${config_file}.tmp" "${config_file}" + + _deploy_and_start_agent "${vm_ip}" "${config_file}" "aks-flex-node-azlinux3" + + log_success "Azure Linux 3 node joined in $(timer_elapsed "${start}")s" +} + # --------------------------------------------------------------------------- # node_unjoin_token - Simulate RP delete and verify node cleanup # --------------------------------------------------------------------------- @@ -92,3 +156,22 @@ node_unjoin_token() { log_success "Token node unjoined in $(timer_elapsed "${start}")s" } + +node_unjoin_azlinux3() { + if [[ "$(state_get azlinux3_enabled 0)" != "1" ]]; then + log_info "Azure Linux 3 scenario disabled; skipping unjoin" + return 0 + fi + + log_section "Unjoining Azure Linux 3 Node" + local start + start=$(timer_start) + + local vm_ip vm_name + vm_ip="$(state_get azlinux3_vm_ip)" + vm_name="$(state_get azlinux3_vm_name)" + + _rp_delete_unjoin_node "${vm_ip}" "${vm_name}" + + log_success "Azure Linux 3 node unjoined in $(timer_elapsed "${start}")s" +} diff --git a/hack/e2e/lib/node-join.sh b/hack/e2e/lib/node-join.sh index a1993671..28655412 100755 --- a/hack/e2e/lib/node-join.sh +++ b/hack/e2e/lib/node-join.sh @@ -8,7 +8,7 @@ # node-join-kubeadm.sh - Kubeadm apply -f join/unjoin (node_join_kubeadm, node_unjoin_kubeadm) # # Functions: -# node_join_all - Join all nodes (MSI, token, and kubeadm) in parallel +# node_join_all - Join all nodes (MSI, token, kubeadm, and optional Azure Linux 3) in parallel # node_unjoin_all - Unjoin all nodes in parallel # ============================================================================= set -euo pipefail @@ -257,8 +257,8 @@ node_join_all() { local start start=$(timer_start) - local msi_pid token_pid kubeadm_pid - local msi_exit=0 token_exit=0 kubeadm_exit=0 + local msi_pid token_pid kubeadm_pid azlinux3_pid="" + local msi_exit=0 token_exit=0 kubeadm_exit=0 azlinux3_exit=0 ensure_daemon_csr_approver @@ -271,9 +271,17 @@ node_join_all() { node_join_kubeadm & kubeadm_pid=$! + if [[ "$(state_get azlinux3_enabled 0)" == "1" ]]; then + node_join_azlinux3 & + azlinux3_pid=$! + fi + wait "${msi_pid}" || msi_exit=$? wait "${token_pid}" || token_exit=$? wait "${kubeadm_pid}" || kubeadm_exit=$? + if [[ -n "${azlinux3_pid}" ]]; then + wait "${azlinux3_pid}" || azlinux3_exit=$? + fi local duration duration=$(timer_elapsed "${start}") @@ -287,8 +295,11 @@ node_join_all() { if [[ "${kubeadm_exit}" -ne 0 ]]; then log_error "Kubeadm node join failed (exit ${kubeadm_exit})" fi + if [[ "${azlinux3_exit}" -ne 0 ]]; then + log_error "Azure Linux 3 node join failed (exit ${azlinux3_exit})" + fi - if [[ "${msi_exit}" -ne 0 || "${token_exit}" -ne 0 || "${kubeadm_exit}" -ne 0 ]]; then + if [[ "${msi_exit}" -ne 0 || "${token_exit}" -ne 0 || "${kubeadm_exit}" -ne 0 || "${azlinux3_exit}" -ne 0 ]]; then log_error "Node joins failed (${duration}s)" return 1 fi @@ -304,8 +315,8 @@ node_unjoin_all() { local start start=$(timer_start) - local msi_pid token_pid kubeadm_pid - local msi_exit=0 token_exit=0 kubeadm_exit=0 + local msi_pid token_pid kubeadm_pid azlinux3_pid="" + local msi_exit=0 token_exit=0 kubeadm_exit=0 azlinux3_exit=0 node_unjoin_msi & msi_pid=$! @@ -316,9 +327,17 @@ node_unjoin_all() { node_unjoin_kubeadm & kubeadm_pid=$! + if [[ "$(state_get azlinux3_enabled 0)" == "1" ]]; then + node_unjoin_azlinux3 & + azlinux3_pid=$! + fi + wait "${msi_pid}" || msi_exit=$? wait "${token_pid}" || token_exit=$? wait "${kubeadm_pid}" || kubeadm_exit=$? + if [[ -n "${azlinux3_pid}" ]]; then + wait "${azlinux3_pid}" || azlinux3_exit=$? + fi local duration duration=$(timer_elapsed "${start}") @@ -332,8 +351,11 @@ node_unjoin_all() { if [[ "${kubeadm_exit}" -ne 0 ]]; then log_error "Kubeadm node unjoin failed (exit ${kubeadm_exit})" fi + if [[ "${azlinux3_exit}" -ne 0 ]]; then + log_error "Azure Linux 3 node unjoin failed (exit ${azlinux3_exit})" + fi - if [[ "${msi_exit}" -ne 0 || "${token_exit}" -ne 0 || "${kubeadm_exit}" -ne 0 ]]; then + if [[ "${msi_exit}" -ne 0 || "${token_exit}" -ne 0 || "${kubeadm_exit}" -ne 0 || "${azlinux3_exit}" -ne 0 ]]; then log_error "Node unjoins failed (${duration}s)" return 1 fi diff --git a/hack/e2e/lib/validate.sh b/hack/e2e/lib/validate.sh index 3bf60097..54f356ba 100755 --- a/hack/e2e/lib/validate.sh +++ b/hack/e2e/lib/validate.sh @@ -4,7 +4,7 @@ # # Functions: # validate_node_joined - Wait for a specific node to appear in kubectl -# validate_all_nodes - Verify MSI, token, and kubeadm nodes joined +# validate_all_nodes - Verify MSI, token, kubeadm, and optional Azure Linux 3 nodes joined # validate_node_absent - Wait for a node to disappear from kubectl # validate_all_nodes_absent - Verify all flex nodes are gone after unjoin # smoke_test