diff --git a/docs/usages/configuration.md b/docs/usages/configuration.md index bdf03812..90bf2754 100644 --- a/docs/usages/configuration.md +++ b/docs/usages/configuration.md @@ -85,6 +85,7 @@ At least one join or Azure authentication method must be configured. `azure.boot | `agent.logLevel` | string | Agent log verbosity. | `info` | | `agent.logDir` | string | Host directory for agent logs. | `/var/log/aks-flex-node` | | `agent.nodeName` | string | Optional Kubernetes node name override. Defaults to the host hostname. | `edge-node-01` | +| `agent.ociImage` | string | Optional nspawn rootfs OCI image. Set an Azure Linux image such as `ghcr.io/azure/agent-azlinux3:` to use Azure Linux in the nspawn machine. | `ghcr.io/azure/agent-azlinux3:` | | `agent.machineReconcileInterval` | duration string | Daemon interval for re-reading machine state. Uses Go duration syntax. | `10m` | | `agent.e2eMode` | boolean | Uses the local file-backed machine client for E2E tests. | `false` | | `agent.requireMachineRegistration` | boolean | Fails bootstrap when the AKS machine resource cannot be read or created. When false, registration is best-effort. | `false` | diff --git a/go.mod b/go.mod index 1a7aa7e6..6fe58b5a 100644 --- a/go.mod +++ b/go.mod @@ -9,14 +9,14 @@ require ( github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerservice/armcontainerservice/v8 v8.3.0-beta.2 github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/hybridcompute/armhybridcompute v1.2.0 github.com/Azure/kubelogin v0.2.15 - github.com/Azure/unbounded v0.1.11 + github.com/Azure/unbounded v0.1.18-0.20260619173346-9a7b731ce879 github.com/google/renameio/v2 v2.0.2 github.com/google/uuid v1.6.0 github.com/spf13/cobra v1.10.2 k8s.io/api v0.35.4 k8s.io/apimachinery v0.35.4 k8s.io/client-go v0.35.4 - k8s.io/utils v0.0.0-20251002143259-bc988d571ff4 + k8s.io/utils v0.0.0-20260319190234-28399d86e0b5 sigs.k8s.io/controller-runtime v0.23.3 ) @@ -38,9 +38,9 @@ require ( github.com/blang/semver/v4 v4.0.0 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/containerd/log v0.1.0 // indirect - github.com/containerd/platforms v0.2.1 // indirect + github.com/containerd/platforms v1.0.0-rc.4 // indirect github.com/cpuguy83/go-md2man/v2 v2.0.7 // indirect - github.com/cyphar/filepath-securejoin v0.5.0 // indirect + github.com/cyphar/filepath-securejoin v0.6.0 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/docker/go-units v0.5.0 // indirect github.com/emicklei/go-restful/v3 v3.13.0 // indirect @@ -72,23 +72,23 @@ require ( github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/opencontainers/go-digest v1.0.0 // indirect github.com/opencontainers/image-spec v1.1.1 // indirect - github.com/opencontainers/runtime-spec v1.2.1 // indirect + github.com/opencontainers/runtime-spec v1.3.0 // indirect github.com/opencontainers/umoci v0.6.0 // indirect github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c // indirect github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/prometheus/client_golang v1.23.2 // indirect github.com/prometheus/client_model v0.6.2 // indirect - github.com/prometheus/common v0.66.1 // indirect - github.com/prometheus/procfs v0.16.1 // indirect + github.com/prometheus/common v0.67.5 // indirect + github.com/prometheus/procfs v0.20.1 // indirect github.com/rootless-containers/proto/go-proto v0.0.0-20230421021042-4cd87ebadd67 // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect github.com/sirupsen/logrus v1.9.4 // indirect github.com/spf13/pflag v1.0.10 // indirect - github.com/urfave/cli v1.22.12 // indirect + github.com/urfave/cli v1.22.16 // indirect github.com/vbatts/go-mtree v0.6.1-0.20250911112631-8307d76bc1b9 // indirect github.com/x448/float16 v0.8.4 // indirect - go.yaml.in/yaml/v2 v2.4.3 // indirect + go.yaml.in/yaml/v2 v2.4.4 // indirect go.yaml.in/yaml/v3 v3.0.4 // indirect golang.org/x/crypto v0.52.0 // indirect golang.org/x/net v0.55.0 // indirect @@ -97,18 +97,18 @@ require ( golang.org/x/sys v0.45.0 // indirect golang.org/x/term v0.43.0 // indirect golang.org/x/text v0.37.0 // indirect - golang.org/x/time v0.11.0 // indirect + golang.org/x/time v0.15.0 // indirect gomodules.xyz/jsonpatch/v2 v2.4.0 // indirect - google.golang.org/protobuf v1.36.11 // indirect + google.golang.org/protobuf v1.36.12-0.20260120151049-f2248ac996af // indirect gopkg.in/evanphx/json-patch.v4 v4.13.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect k8s.io/apiextensions-apiserver v0.35.0 // indirect - k8s.io/klog/v2 v2.130.1 // indirect - k8s.io/kube-openapi v0.0.0-20250910181357-589584f1c912 // indirect + k8s.io/klog/v2 v2.140.0 // indirect + k8s.io/kube-openapi v0.0.0-20260319004828-5883c5ee87b9 // indirect oras.land/oras-go/v2 v2.6.0 // indirect sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730 // indirect sigs.k8s.io/randfill v1.0.0 // indirect - sigs.k8s.io/structured-merge-diff/v6 v6.3.2-0.20260122202528-d9cc6641c482 // indirect + sigs.k8s.io/structured-merge-diff/v6 v6.3.2 // indirect sigs.k8s.io/yaml v1.6.0 // indirect ) diff --git a/go.sum b/go.sum index a7ef61c0..8b7373ae 100644 --- a/go.sum +++ b/go.sum @@ -36,13 +36,13 @@ github.com/Azure/go-autorest/tracing v0.6.0 h1:TYi4+3m5t6K48TGI9AUdb+IzbnSxvnvUM github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU= github.com/Azure/kubelogin v0.2.15 h1:oJqD8Dvput3rO/xZgMTU+hBrcgg0BfQGPCNHJ2dEmys= github.com/Azure/kubelogin v0.2.15/go.mod h1:RwJS8TzSHTVQhfIZA4HLS79QGfvIp0ocIVLT5oHS/ls= -github.com/Azure/unbounded v0.1.11 h1:U0tQa/K2F1WVKbMFf7A1KD+RzpjiMvCdY2cMpBtpiJM= -github.com/Azure/unbounded v0.1.11/go.mod h1:8/ekWflNUvo8KTVPSYBX+7w/JpcoaeE7DTsdMfVfI6c= +github.com/Azure/unbounded v0.1.18-0.20260619173346-9a7b731ce879 h1:Puq+/5KWe86I5eTIPQAiAubCh8Wj2U0Nbn6qc/jmBQY= +github.com/Azure/unbounded v0.1.18-0.20260619173346-9a7b731ce879/go.mod h1:nTkSAO2q0NLMbdNDSDqd9j32wvoZUtheAQ5zsIRvRwg= github.com/AzureAD/microsoft-authentication-extensions-for-go/cache v0.1.1 h1:WJTmL004Abzc5wDB5VtZG2PJk5ndYDgVacGqfirKxjM= github.com/AzureAD/microsoft-authentication-extensions-for-go/cache v0.1.1/go.mod h1:tCcJZ0uHAmvjsVYzEFivsRTN00oz5BEsRgQHu5JZ9WE= github.com/AzureAD/microsoft-authentication-library-for-go v1.7.2 h1:RHK7bS+HQMslb1sZpAokUt+zTVmue0hKSs2C791hhzU= github.com/AzureAD/microsoft-authentication-library-for-go v1.7.2/go.mod h1:HKpQxkWaGLJ+D/5H8QRpyQXA1eKjxkFlOMwck5+33Jk= -github.com/BurntSushi/toml v1.2.1/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= +github.com/BurntSushi/toml v1.4.0/go.mod h1:ukJfTF/6rtPPRCnwkur4qwRxa8vTRFBF0uk2lLoLwho= github.com/Masterminds/semver/v3 v3.5.0 h1:kQceYJfbupGfZOKZQg0kou0DgAKhzDg2NZPAwZ/2OOE= github.com/Masterminds/semver/v3 v3.5.0/go.mod h1:4V+yj/TJE1HU9XfppCwVMZq3I84lprf4nC11bSS5beM= github.com/apex/log v1.9.0 h1:FHtw/xuaM8AgmvDDTI9fiwoAL25Sq2cxojnZICUU8l0= @@ -60,15 +60,15 @@ github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UF github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/containerd/log v0.1.0 h1:TCJt7ioM2cr/tfR8GPbGf9/VRAX8D2B4PjzCpfX540I= github.com/containerd/log v0.1.0/go.mod h1:VRRf09a7mHDIRezVKTRCrOq78v577GXq3bSa3EhrzVo= -github.com/containerd/platforms v0.2.1 h1:zvwtM3rz2YHPQsF2CHYM8+KtB5dvhISiXh5ZpSBQv6A= -github.com/containerd/platforms v0.2.1/go.mod h1:XHCb+2/hzowdiut9rkudds9bE5yJ7npe7dG/wG+uFPw= -github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= +github.com/containerd/platforms v1.0.0-rc.4 h1:M42JrUT4zfZTqtkUwkr0GzmUWbfyO5VO0Q5b3op97T4= +github.com/containerd/platforms v1.0.0-rc.4/go.mod h1:lKlMXyLybmBedS/JJm11uDofzI8L2v0J2ZbYvNsbq1A= +github.com/cpuguy83/go-md2man/v2 v2.0.5/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g= github.com/cpuguy83/go-md2man/v2 v2.0.7 h1:zbFlGlXEAKlwXpmvle3d8Oe3YnkKIK4xSRTd3sHPnBo= github.com/cpuguy83/go-md2man/v2 v2.0.7/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= -github.com/cyphar/filepath-securejoin v0.5.0 h1:hIAhkRBMQ8nIeuVwcAoymp7MY4oherZdAxD+m0u9zaw= -github.com/cyphar/filepath-securejoin v0.5.0/go.mod h1:Sdj7gXlvMcPZsbhwhQ33GguGLDGQL7h7bg04C/+u9jI= +github.com/cyphar/filepath-securejoin v0.6.0 h1:BtGB77njd6SVO6VztOHfPxKitJvd/VPT+OFBFMOi1Is= +github.com/cyphar/filepath-securejoin v0.6.0/go.mod h1:A8hd4EnAeyujCJRrICiOWqjS1AX0a9kM5XL+NwKoYSc= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= @@ -186,8 +186,8 @@ github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8 github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= github.com/opencontainers/image-spec v1.1.1 h1:y0fUlFfIZhPF1W537XOLg0/fcx6zcHCJwooC2xJA040= github.com/opencontainers/image-spec v1.1.1/go.mod h1:qpqAh3Dmcf36wStyyWU+kCeDgrGnAve2nCC8+7h8Q0M= -github.com/opencontainers/runtime-spec v1.2.1 h1:S4k4ryNgEpxW1dzyqffOmhI1BHYcjzU8lpJfSlR0xww= -github.com/opencontainers/runtime-spec v1.2.1/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= +github.com/opencontainers/runtime-spec v1.3.0 h1:YZupQUdctfhpZy3TM39nN9Ika5CBWT5diQ8ibYCRkxg= +github.com/opencontainers/runtime-spec v1.3.0/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= github.com/opencontainers/umoci v0.6.0 h1:Dsm4beJpglN5y2E2EUSZZcNey4Ml4+nKepvwLQwgIec= github.com/opencontainers/umoci v0.6.0/go.mod h1:2DS3cxVN9pRJGYaCK5mnmmwVKV5vd9r6HIYAV0IvdbI= github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c h1:+mdjkGKdHQG3305AYmdv1U2eRNDiU2ErMBj1gwrq8eQ= @@ -202,10 +202,10 @@ github.com/prometheus/client_golang v1.23.2 h1:Je96obch5RDVy3FDMndoUsjAhG5Edi49h github.com/prometheus/client_golang v1.23.2/go.mod h1:Tb1a6LWHB3/SPIzCoaDXI4I8UHKeFTEQ1YCr+0Gyqmg= github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNwqPLxwZyk= github.com/prometheus/client_model v0.6.2/go.mod h1:y3m2F6Gdpfy6Ut/GBsUqTWZqCUvMVzSfMLjcu6wAwpE= -github.com/prometheus/common v0.66.1 h1:h5E0h5/Y8niHc5DlaLlWLArTQI7tMrsfQjHV+d9ZoGs= -github.com/prometheus/common v0.66.1/go.mod h1:gcaUsgf3KfRSwHY4dIMXLPV0K/Wg1oZ8+SbZk/HH/dA= -github.com/prometheus/procfs v0.16.1 h1:hZ15bTNuirocR6u0JZ6BAHHmwS1p8B4P6MRqxtzMyRg= -github.com/prometheus/procfs v0.16.1/go.mod h1:teAbpZRB1iIAJYREa1LsoWUXykVXA1KlTmWl8x/U+Is= +github.com/prometheus/common v0.67.5 h1:pIgK94WWlQt1WLwAC5j2ynLaBRDiinoAb86HZHTUGI4= +github.com/prometheus/common v0.67.5/go.mod h1:SjE/0MzDEEAyrdr5Gqc6G+sXI67maCxzaT3A2+HqjUw= +github.com/prometheus/procfs v0.20.1 h1:XwbrGOIplXW/AU3YhIhLODXMJYyC1isLFfYCsTEycfc= +github.com/prometheus/procfs v0.20.1/go.mod h1:o9EMBZGRyvDrSPH1RqdxhojkuXstoe4UlK79eF5TGGo= github.com/rogpeppe/fastuuid v1.1.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= @@ -235,6 +235,8 @@ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= github.com/tj/assert v0.0.0-20171129193455-018094318fb0/go.mod h1:mZ9/Rh9oLWpLLDRpvE+3b7gP/C2YyLFYxNmcLnPTMe0= @@ -244,8 +246,8 @@ github.com/tj/go-buffer v1.1.0/go.mod h1:iyiJpfFcR2B9sXu7KvjbT9fpM4mOelRSDTbntVj github.com/tj/go-elastic v0.0.0-20171221160941-36157cbbebc2/go.mod h1:WjeM0Oo1eNAjXGDx2yma7uG2XoyRZTq1uv3M/o7imD0= github.com/tj/go-kinesis v0.0.0-20171128231115-08b17f58cb1b/go.mod h1:/yhzCV0xPfx6jb1bBgRFjl5lytqVqZXEaeqWP8lTEao= github.com/tj/go-spin v1.1.0/go.mod h1:Mg1mzmePZm4dva8Qz60H2lHwmJ2loum4VIrLgVnKwh4= -github.com/urfave/cli v1.22.12 h1:igJgVw1JdKH+trcLWLeLwZjU9fEfPesQ+9/e4MQ44S8= -github.com/urfave/cli v1.22.12/go.mod h1:sSBEIC79qR6OvcmsD4U3KABeOTxDqQtdDnaFuUN30b8= +github.com/urfave/cli v1.22.16 h1:MH0k6uJxdwdeWQTwhSO42Pwr4YLrNLwBtg1MRgTqPdQ= +github.com/urfave/cli v1.22.16/go.mod h1:EeJR6BKodywf4zciqrdw6hpCPk68JO9z5LazXZMn5Po= github.com/vbatts/go-mtree v0.6.1-0.20250911112631-8307d76bc1b9 h1:R6l9BtUe83abUGu1YKGkfa17wMMFLt6mhHVQ8MxpfRE= github.com/vbatts/go-mtree v0.6.1-0.20250911112631-8307d76bc1b9/go.mod h1:W7bcG9PCn6lFY+ljGlZxx9DONkxL3v8a7HyN+PrSrjA= github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= @@ -253,14 +255,14 @@ github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcY github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= -go.uber.org/mock v0.5.0 h1:KAMbZvZPyBPWgD14IrIQ38QCyjwpvVVV6K/bHl1IwQU= -go.uber.org/mock v0.5.0/go.mod h1:ge71pBPLYDk7QIi1LupWxdAykm7KIEFchiOqd6z7qMM= +go.uber.org/mock v0.6.0 h1:hyF9dfmbgIX5EfOdasqLsWD6xqpNZlXblLB/Dbnwv3Y= +go.uber.org/mock v0.6.0/go.mod h1:KiVJ4BqZJaMj4svdfmHM0AUx4NJYO8ZNpPnZn1Z+BBU= go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= -go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8= -go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= -go.yaml.in/yaml/v2 v2.4.3 h1:6gvOSjQoTB3vt1l+CU+tSyi/HOjfOjRLJ4YwYZGwRO0= -go.yaml.in/yaml/v2 v2.4.3/go.mod h1:zSxWcmIDjOzPXpjlTTbAsKokqkDNAVtZO0WOMiT90s8= +go.uber.org/zap v1.27.1 h1:08RqriUEv8+ArZRYSTXy1LeBScaMpVSTBhCeaZYfMYc= +go.uber.org/zap v1.27.1/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= +go.yaml.in/yaml/v2 v2.4.4 h1:tuyd0P+2Ont/d6e2rl3be67goVK4R6deVxCUX5vyPaQ= +go.yaml.in/yaml/v2 v2.4.4/go.mod h1:gMZqIpDtDqOfM0uNfy0SkpRhvUryYH0Z6wdMYcacYXQ= go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc= go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= @@ -315,8 +317,8 @@ golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.37.0 h1:Cqjiwd9eSg8e0QAkyCaQTNHFIIzWtidPahFWR83rTrc= golang.org/x/text v0.37.0/go.mod h1:a5sjxXGs9hsn/AJVwuElvCAo9v8QYLzvavO5z2PiM38= -golang.org/x/time v0.11.0 h1:/bpjEDfN9tkoN/ryeYHnv5hcMlc8ncjMcM4XBk5NWV0= -golang.org/x/time v0.11.0/go.mod h1:CDIdPxbZBQxdj6cxyCIdrNogrJKMJ7pr37NYpMcMDSg= +golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U= +golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= @@ -328,8 +330,8 @@ gomodules.xyz/jsonpatch/v2 v2.4.0 h1:Ci3iUJyx9UeRx7CeFN8ARgGbkESwJK+KB9lLcWxY/Zw gomodules.xyz/jsonpatch/v2 v2.4.0/go.mod h1:AH3dM2RI6uoBZxn3LVrfvJ3E0/9dG4cSrbuBJT4moAY= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE= -google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= +google.golang.org/protobuf v1.36.12-0.20260120151049-f2248ac996af h1:+5/Sw3GsDNlEmu7TfklWKPdQ0Ykja5VEmq2i817+jbI= +google.golang.org/protobuf v1.36.12-0.20260120151049-f2248ac996af/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= @@ -357,12 +359,12 @@ k8s.io/apimachinery v0.35.4 h1:xtdom9RG7e+yDp71uoXoJDWEE2eOiHgeO4GdBzwWpds= k8s.io/apimachinery v0.35.4/go.mod h1:NNi1taPOpep0jOj+oRha3mBJPqvi0hGdaV8TCqGQ+cc= k8s.io/client-go v0.35.4 h1:DN6fyaGuzK64UvnKO5fOA6ymSjvfGAnCAHAR0C66kD8= k8s.io/client-go v0.35.4/go.mod h1:2Pg9WpsS4NeOpoYTfHHfMxBG8zFMSAUi4O/qoiJC3nY= -k8s.io/klog/v2 v2.130.1 h1:n9Xl7H1Xvksem4KFG4PYbdQCQxqc/tTUyrgXaOhHSzk= -k8s.io/klog/v2 v2.130.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE= -k8s.io/kube-openapi v0.0.0-20250910181357-589584f1c912 h1:Y3gxNAuB0OBLImH611+UDZcmKS3g6CthxToOb37KgwE= -k8s.io/kube-openapi v0.0.0-20250910181357-589584f1c912/go.mod h1:kdmbQkyfwUagLfXIad1y2TdrjPFWp2Q89B3qkRwf/pQ= -k8s.io/utils v0.0.0-20251002143259-bc988d571ff4 h1:SjGebBtkBqHFOli+05xYbK8YF1Dzkbzn+gDM4X9T4Ck= -k8s.io/utils v0.0.0-20251002143259-bc988d571ff4/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= +k8s.io/klog/v2 v2.140.0 h1:Tf+J3AH7xnUzZyVVXhTgGhEKnFqye14aadWv7bzXdzc= +k8s.io/klog/v2 v2.140.0/go.mod h1:o+/RWfJ6PwpnFn7OyAG3QnO47BFsymfEfrz6XyYSSp0= +k8s.io/kube-openapi v0.0.0-20260319004828-5883c5ee87b9 h1:Sztf7ESG9tAXRW/ACJZjrj5jhdOUqS2KFRQT+CTvu78= +k8s.io/kube-openapi v0.0.0-20260319004828-5883c5ee87b9/go.mod h1:uGBT7iTA6c6MvqUvSXIaYZo9ukscABYi2btjhvgKGZ0= +k8s.io/utils v0.0.0-20260319190234-28399d86e0b5 h1:kBawHLSnx/mYHmRnNUf9d4CpjREbeZuxoSGOX/J+aYM= +k8s.io/utils v0.0.0-20260319190234-28399d86e0b5/go.mod h1:xDxuJ0whA3d0I4mf/C4ppKHxXynQ+fxnkmQH0vTHnuk= oras.land/oras-go/v2 v2.6.0 h1:X4ELRsiGkrbeox69+9tzTu492FMUu7zJQW6eJU+I2oc= oras.land/oras-go/v2 v2.6.0/go.mod h1:magiQDfG6H1O9APp+rOsvCPcW1GD2MM7vgnKY0Y+u1o= sigs.k8s.io/controller-runtime v0.23.3 h1:VjB/vhoPoA9l1kEKZHBMnQF33tdCLQKJtydy4iqwZ80= @@ -371,7 +373,7 @@ sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730 h1:IpInykpT6ceI+QxKBbEflcR5E sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730/go.mod h1:mdzfpAEoE6DHQEN0uh9ZbOCuHbLK5wOm7dK4ctXE9Tg= sigs.k8s.io/randfill v1.0.0 h1:JfjMILfT8A6RbawdsK2JXGBR5AQVfd+9TbzrlneTyrU= sigs.k8s.io/randfill v1.0.0/go.mod h1:XeLlZ/jmk4i1HRopwe7/aU3H5n1zNUcX6TM94b3QxOY= -sigs.k8s.io/structured-merge-diff/v6 v6.3.2-0.20260122202528-d9cc6641c482 h1:2WOzJpHUBVrrkDjU4KBT8n5LDcj824eX0I5UKcgeRUs= -sigs.k8s.io/structured-merge-diff/v6 v6.3.2-0.20260122202528-d9cc6641c482/go.mod h1:M3W8sfWvn2HhQDIbGWj3S099YozAsymCo/wrT5ohRUE= +sigs.k8s.io/structured-merge-diff/v6 v6.3.2 h1:kwVWMx5yS1CrnFWA/2QHyRVJ8jM6dBA80uLmm0wJkk8= +sigs.k8s.io/structured-merge-diff/v6 v6.3.2/go.mod h1:M3W8sfWvn2HhQDIbGWj3S099YozAsymCo/wrT5ohRUE= sigs.k8s.io/yaml v1.6.0 h1:G8fkbMSAFqgEFgh4b1wmtzDnioxFCUgTZhlbj5P9QYs= sigs.k8s.io/yaml v1.6.0/go.mod h1:796bPqUfzR/0jLAl6XjHl3Ck7MiyVv8dbTdyT3/pMf4= diff --git a/hack/e2e/README.md b/hack/e2e/README.md index b86b9018..5acfec57 100644 --- a/hack/e2e/README.md +++ b/hack/e2e/README.md @@ -1,6 +1,6 @@ # AKS Flex Node E2E Tests -The E2E suite provisions an AKS cluster and three Ubuntu VMs in Azure, joins the VMs as Flex Nodes, validates workloads, exercises unjoin/rejoin behavior, validates repave, collects logs, and tears down the resources. +The E2E suite provisions an AKS cluster, three Ubuntu VMs, and one Azure Linux 3 VM in Azure. It joins the VMs as Flex Nodes, validates workloads, exercises unjoin/rejoin behavior, validates repave, collects logs, and tears down the resources. ## Prerequisites @@ -27,8 +27,8 @@ make e2e The default `all` command runs: 1. Build the local `aks-flex-node` binary unless `--binary` or `--skip-build` is used. -2. Deploy AKS and three VMs with Bicep. -3. Join all three VMs. +2. Deploy AKS and four VMs with Bicep. +3. Join all four VMs. 4. Validate node readiness, node-problem-detector status, and run smoke workloads. 5. Unjoin all Flex Nodes and verify they are absent. 6. Rejoin all Flex Nodes and validate again. @@ -42,15 +42,17 @@ The default `all` command runs: | Command | Description | |---------|-------------| | `all` | Full flow: build, infra, join, validate, unjoin, validate absent, rejoin, validate, repave, logs, cleanup. | -| `infra` | Deploy AKS cluster and three VMs via Bicep. | +| `infra` | Deploy AKS cluster and four VMs via Bicep. | | `join` | Join all Flex Node VMs. | | `join-msi` | Join only the managed-identity node. | | `join-token` | Join only the bootstrap-token node. | | `join-kubeadm` | Join only the kubeadm-style bootstrap-token node. | +| `join-azlinux3` | Join only the Azure Linux 3 bootstrap-token node. | | `unjoin` | Unjoin all Flex Node VMs. | | `unjoin-msi` | Unjoin only the managed-identity node. | | `unjoin-token` | Unjoin only the bootstrap-token node. | | `unjoin-kubeadm` | Unjoin only the kubeadm-style node. | +| `unjoin-azlinux3` | Unjoin only the Azure Linux 3 bootstrap-token node. | | `validate` | Verify joined nodes, node-problem-detector status, and run smoke tests. | | `validate-absent` | Verify Flex Node objects are absent after unjoin. | | `smoke` | Run smoke workloads only. | @@ -77,6 +79,7 @@ Additional environment variables: |----------|---------|-------------| | `E2E_SSH_KEY_FILE` | auto-detected | SSH public key used for VM access. | | `E2E_WORK_DIR` | `/tmp/aks-flex-node-e2e` | Working directory for state, configs, and logs. | +| `E2E_KUBECONFIG` | `$E2E_WORK_DIR/kubeconfig` | Per-run kubeconfig path. Defaults to an isolated file instead of the runner-global kubeconfig. | | `E2E_KUBERNETES_VERSION` | `1.35.0` | Kubernetes version used in generated node configs. | | `E2E_CONTAINERD_VERSION` | `2.0.4` | Containerd version used in generated node configs. | | `E2E_RUNC_VERSION` | `1.1.12` | Runc version used in generated node configs. | @@ -90,13 +93,14 @@ Additional environment variables: ## Join Modes -The suite validates three join paths: +The suite validates four join paths: | VM | Auth Mode | Join Path | |----|-----------|-----------| | `vm-e2e-msi-*` | Managed Identity | Generated managed-identity config and `aks-flex-node start` flow. | | `vm-e2e-token-*` | Bootstrap Token | Kubernetes bootstrap token, RBAC, generated config, and `aks-flex-node start` flow. | | `vm-e2e-kubeadm-*` | Bootstrap Token | Kubeadm-style bootstrap resources plus generated config and `aks-flex-node start` flow. | +| `vm-e2e-azlinux3-*` | Bootstrap Token | Official Azure Linux 3 host image plus generated config with `agent.ociImage` pointing at the Azure Linux 3 nspawn image. | The bootstrap-token VM is provisioned with an uppercase guest OS hostname while its Azure resource name remains lowercase. This verifies that an omitted @@ -105,6 +109,8 @@ cluster under the lowercase VM name. Each join path uploads the locally built binary, renders a config file, installs the binary through `scripts/install.sh` with `AKS_FLEX_NODE_LOCAL_BINARY`, and starts the node through a transient systemd unit. The installed agent service is then validated with systemd checks. +The Azure Linux 3 path uses the official Azure marketplace image `MicrosoftCBLMariner:azure-linux-3:azure-linux-3-gen2:latest` for the VM and `ghcr.io/azure/agent-azlinux3:v20260619` for the nspawn rootfs. + ## Repave Validation The `upgrade-drift` command validates the local-machine-driven repave path: diff --git a/hack/e2e/infra/main.bicep b/hack/e2e/infra/main.bicep index 20f19db2..64ae401c 100644 --- a/hack/e2e/infra/main.bicep +++ b/hack/e2e/infra/main.bicep @@ -6,8 +6,9 @@ // - VM with system-assigned managed identity (MSI auth mode) // - VM without managed identity (bootstrap token auth mode) // - VM without managed identity (kubeadm apply -f auth mode) +// - Azure Linux 3 VM (bootstrap token auth mode) // -// All flex-node VMs run Ubuntu 22.04 LTS, have public IPs, and allow SSH +// Flex-node VMs have public IPs and allow SSH // ingress. VM creation is delegated to the reusable modules/vm.bicep module. // ============================================================================= @@ -40,6 +41,7 @@ var clusterName = 'aks-e2e-${nameSuffix}' var msiVmName = 'vm-e2e-msi-${nameSuffix}' var tokenVmName = 'vm-e2e-token-${nameSuffix}' var kubeadmVmName = 'vm-e2e-kubeadm-${nameSuffix}' +var azLinux3VmName = 'vm-e2e-azlinux3-${nameSuffix}' var vnetName = 'vnet-e2e-${nameSuffix}' var nsgName = 'nsg-e2e-${nameSuffix}' @@ -185,6 +187,24 @@ module vmKubeadm 'modules/vm.bicep' = { } } +module vmAzLinux3 'modules/vm.bicep' = { + name: 'deploy-vm-azlinux3' + params: { + location: location + vmName: azLinux3VmName + vmSize: vmSize + adminUsername: adminUsername + sshPublicKey: sshPublicKey + subnetId: vnet.properties.subnets[1].id + assignManagedIdentity: false + imagePublisher: 'MicrosoftCBLMariner' + imageOffer: 'azure-linux-3' + imageSku: 'azure-linux-3-gen2' + imageVersion: 'latest' + tags: tags + } +} + // --------------------------------------------------------------------------- // Role assignments: grant MSI VM permissions on the AKS cluster // --------------------------------------------------------------------------- @@ -228,4 +248,8 @@ output tokenVmPrivateIp string = vmToken.outputs.privateIpAddress output kubeadmVmName string = vmKubeadm.outputs.vmName output kubeadmVmIp string = vmKubeadm.outputs.publicIpAddress +output azLinux3VmName string = vmAzLinux3.outputs.vmName +output azLinux3VmIp string = vmAzLinux3.outputs.publicIpAddress +output azLinux3VmPrivateIp string = vmAzLinux3.outputs.privateIpAddress + output adminUsername string = adminUsername diff --git a/hack/e2e/infra/modules/vm.bicep b/hack/e2e/infra/modules/vm.bicep index 6c90df49..4eaf3000 100644 --- a/hack/e2e/infra/modules/vm.bicep +++ b/hack/e2e/infra/modules/vm.bicep @@ -1,8 +1,8 @@ // ============================================================================= -// modules/vm.bicep - Reusable Ubuntu flex-node VM module +// modules/vm.bicep - Reusable flex-node VM module // -// Creates a public IP, NIC, and Ubuntu VM in the given subnet. -// The VHD image defaults to Ubuntu 24.04 LTS (Noble) but can be overridden. +// Creates a public IP, NIC, and Linux VM in the given subnet. The marketplace +// image defaults to Ubuntu 24.04 LTS (Noble) but can be overridden. // ============================================================================= @description('Azure region for all resources.') diff --git a/hack/e2e/lib/cleanup.sh b/hack/e2e/lib/cleanup.sh index f4957e60..d7148a00 100755 --- a/hack/e2e/lib/cleanup.sh +++ b/hack/e2e/lib/cleanup.sh @@ -109,10 +109,11 @@ collect_logs() { mkdir -p "${E2E_LOG_DIR}" - local msi_vm_ip token_vm_ip kubeadm_vm_ip + local msi_vm_ip token_vm_ip kubeadm_vm_ip azlinux3_vm_ip msi_vm_ip="$(state_get msi_vm_ip)" token_vm_ip="$(state_get token_vm_ip)" kubeadm_vm_ip="$(state_get kubeadm_vm_ip)" + azlinux3_vm_ip="$(state_get azlinux3_vm_ip)" if [[ -n "${msi_vm_ip}" ]]; then _collect_vm_logs "${msi_vm_ip}" "msi" || true @@ -126,6 +127,10 @@ collect_logs() { _collect_vm_logs "${kubeadm_vm_ip}" "kubeadm" || true fi + if [[ -n "${azlinux3_vm_ip}" ]]; then + _collect_vm_logs "${azlinux3_vm_ip}" "azlinux3" || true + fi + # Also capture cluster-side info { echo "=== Nodes ===" @@ -167,12 +172,13 @@ cleanup() { stop_daemon_csr_approver - local resource_group cluster_name msi_vm_name token_vm_name kubeadm_vm_name + local resource_group cluster_name msi_vm_name token_vm_name kubeadm_vm_name azlinux3_vm_name resource_group="$(state_get resource_group)" cluster_name="$(state_get cluster_name)" msi_vm_name="$(state_get msi_vm_name)" token_vm_name="$(state_get token_vm_name)" kubeadm_vm_name="$(state_get kubeadm_vm_name)" + azlinux3_vm_name="$(state_get azlinux3_vm_name)" local deployment_name deployment_name="$(state_get deployment_name)" @@ -194,6 +200,12 @@ cleanup() { az vm delete --resource-group "${resource_group}" --name "${kubeadm_vm_name}" \ --force-deletion yes --yes --no-wait 2>/dev/null || true + if [[ -n "${azlinux3_vm_name}" ]]; then + log_info "[3/5] Deleting Azure Linux 3 VM: ${azlinux3_vm_name}..." + az vm delete --resource-group "${resource_group}" --name "${azlinux3_vm_name}" \ + --force-deletion yes --yes --no-wait 2>/dev/null || true + fi + # Clean up leftover networking resources tied to our deployment log_info "[4/5] Cleaning up networking resources..." local run_id="${GITHUB_RUN_ID:-}" diff --git a/hack/e2e/lib/common.sh b/hack/e2e/lib/common.sh index 10c6611d..45a74d52 100755 --- a/hack/e2e/lib/common.sh +++ b/hack/e2e/lib/common.sh @@ -173,6 +173,10 @@ load_config() { E2E_BINARY="${E2E_BINARY:-}" E2E_HELPER_BINARY="${E2E_HELPER_BINARY:-}" + # Keep E2E runs isolated from stale or corrupt runner-global kubeconfig state. + E2E_KUBECONFIG="${E2E_KUBECONFIG:-${E2E_WORK_DIR}/kubeconfig}" + export KUBECONFIG="${E2E_KUBECONFIG}" + # Skip cleanup for debugging E2E_SKIP_CLEANUP="${E2E_SKIP_CLEANUP:-0}" @@ -198,6 +202,7 @@ load_config() { log_info " Subscription: ${AZURE_SUBSCRIPTION_ID}" log_info " Name Suffix: ${E2E_NAME_SUFFIX}" log_info " Agent Pool: ${E2E_TARGET_AGENT_POOL_NAME}" + log_info " Kubeconfig: ${KUBECONFIG}" log_info " Skip Cleanup: ${E2E_SKIP_CLEANUP}" } diff --git a/hack/e2e/lib/infra.sh b/hack/e2e/lib/infra.sh index d0307984..c56cadff 100755 --- a/hack/e2e/lib/infra.sh +++ b/hack/e2e/lib/infra.sh @@ -106,6 +106,7 @@ infra_deploy() { local cluster_name cluster_id msi_vm_name msi_vm_ip msi_vm_principal_id local token_vm_name token_vm_ip token_vm_private_ip kubeadm_vm_name kubeadm_vm_ip admin_username + local azlinux3_vm_name azlinux3_vm_ip azlinux3_vm_private_ip cluster_name=$(echo "${outputs}" | jq -r '.clusterName.value') cluster_id=$(echo "${outputs}" | jq -r '.clusterId.value') @@ -117,6 +118,9 @@ infra_deploy() { token_vm_private_ip=$(echo "${outputs}" | jq -r '.tokenVmPrivateIp.value // ""') kubeadm_vm_name=$(echo "${outputs}" | jq -r '.kubeadmVmName.value') kubeadm_vm_ip=$(echo "${outputs}" | jq -r '.kubeadmVmIp.value') + azlinux3_vm_name=$(echo "${outputs}" | jq -r '.azLinux3VmName.value // ""') + azlinux3_vm_ip=$(echo "${outputs}" | jq -r '.azLinux3VmIp.value // ""') + azlinux3_vm_private_ip=$(echo "${outputs}" | jq -r '.azLinux3VmPrivateIp.value // ""') admin_username=$(echo "${outputs}" | jq -r '.adminUsername.value') if [[ -z "${token_vm_private_ip}" ]] || ! is_valid_ipv4 "${token_vm_private_ip}"; then @@ -124,6 +128,11 @@ infra_deploy() { return 1 fi + if [[ -z "${azlinux3_vm_private_ip}" ]] || ! is_valid_ipv4 "${azlinux3_vm_private_ip}"; then + log_error "Missing or invalid Azure Linux 3 VM private IP from deployment outputs: '${azlinux3_vm_private_ip}'" + return 1 + fi + # Persist to state state_set "cluster_name" "${cluster_name}" state_set "cluster_id" "${cluster_id}" @@ -135,6 +144,9 @@ infra_deploy() { state_set "token_vm_private_ip" "${token_vm_private_ip}" state_set "kubeadm_vm_name" "${kubeadm_vm_name}" state_set "kubeadm_vm_ip" "${kubeadm_vm_ip}" + state_set "azlinux3_vm_name" "${azlinux3_vm_name}" + state_set "azlinux3_vm_ip" "${azlinux3_vm_ip}" + state_set "azlinux3_vm_private_ip" "${azlinux3_vm_private_ip}" state_set "admin_username" "${admin_username}" state_set "resource_group" "${E2E_RESOURCE_GROUP}" state_set "location" "${E2E_LOCATION}" @@ -146,6 +158,7 @@ infra_deploy() { log_info "MSI VM: ${msi_vm_name} @ ${msi_vm_ip}" log_info "Token VM: ${token_vm_name} @ ${token_vm_ip}" log_info "Kubeadm VM: ${kubeadm_vm_name} @ ${kubeadm_vm_ip}" + log_info "AzLinux3 VM: ${azlinux3_vm_name} @ ${azlinux3_vm_ip}" # Get kubeconfig and extract cluster info infra_get_kubeconfig @@ -158,11 +171,14 @@ infra_deploy() { local pid_token=$! wait_for_ssh "${kubeadm_vm_ip}" & local pid_kubeadm=$! + wait_for_ssh "${azlinux3_vm_ip}" & + local pid_azlinux3=$! local ssh_failed=0 wait "${pid_msi}" || ssh_failed=1 wait "${pid_token}" || ssh_failed=1 wait "${pid_kubeadm}" || ssh_failed=1 + wait "${pid_azlinux3}" || ssh_failed=1 if [[ "${ssh_failed}" -eq 1 ]]; then log_error "One or more VMs not reachable via SSH" diff --git a/hack/e2e/lib/node-join-kubeadm.sh b/hack/e2e/lib/node-join-kubeadm.sh index da348161..5cd40086 100644 --- a/hack/e2e/lib/node-join-kubeadm.sh +++ b/hack/e2e/lib/node-join-kubeadm.sh @@ -46,6 +46,9 @@ subjects: - apiGroup: rbac.authorization.k8s.io kind: Group name: system:bootstrappers:aks-flex-node +- apiGroup: rbac.authorization.k8s.io + kind: Group + name: system:bootstrappers:kubeadm:default-node-token --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding @@ -59,6 +62,9 @@ subjects: - apiGroup: rbac.authorization.k8s.io kind: Group name: system:bootstrappers:aks-flex-node +- apiGroup: rbac.authorization.k8s.io + kind: Group + name: system:bootstrappers:kubeadm:default-node-token --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding @@ -85,6 +91,9 @@ subjects: - apiGroup: rbac.authorization.k8s.io kind: Group name: system:bootstrappers:aks-flex-node +- apiGroup: rbac.authorization.k8s.io + kind: Group + name: system:bootstrappers:kubeadm:default-node-token --- apiVersion: rbac.authorization.k8s.io/v1 kind: Role @@ -110,6 +119,9 @@ subjects: - kind: Group apiGroup: rbac.authorization.k8s.io name: system:bootstrappers:aks-flex-node +- kind: Group + apiGroup: rbac.authorization.k8s.io + name: system:bootstrappers:kubeadm:default-node-token --- apiVersion: rbac.authorization.k8s.io/v1 kind: Role @@ -135,6 +147,9 @@ subjects: - kind: Group apiGroup: rbac.authorization.k8s.io name: system:bootstrappers:aks-flex-node +- kind: Group + apiGroup: rbac.authorization.k8s.io + name: system:bootstrappers:kubeadm:default-node-token --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole @@ -157,6 +172,9 @@ subjects: - kind: Group apiGroup: rbac.authorization.k8s.io name: system:bootstrappers:aks-flex-node +- kind: Group + apiGroup: rbac.authorization.k8s.io + name: system:bootstrappers:kubeadm:default-node-token EOF # Publish the ConfigMaps that kubeadm join reads during its preflight phase. @@ -246,7 +264,7 @@ stringData: expiration: "${expiration}" usage-bootstrap-authentication: "true" usage-bootstrap-signing: "true" - auth-extra-groups: "system:bootstrappers:aks-flex-node" + auth-extra-groups: "system:bootstrappers:aks-flex-node,system:bootstrappers:kubeadm:default-node-token" EOF echo "${bootstrap_token}" diff --git a/hack/e2e/lib/node-join-token.sh b/hack/e2e/lib/node-join-token.sh index e925f9c0..dd19be59 100644 --- a/hack/e2e/lib/node-join-token.sh +++ b/hack/e2e/lib/node-join-token.sh @@ -79,6 +79,67 @@ node_join_token() { log_success "Token node joined in $(timer_elapsed "${start}")s" } +# --------------------------------------------------------------------------- +# node_join_azlinux3 - Join the optional Azure Linux 3 host VM using the +# Azure Linux 3 nspawn OCI image. +# --------------------------------------------------------------------------- +node_join_azlinux3() { + log_section "Joining Azure Linux 3 Node" + local start + start=$(timer_start) + + local vm_ip vm_private_ip cluster_name resource_group subscription_id + vm_ip="$(state_get azlinux3_vm_ip)" + vm_private_ip="$(state_get azlinux3_vm_private_ip)" + cluster_name="$(state_get cluster_name)" + resource_group="$(state_get resource_group)" + subscription_id="$(state_get subscription_id)" + + if [[ -z "${vm_private_ip}" ]] || ! is_valid_ipv4 "${vm_private_ip}"; then + log_error "Invalid Azure Linux 3 VM private IP in state: '${vm_private_ip}'" + return 1 + fi + + log_info "Setting up bootstrap token RBAC resources..." + "${REPO_ROOT}/scripts/aks-flex-config" setup-node-rbac \ + --resource-group "${resource_group}" \ + --cluster-name "${cluster_name}" \ + --subscription "${subscription_id}" + + ensure_daemon_csr_approver + + log_info "Generating Azure Linux 3 token config..." + local config_file="${E2E_WORK_DIR}/config-azlinux3.json" + "${REPO_ROOT}/scripts/aks-flex-config" generate-node-config \ + --resource-group "${resource_group}" \ + --cluster-name "${cluster_name}" \ + --subscription "${subscription_id}" \ + --agent-pool-name "${E2E_TARGET_AGENT_POOL_NAME}" \ + --bootstrap-token \ + --output "${config_file}" + + jq \ + --arg nodeIP "${vm_private_ip}" \ + --arg kubernetesVersion "${E2E_KUBERNETES_VERSION}" \ + --arg containerdVersion "${E2E_CONTAINERD_VERSION}" \ + --arg runcVersion "${E2E_RUNC_VERSION}" \ + '.agent.logLevel = "debug" + | .agent.e2eMode = true + | .agent.ociImage = "ghcr.io/azure/agent-azlinux3:v20260619" + | .node.kubelet.nodeIP = $nodeIP + | .components = (.components // {}) + | .components.kubernetes = $kubernetesVersion + | .components.containerd = $containerdVersion + | .components.runc = $runcVersion + | del(.kubernetes, .containerd, .runc)' \ + "${config_file}" > "${config_file}.tmp" + mv "${config_file}.tmp" "${config_file}" + + _deploy_and_start_agent "${vm_ip}" "${config_file}" "aks-flex-node-azlinux3" + + log_success "Azure Linux 3 node joined in $(timer_elapsed "${start}")s" +} + # --------------------------------------------------------------------------- # node_unjoin_token - Simulate RP delete and verify node cleanup # --------------------------------------------------------------------------- @@ -95,3 +156,17 @@ node_unjoin_token() { log_success "Token node unjoined in $(timer_elapsed "${start}")s" } + +node_unjoin_azlinux3() { + log_section "Unjoining Azure Linux 3 Node" + local start + start=$(timer_start) + + local vm_ip vm_name + vm_ip="$(state_get azlinux3_vm_ip)" + vm_name="$(state_get azlinux3_vm_name)" + + _rp_delete_unjoin_node "${vm_ip}" "${vm_name}" + + log_success "Azure Linux 3 node unjoined in $(timer_elapsed "${start}")s" +} diff --git a/hack/e2e/lib/node-join.sh b/hack/e2e/lib/node-join.sh index a1993671..97ce936c 100755 --- a/hack/e2e/lib/node-join.sh +++ b/hack/e2e/lib/node-join.sh @@ -8,7 +8,7 @@ # node-join-kubeadm.sh - Kubeadm apply -f join/unjoin (node_join_kubeadm, node_unjoin_kubeadm) # # Functions: -# node_join_all - Join all nodes (MSI, token, and kubeadm) in parallel +# node_join_all - Join all nodes (MSI, token, kubeadm, and optional Azure Linux 3) in parallel # node_unjoin_all - Unjoin all nodes in parallel # ============================================================================= set -euo pipefail @@ -257,8 +257,8 @@ node_join_all() { local start start=$(timer_start) - local msi_pid token_pid kubeadm_pid - local msi_exit=0 token_exit=0 kubeadm_exit=0 + local msi_pid token_pid kubeadm_pid azlinux3_pid="" + local msi_exit=0 token_exit=0 kubeadm_exit=0 azlinux3_exit=0 ensure_daemon_csr_approver @@ -271,9 +271,13 @@ node_join_all() { node_join_kubeadm & kubeadm_pid=$! + node_join_azlinux3 & + azlinux3_pid=$! + wait "${msi_pid}" || msi_exit=$? wait "${token_pid}" || token_exit=$? wait "${kubeadm_pid}" || kubeadm_exit=$? + wait "${azlinux3_pid}" || azlinux3_exit=$? local duration duration=$(timer_elapsed "${start}") @@ -287,8 +291,11 @@ node_join_all() { if [[ "${kubeadm_exit}" -ne 0 ]]; then log_error "Kubeadm node join failed (exit ${kubeadm_exit})" fi + if [[ "${azlinux3_exit}" -ne 0 ]]; then + log_error "Azure Linux 3 node join failed (exit ${azlinux3_exit})" + fi - if [[ "${msi_exit}" -ne 0 || "${token_exit}" -ne 0 || "${kubeadm_exit}" -ne 0 ]]; then + if [[ "${msi_exit}" -ne 0 || "${token_exit}" -ne 0 || "${kubeadm_exit}" -ne 0 || "${azlinux3_exit}" -ne 0 ]]; then log_error "Node joins failed (${duration}s)" return 1 fi @@ -304,8 +311,8 @@ node_unjoin_all() { local start start=$(timer_start) - local msi_pid token_pid kubeadm_pid - local msi_exit=0 token_exit=0 kubeadm_exit=0 + local msi_pid token_pid kubeadm_pid azlinux3_pid="" + local msi_exit=0 token_exit=0 kubeadm_exit=0 azlinux3_exit=0 node_unjoin_msi & msi_pid=$! @@ -316,9 +323,13 @@ node_unjoin_all() { node_unjoin_kubeadm & kubeadm_pid=$! + node_unjoin_azlinux3 & + azlinux3_pid=$! + wait "${msi_pid}" || msi_exit=$? wait "${token_pid}" || token_exit=$? wait "${kubeadm_pid}" || kubeadm_exit=$? + wait "${azlinux3_pid}" || azlinux3_exit=$? local duration duration=$(timer_elapsed "${start}") @@ -332,8 +343,11 @@ node_unjoin_all() { if [[ "${kubeadm_exit}" -ne 0 ]]; then log_error "Kubeadm node unjoin failed (exit ${kubeadm_exit})" fi + if [[ "${azlinux3_exit}" -ne 0 ]]; then + log_error "Azure Linux 3 node unjoin failed (exit ${azlinux3_exit})" + fi - if [[ "${msi_exit}" -ne 0 || "${token_exit}" -ne 0 || "${kubeadm_exit}" -ne 0 ]]; then + if [[ "${msi_exit}" -ne 0 || "${token_exit}" -ne 0 || "${kubeadm_exit}" -ne 0 || "${azlinux3_exit}" -ne 0 ]]; then log_error "Node unjoins failed (${duration}s)" return 1 fi diff --git a/hack/e2e/lib/validate.sh b/hack/e2e/lib/validate.sh index 4b24edda..dbe7bad5 100755 --- a/hack/e2e/lib/validate.sh +++ b/hack/e2e/lib/validate.sh @@ -4,7 +4,7 @@ # # Functions: # validate_node_joined - Wait for a specific node to appear in kubectl -# validate_all_nodes - Verify MSI, token, and kubeadm nodes joined +# validate_all_nodes - Verify MSI, token, kubeadm, and optional Azure Linux 3 nodes joined # validate_npd_status - Verify node-problem-detector is active # validate_node_absent - Wait for a node to disappear from kubectl # validate_all_nodes_absent - Verify all flex nodes are gone after unjoin @@ -178,25 +178,31 @@ validate_all_nodes() { --overwrite-existing \ --admin - local msi_vm_name token_vm_name kubeadm_vm_name - local msi_vm_ip token_vm_ip kubeadm_vm_ip - local token_vm_private_ip + local msi_vm_name token_vm_name kubeadm_vm_name azlinux3_vm_name + local msi_vm_ip token_vm_ip kubeadm_vm_ip azlinux3_vm_ip + local token_vm_private_ip azlinux3_vm_private_ip msi_vm_name="$(state_get msi_vm_name)" token_vm_name="$(state_get token_vm_name)" kubeadm_vm_name="$(state_get kubeadm_vm_name)" + azlinux3_vm_name="$(state_get azlinux3_vm_name)" msi_vm_ip="$(state_get msi_vm_ip)" token_vm_ip="$(state_get token_vm_ip)" kubeadm_vm_ip="$(state_get kubeadm_vm_ip)" + azlinux3_vm_ip="$(state_get azlinux3_vm_ip)" token_vm_private_ip="$(state_get token_vm_private_ip)" + azlinux3_vm_private_ip="$(state_get azlinux3_vm_private_ip)" local failed=0 validate_node_joined "${msi_vm_name}" || failed=1 validate_node_joined "${token_vm_name}" || failed=1 validate_node_joined "${kubeadm_vm_name}" || failed=1 validate_node_ip "${token_vm_name}" "${token_vm_private_ip}" || failed=1 + validate_node_joined "${azlinux3_vm_name}" || failed=1 + validate_node_ip "${azlinux3_vm_name}" "${azlinux3_vm_private_ip}" || failed=1 validate_npd_status "${msi_vm_name}" "${msi_vm_ip}" || failed=1 validate_npd_status "${token_vm_name}" "${token_vm_ip}" || failed=1 validate_npd_status "${kubeadm_vm_name}" "${kubeadm_vm_ip}" || failed=1 + validate_npd_status "${azlinux3_vm_name}" "${azlinux3_vm_ip}" || failed=1 if [[ "${failed}" -eq 1 ]]; then log_error "One or more nodes failed to join" @@ -241,16 +247,18 @@ validate_node_absent() { validate_all_nodes_absent() { log_section "Validating Nodes Absent After Unjoin" - local msi_vm_name token_vm_name kubeadm_vm_name + local msi_vm_name token_vm_name kubeadm_vm_name azlinux3_vm_name msi_vm_name="$(state_get msi_vm_name)" token_vm_name="$(state_get token_vm_name)" kubeadm_vm_name="$(state_get kubeadm_vm_name)" + azlinux3_vm_name="$(state_get azlinux3_vm_name)" local failed=0 # TODO: MSI validation skipped until credential plugin auth is supported log_info "Skipping MSI node absence validation (credential plugin auth not yet supported)" validate_node_absent "${token_vm_name}" || failed=1 validate_node_absent "${kubeadm_vm_name}" || failed=1 + validate_node_absent "${azlinux3_vm_name}" || failed=1 if [[ "${failed}" -eq 1 ]]; then log_error "One or more nodes still present after unjoin" @@ -319,10 +327,11 @@ EOF smoke_test_all() { log_section "Running Smoke Tests" - local msi_vm_name token_vm_name kubeadm_vm_name + local msi_vm_name token_vm_name kubeadm_vm_name azlinux3_vm_name msi_vm_name="$(state_get msi_vm_name)" token_vm_name="$(state_get token_vm_name)" kubeadm_vm_name="$(state_get kubeadm_vm_name)" + azlinux3_vm_name="$(state_get azlinux3_vm_name)" # A default bridge CNI config (99-bridge.conf) is written during bootstrap, # making nodes Ready without requiring unbounded-net-node DaemonSet. @@ -330,6 +339,7 @@ smoke_test_all() { smoke_test "${msi_vm_name}" "msi" || failed=1 smoke_test "${token_vm_name}" "token" || failed=1 smoke_test "${kubeadm_vm_name}" "kubeadm" || failed=1 + smoke_test "${azlinux3_vm_name}" "azlinux3" || failed=1 if [[ "${failed}" -eq 1 ]]; then log_error "One or more smoke tests failed" diff --git a/hack/e2e/run.sh b/hack/e2e/run.sh index b648468f..35b9d577 100755 --- a/hack/e2e/run.sh +++ b/hack/e2e/run.sh @@ -12,10 +12,12 @@ # join Join all nodes to the cluster (requires prior infra) # join-msi Join only the MSI node # join-token Join only the token node +# join-azlinux3 Join only the optional Azure Linux 3 token node # join-kubeadm Join only the kubeadm node (apply -f with KubeadmNodeJoin) # unjoin Unjoin all nodes from the cluster # unjoin-msi Unjoin only the MSI node # unjoin-token Unjoin only the token node +# unjoin-azlinux3 Unjoin only the optional Azure Linux 3 token node # unjoin-kubeadm Reset the kubeadm node and remove it from the cluster # validate Verify nodes joined + run smoke tests # validate-absent Verify all flex nodes are gone after unjoin @@ -115,7 +117,7 @@ usage() { parse_args() { while [[ $# -gt 0 ]]; do case "$1" in - all|infra|join|join-msi|join-token|join-kubeadm|unjoin|unjoin-msi|unjoin-token|unjoin-kubeadm|validate|validate-absent|smoke|upgrade-drift|logs|cleanup|status) + all|infra|join|join-msi|join-token|join-kubeadm|join-azlinux3|unjoin|unjoin-msi|unjoin-token|unjoin-kubeadm|unjoin-azlinux3|validate|validate-absent|smoke|upgrade-drift|logs|cleanup|status) COMMAND="$1"; shift ;; -g|--resource-group) export E2E_RESOURCE_GROUP="$2"; shift 2 ;; -l|--location) export E2E_LOCATION="$2"; shift 2 ;; @@ -249,6 +251,10 @@ main() { ensure_binary node_join_kubeadm ;; + join-azlinux3) + ensure_binary + node_join_azlinux3 + ;; unjoin) node_unjoin_all ;; @@ -261,6 +267,9 @@ main() { unjoin-kubeadm) node_unjoin_kubeadm ;; + unjoin-azlinux3) + node_unjoin_azlinux3 + ;; validate) validate_all_nodes smoke_test_all diff --git a/pkg/config/adapter.go b/pkg/config/adapter.go index 9330a7e3..4bbceb78 100644 --- a/pkg/config/adapter.go +++ b/pkg/config/adapter.go @@ -24,9 +24,7 @@ func ToAgentConfig(cfg *Config, machineName string) *agentconfig.AgentConfig { ac := &agentconfig.AgentConfig{ MachineName: machineName, NodeName: cfg.Agent.NodeName, - // TODO: implement support for overriding rootfs image from flex node config. - // Using empty string here means the agent will detect and use the default image. - // OCIImage: "", + OCIImage: cfg.Agent.OCIImage, Cluster: agentconfig.AgentClusterConfig{ CaCertBase64: cfg.Node.Kubelet.CACertData, ClusterDNS: cfg.Networking.DNSServiceIP, diff --git a/pkg/config/adapter_test.go b/pkg/config/adapter_test.go index e48d4213..a4ecb8da 100644 --- a/pkg/config/adapter_test.go +++ b/pkg/config/adapter_test.go @@ -65,13 +65,16 @@ func TestToAgentConfig_BootstrapToken(t *testing.T) { func TestToAgentConfig_NodeName(t *testing.T) { t.Parallel() - cfg := &Config{Agent: AgentConfig{NodeName: "worker-1"}} + cfg := &Config{Agent: AgentConfig{NodeName: "worker-1", OCIImage: "ghcr.io/azure/agent-azlinux3:tag"}} ac := ToAgentConfig(cfg, "kube1") if ac.NodeName != "worker-1" { t.Fatalf("NodeName=%q, want worker-1", ac.NodeName) } + if ac.OCIImage != "ghcr.io/azure/agent-azlinux3:tag" { + t.Fatalf("OCIImage=%q, want ghcr.io/azure/agent-azlinux3:tag", ac.OCIImage) + } } func TestToAgentConfig_ServicePrincipal(t *testing.T) { diff --git a/pkg/config/config.go b/pkg/config/config.go index a64483b0..aca329cf 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -136,6 +136,9 @@ type AgentConfig struct { LogDir string `json:"logDir"` // Directory for log files // NodeName is resolved from the host hostname when omitted. NodeName string `json:"nodeName,omitempty"` + // OCIImage optionally selects the nspawn rootfs image. Leave empty to use + // the unbounded default image. + OCIImage string `json:"ociImage,omitempty"` // MachineReconcileInterval controls how often the daemon re-reads the AKS // machine resource when no Kubernetes Node event wakes the controller. diff --git a/pkg/config/config_test.go b/pkg/config/config_test.go index cff4b8d1..44967748 100644 --- a/pkg/config/config_test.go +++ b/pkg/config/config_test.go @@ -50,11 +50,13 @@ func TestSetDefaults(t *testing.T) { Agent: AgentConfig{ LogLevel: "debug", LogDir: "/custom/log/dir", + OCIImage: "ghcr.io/azure/agent-azlinux3:tag", }, }, want: func(c *Config) bool { return c.Agent.LogLevel == "debug" && c.Agent.LogDir == "/custom/log/dir" && + c.Agent.OCIImage == "ghcr.io/azure/agent-azlinux3:tag" && c.Azure.Cloud == "AzurePublicCloud" && c.Azure.ResourceManagerEndpointURL == "https://management.example.test" && c.Azure.TargetAgentPoolName == "flexnode-edge"