From cf96d61e33bf12fd29c1f506cc8ed46d3751b72c Mon Sep 17 00:00:00 2001 From: Pete Cornish Date: Tue, 1 Sep 2026 21:01:39 +0100 Subject: [PATCH] fix(remote): let the image recipe survive a new base AMI MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The recipe reads the "current" Ubuntu 24.04 SSM alias, so Canonical publishing a new image changes the parent between two deploys of unchanged code. Every recipe property is create-only, so that forces a replacement, which Image Builder then rejects for colliding with the recipe it is replacing — bootstrap fails on an account that was bootstrapped before the release. Name the recipe after its base AMI so the replacement lands. Also bump RUNNER_VERSION to 3.5.0: the outfit -> spinloop rename moved the baked daemon directory and the crash-nudge unit, which the boot script pins by their new names, so 3.4.0 AMIs need a rebake anyway. --- remote/README.md | 4 +++- remote/lib/image-stack.ts | 18 ++++++++++++++++-- remote/test/stack.test.ts | 14 ++++++++++++++ 3 files changed, 33 insertions(+), 3 deletions(-) diff --git a/remote/README.md b/remote/README.md index c5d21d88..1a3bcd4d 100644 --- a/remote/README.md +++ b/remote/README.md @@ -504,7 +504,9 @@ deregister the AMIs, and delete their snapshots by hand to reclaim that storage. `nvidiaDriverPackage`. - **`deploy:image` fails with "recipe/component version already exists"**: you changed a baked-in setting without bumping the `version` on the recipe (or - component) in `lib/image-stack.ts`. Bump and redeploy. + component) in `lib/image-stack.ts`. Bump and redeploy. The base Ubuntu image + is exempt — the recipe name carries the AMI id, so a new Canonical release + replaces the recipe on its own. - **`start` reaches `running` but never `ready`, or the model is empty**: the weights aren't in S3 yet, or a seed is still running. Ask the seed: `spinloop remote seed ls`, then `spinloop remote seed status `. A seed diff --git a/remote/lib/image-stack.ts b/remote/lib/image-stack.ts index d4b3924b..d9500c94 100644 --- a/remote/lib/image-stack.ts +++ b/remote/lib/image-stack.ts @@ -30,7 +30,11 @@ const AMI_ROOT_DEVICE = '/dev/sda1'; // immutable, so bump a runner's version to force a fresh AMI for just it. // 3.4.0: spinloop is no longer baked — each instance's boot installs it (the // deploy config's pin, or latest), so the AMI needs no release of it at all. -const RUNNER_VERSION = { vllm: '3.4.0', llamacpp: '3.4.0' } as const; +// 3.5.0: the outfit -> spinloop rename moved the baked daemon directory to +// /var/lib/spinloop and the crash-nudge unit to spinloop-nudge; the boot script +// pins both names, so a 3.4.0 AMI logrotates the wrong path and has no timer to +// enable. +const RUNNER_VERSION = { vllm: '3.5.0', llamacpp: '3.5.0' } as const; /** * Bakes a slim, model-agnostic AMI **per runner** — vLLM (a `uv` venv) and @@ -81,6 +85,8 @@ export class ImageStack extends cdk.Stack { terminateInstanceOnFailure: true, }); + // Resolved at deploy time, so Canonical publishing a new Ubuntu 24.04 image + // silently changes it between two deploys of an unchanged stack. const parentImage = ssm.StringParameter.valueForStringParameter(this, UBUNTU_SSM_PARAMETER); const nvidiaParam = { name: 'NvidiaDriverPackage', value: [cfg.nvidiaDriverPackage] }; @@ -106,7 +112,15 @@ export class ImageStack extends cdk.Stack { }); const recipe = new imagebuilder.CfnImageRecipe(this, `${build.runner}Recipe`, { - name: `${this.stackName}-${build.runner}-recipe`, + // Every recipe property is create-only, so any change replaces the + // recipe — and Image Builder rejects the replacement if a recipe of the + // same name and version already exists. `parentImage` moves on its own + // whenever Canonical publishes, so the name carries the base AMI id: + // a new base image means a new recipe name, and the replacement lands + // instead of colliding with the one it is replacing. Changes we control + // (component data, driver, volume size) are still covered by bumping + // RUNNER_VERSION. + name: `${this.stackName}-${build.runner}-recipe-${parentImage}`, version: RUNNER_VERSION[build.runner as keyof typeof RUNNER_VERSION], parentImage, components: [{ componentArn: component.attrArn, parameters: [...build.parameters] }], diff --git a/remote/test/stack.test.ts b/remote/test/stack.test.ts index 39a9ee73..1c312c73 100644 --- a/remote/test/stack.test.ts +++ b/remote/test/stack.test.ts @@ -445,6 +445,20 @@ describe('ImageStack', () => { template.resourceCountIs('AWS::ImageBuilder::Image', 0); }); + it('names each recipe after the base AMI it was built on', () => { + // Recipe properties are create-only and Image Builder rejects a create at + // an existing name+version. The parent image moves whenever Canonical + // publishes a new Ubuntu, so the name has to move with it or the + // replacement collides with the recipe it is replacing. + const recipes = Object.values(template.findResources('AWS::ImageBuilder::ImageRecipe')); + expect(recipes).toHaveLength(2); + for (const recipe of recipes) { + const [prefix, parentRef] = recipe.Properties.Name['Fn::Join'][1]; + expect(prefix).toMatch(/-recipe-$/); + expect(parentRef).toEqual(recipe.Properties.ParentImage); + } + }); + it('resizes the AMI root to the configured size on the right device', () => { template.hasResourceProperties('AWS::ImageBuilder::ImageRecipe', { BlockDeviceMappings: [