Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 3 additions & 1 deletion remote/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -504,7 +504,9 @@ deregister the AMIs, and delete their snapshots by hand to reclaim that storage.
`nvidiaDriverPackage`.
- **`deploy:image` fails with "recipe/component version already exists"**: you
changed a baked-in setting without bumping the `version` on the recipe (or
component) in `lib/image-stack.ts`. Bump and redeploy.
component) in `lib/image-stack.ts`. Bump and redeploy. The base Ubuntu image
is exempt — the recipe name carries the AMI id, so a new Canonical release
replaces the recipe on its own.
- **`start` reaches `running` but never `ready`, or the model is empty**: the
weights aren't in S3 yet, or a seed is still running. Ask the seed:
`spinloop remote seed ls`, then `spinloop remote seed status <seed-id>`. A seed
Expand Down
18 changes: 16 additions & 2 deletions remote/lib/image-stack.ts
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,11 @@ const AMI_ROOT_DEVICE = '/dev/sda1';
// immutable, so bump a runner's version to force a fresh AMI for just it.
// 3.4.0: spinloop is no longer baked — each instance's boot installs it (the
// deploy config's pin, or latest), so the AMI needs no release of it at all.
const RUNNER_VERSION = { vllm: '3.4.0', llamacpp: '3.4.0' } as const;
// 3.5.0: the outfit -> spinloop rename moved the baked daemon directory to
// /var/lib/spinloop and the crash-nudge unit to spinloop-nudge; the boot script
// pins both names, so a 3.4.0 AMI logrotates the wrong path and has no timer to
// enable.
const RUNNER_VERSION = { vllm: '3.5.0', llamacpp: '3.5.0' } as const;

/**
* Bakes a slim, model-agnostic AMI **per runner** — vLLM (a `uv` venv) and
Expand Down Expand Up @@ -81,6 +85,8 @@ export class ImageStack extends cdk.Stack {
terminateInstanceOnFailure: true,
});

// Resolved at deploy time, so Canonical publishing a new Ubuntu 24.04 image
// silently changes it between two deploys of an unchanged stack.
const parentImage = ssm.StringParameter.valueForStringParameter(this, UBUNTU_SSM_PARAMETER);
const nvidiaParam = { name: 'NvidiaDriverPackage', value: [cfg.nvidiaDriverPackage] };

Expand All @@ -106,7 +112,15 @@ export class ImageStack extends cdk.Stack {
});

const recipe = new imagebuilder.CfnImageRecipe(this, `${build.runner}Recipe`, {
name: `${this.stackName}-${build.runner}-recipe`,
// Every recipe property is create-only, so any change replaces the
// recipe — and Image Builder rejects the replacement if a recipe of the
// same name and version already exists. `parentImage` moves on its own
// whenever Canonical publishes, so the name carries the base AMI id:
// a new base image means a new recipe name, and the replacement lands
// instead of colliding with the one it is replacing. Changes we control
// (component data, driver, volume size) are still covered by bumping
// RUNNER_VERSION.
name: `${this.stackName}-${build.runner}-recipe-${parentImage}`,
version: RUNNER_VERSION[build.runner as keyof typeof RUNNER_VERSION],
parentImage,
components: [{ componentArn: component.attrArn, parameters: [...build.parameters] }],
Expand Down
14 changes: 14 additions & 0 deletions remote/test/stack.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -445,6 +445,20 @@ describe('ImageStack', () => {
template.resourceCountIs('AWS::ImageBuilder::Image', 0);
});

it('names each recipe after the base AMI it was built on', () => {
// Recipe properties are create-only and Image Builder rejects a create at
// an existing name+version. The parent image moves whenever Canonical
// publishes a new Ubuntu, so the name has to move with it or the
// replacement collides with the recipe it is replacing.
const recipes = Object.values(template.findResources('AWS::ImageBuilder::ImageRecipe'));
expect(recipes).toHaveLength(2);
for (const recipe of recipes) {
const [prefix, parentRef] = recipe.Properties.Name['Fn::Join'][1];
expect(prefix).toMatch(/-recipe-$/);
expect(parentRef).toEqual(recipe.Properties.ParentImage);
}
});

it('resizes the AMI root to the configured size on the right device', () => {
template.hasResourceProperties('AWS::ImageBuilder::ImageRecipe', {
BlockDeviceMappings: [
Expand Down