From 8dec0b4d2ab048859580ecfcaf92bec2aef3f1ac Mon Sep 17 00:00:00 2001 From: ValentaTomas Date: Tue, 5 May 2026 01:18:39 -0700 Subject: [PATCH 01/10] feat(uffd): wire free-page-reporting through template build to FC balloon MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds the FC-side integration plumbing for free page reporting on top of the UFFD REMOVE-event handling in #2520: - template-manager proto: optional bool freePageReporting (field 17). - TemplateConfig + sandbox.Config gain a FreePageReporting bool that flows from template create → build phases (base/steps/finalize) → sandbox factory → fc.Process.Create. - fc.apiClient.enableFreePageReporting calls PUT /balloon with free_page_reporting=true after entropy setup and before VM start. - fcversion.HasFreePageReporting + auto-detect in TemplateCreate gate rollout to FC v1.14+. - LaunchDarkly free-page-reporting flag (default off). --- .../orchestrator/pkg/sandbox/fc/client.go | 24 ++ .../orchestrator/pkg/sandbox/fc/process.go | 11 + packages/orchestrator/pkg/sandbox/sandbox.go | 4 + .../pkg/template/build/config/config.go | 3 + .../pkg/template/build/phases/base/builder.go | 7 +- .../template/build/phases/finalize/builder.go | 7 +- .../template/build/phases/steps/builder.go | 7 +- .../pkg/template/server/create_template.go | 1 + packages/orchestrator/template-manager.proto | 2 + .../shared/pkg/fcversion/sandbox_features.go | 4 + packages/shared/pkg/featureflags/flags.go | 1 + .../template-manager/template-manager.pb.go | 265 +++++++++--------- 12 files changed, 201 insertions(+), 135 deletions(-) diff --git a/packages/orchestrator/pkg/sandbox/fc/client.go b/packages/orchestrator/pkg/sandbox/fc/client.go index f5c9eb30d0..d35d28040c 100644 --- a/packages/orchestrator/pkg/sandbox/fc/client.go +++ b/packages/orchestrator/pkg/sandbox/fc/client.go @@ -427,6 +427,30 @@ func (c *apiClient) startVM(ctx context.Context) error { return nil } +func (c *apiClient) enableFreePageReporting(ctx context.Context) error { + ctx, span := tracer.Start(ctx, "enable-free-page-reporting") + defer span.End() + + amountMib := int64(0) + deflateOnOom := false + + balloonConfig := operations.PutBalloonParams{ + Context: ctx, + Body: &models.Balloon{ + AmountMib: &amountMib, + DeflateOnOom: &deflateOnOom, + FreePageReporting: true, + }, + } + + _, err := c.client.Operations.PutBalloon(&balloonConfig) + if err != nil { + return fmt.Errorf("error setting up balloon device: %w", err) + } + + return nil +} + func (c *apiClient) memoryMapping(ctx context.Context) (*memory.Mapping, error) { params := operations.GetMemoryMappingsParams{ Context: ctx, diff --git a/packages/orchestrator/pkg/sandbox/fc/process.go b/packages/orchestrator/pkg/sandbox/fc/process.go index a29f56072b..977091304d 100644 --- a/packages/orchestrator/pkg/sandbox/fc/process.go +++ b/packages/orchestrator/pkg/sandbox/fc/process.go @@ -299,6 +299,7 @@ func (p *Process) Create( vCPUCount int64, memoryMB int64, hugePages bool, + freePageReporting bool, options ProcessOptions, txRateLimit RateLimiterConfig, driveRateLimit RateLimiterConfig, @@ -438,6 +439,16 @@ func (p *Process) Create( } telemetry.ReportEvent(ctx, "set fc entropy config") + if freePageReporting { + err = p.client.enableFreePageReporting(ctx) + if err != nil { + fcStopErr := p.Stop(ctx) + + return errors.Join(fmt.Errorf("error enabling free page reporting: %w", err), fcStopErr) + } + telemetry.ReportEvent(ctx, "enabled free page reporting") + } + err = p.client.startVM(ctx) if err != nil { fcStopErr := p.Stop(ctx) diff --git a/packages/orchestrator/pkg/sandbox/sandbox.go b/packages/orchestrator/pkg/sandbox/sandbox.go index f69100015d..87c83c9f2e 100644 --- a/packages/orchestrator/pkg/sandbox/sandbox.go +++ b/packages/orchestrator/pkg/sandbox/sandbox.go @@ -71,6 +71,9 @@ type Config struct { TotalDiskSizeMB int64 HugePages bool + // Enable free page reporting + FreePageReporting bool + Envd EnvdMetadata FirecrackerConfig fc.Config @@ -495,6 +498,7 @@ func (f *Factory) CreateSandbox( config.Vcpu, config.RamMB, config.HugePages, + config.FreePageReporting, processOptions, fc.RateLimiterConfig{ Ops: fc.TokenBucketConfig(throttleConfig.Ops), diff --git a/packages/orchestrator/pkg/template/build/config/config.go b/packages/orchestrator/pkg/template/build/config/config.go index 713de95ec0..f780b1ca3d 100644 --- a/packages/orchestrator/pkg/template/build/config/config.go +++ b/packages/orchestrator/pkg/template/build/config/config.go @@ -41,6 +41,9 @@ type TemplateConfig struct { // HugePages sets whether the VM use huge pages. HugePages bool + // FreePageReporting enables the corresponding feature in Firecracker + FreePageReporting bool + // Command to run to check if the template is ready. ReadyCmd string diff --git a/packages/orchestrator/pkg/template/build/phases/base/builder.go b/packages/orchestrator/pkg/template/build/phases/base/builder.go index b986d4de1c..7638d9b1cf 100644 --- a/packages/orchestrator/pkg/template/build/phases/base/builder.go +++ b/packages/orchestrator/pkg/template/build/phases/base/builder.go @@ -200,9 +200,10 @@ func (bb *BaseBuilder) buildLayerFromOCI( // Allow sandbox internet access during provisioning (nil network = no restrictions). baseSbxConfig := sandbox.NewConfig(sandbox.Config{ - Vcpu: bb.Config.VCpuCount, - RamMB: bb.Config.MemoryMB, - HugePages: bb.Config.HugePages, + Vcpu: bb.Config.VCpuCount, + RamMB: bb.Config.MemoryMB, + HugePages: bb.Config.HugePages, + FreePageReporting: bb.Config.FreePageReporting, Envd: sandbox.EnvdMetadata{ Version: bb.EnvdVersion, diff --git a/packages/orchestrator/pkg/template/build/phases/finalize/builder.go b/packages/orchestrator/pkg/template/build/phases/finalize/builder.go index 998aca413f..1cc0cb19cd 100644 --- a/packages/orchestrator/pkg/template/build/phases/finalize/builder.go +++ b/packages/orchestrator/pkg/template/build/phases/finalize/builder.go @@ -149,9 +149,10 @@ func (ppb *PostProcessingBuilder) Build( // Configure sandbox for final layer sbxConfig := sandbox.NewConfig(sandbox.Config{ - Vcpu: ppb.Config.VCpuCount, - RamMB: ppb.Config.MemoryMB, - HugePages: ppb.Config.HugePages, + Vcpu: ppb.Config.VCpuCount, + RamMB: ppb.Config.MemoryMB, + HugePages: ppb.Config.HugePages, + FreePageReporting: ppb.Config.FreePageReporting, Envd: sandbox.EnvdMetadata{ Version: ppb.EnvdVersion, diff --git a/packages/orchestrator/pkg/template/build/phases/steps/builder.go b/packages/orchestrator/pkg/template/build/phases/steps/builder.go index 0ff423e0b8..507312c5ac 100644 --- a/packages/orchestrator/pkg/template/build/phases/steps/builder.go +++ b/packages/orchestrator/pkg/template/build/phases/steps/builder.go @@ -163,9 +163,10 @@ func (sb *StepBuilder) Build( step := sb.step sbxConfig := sandbox.NewConfig(sandbox.Config{ - Vcpu: sb.Config.VCpuCount, - RamMB: sb.Config.MemoryMB, - HugePages: sb.Config.HugePages, + Vcpu: sb.Config.VCpuCount, + RamMB: sb.Config.MemoryMB, + HugePages: sb.Config.HugePages, + FreePageReporting: sb.Config.FreePageReporting, Envd: sandbox.EnvdMetadata{ Version: sb.EnvdVersion, diff --git a/packages/orchestrator/pkg/template/server/create_template.go b/packages/orchestrator/pkg/template/server/create_template.go index 0e179709a0..2aa4dd8588 100644 --- a/packages/orchestrator/pkg/template/server/create_template.go +++ b/packages/orchestrator/pkg/template/server/create_template.go @@ -88,6 +88,7 @@ func (s *ServerStore) TemplateCreate(ctx context.Context, templateRequest *templ ReadyCmd: cfg.GetReadyCommand(), DiskSizeMB: int64(cfg.GetDiskSizeMB()), HugePages: hugePages, + FreePageReporting: cfg.GetFreePageReporting(), FromImage: cfg.GetFromImage(), FromTemplate: cfg.GetFromTemplate(), RegistryAuthProvider: authProvider, diff --git a/packages/orchestrator/template-manager.proto b/packages/orchestrator/template-manager.proto index 1e70950637..3187950566 100644 --- a/packages/orchestrator/template-manager.proto +++ b/packages/orchestrator/template-manager.proto @@ -86,6 +86,8 @@ message TemplateConfig { optional FromImageRegistry fromImageRegistry = 15; string teamID = 16; + + optional bool freePageReporting = 17; } message TemplateCreateRequest { diff --git a/packages/shared/pkg/fcversion/sandbox_features.go b/packages/shared/pkg/fcversion/sandbox_features.go index f6fc9e19a5..b768f199d4 100644 --- a/packages/shared/pkg/fcversion/sandbox_features.go +++ b/packages/shared/pkg/fcversion/sandbox_features.go @@ -7,3 +7,7 @@ func (v *Info) HasHugePages() bool { return false } + +func (v *Info) HasFreePageReporting() bool { + return v.lastReleaseVersion.Major() > 1 || (v.lastReleaseVersion.Major() == 1 && v.lastReleaseVersion.Minor() >= 14) +} diff --git a/packages/shared/pkg/featureflags/flags.go b/packages/shared/pkg/featureflags/flags.go index 8ea8d491f3..615da187da 100644 --- a/packages/shared/pkg/featureflags/flags.go +++ b/packages/shared/pkg/featureflags/flags.go @@ -120,6 +120,7 @@ var ( ExecutionMetricsOnWebhooksFlag = NewBoolFlag("execution-metrics-on-webhooks", false) // TODO: Remove NLT 20250315 SandboxLabelBasedSchedulingFlag = NewBoolFlag("sandbox-label-based-scheduling", false) OptimisticResourceAccountingFlag = NewBoolFlag("sandbox-placement-optimistic-resource-accounting", false) + FreePageReportingFlag = NewBoolFlag("free-page-reporting", false) ) type IntFlag struct { diff --git a/packages/shared/pkg/grpc/template-manager/template-manager.pb.go b/packages/shared/pkg/grpc/template-manager/template-manager.pb.go index 131823fd24..8c6ceae914 100644 --- a/packages/shared/pkg/grpc/template-manager/template-manager.pb.go +++ b/packages/shared/pkg/grpc/template-manager/template-manager.pb.go @@ -706,6 +706,7 @@ type TemplateConfig struct { Source isTemplateConfig_Source `protobuf_oneof:"source"` FromImageRegistry *FromImageRegistry `protobuf:"bytes,15,opt,name=fromImageRegistry,proto3,oneof" json:"fromImageRegistry,omitempty"` TeamID string `protobuf:"bytes,16,opt,name=teamID,proto3" json:"teamID,omitempty"` + FreePageReporting *bool `protobuf:"varint,17,opt,name=freePageReporting,proto3,oneof" json:"freePageReporting,omitempty"` } func (x *TemplateConfig) Reset() { @@ -862,6 +863,13 @@ func (x *TemplateConfig) GetTeamID() string { return "" } +func (x *TemplateConfig) GetFreePageReporting() bool { + if x != nil && x.FreePageReporting != nil { + return *x.FreePageReporting + } + return false +} + type isTemplateConfig_Source interface { isTemplateConfig_Source() } @@ -1432,7 +1440,7 @@ var file_template_manager_proto_rawDesc = []byte{ 0x03, 0x67, 0x63, 0x70, 0x12, 0x2c, 0x0a, 0x07, 0x67, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x6c, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x10, 0x2e, 0x47, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x6c, 0x52, 0x65, 0x67, 0x69, 0x73, 0x74, 0x72, 0x79, 0x48, 0x00, 0x52, 0x07, 0x67, 0x65, 0x6e, 0x65, 0x72, - 0x61, 0x6c, 0x42, 0x06, 0x0a, 0x04, 0x74, 0x79, 0x70, 0x65, 0x22, 0x90, 0x05, 0x0a, 0x0e, 0x54, + 0x61, 0x6c, 0x42, 0x06, 0x0a, 0x04, 0x74, 0x79, 0x70, 0x65, 0x22, 0xd9, 0x05, 0x0a, 0x0e, 0x54, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x1e, 0x0a, 0x0a, 0x74, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x49, 0x44, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x74, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x49, 0x44, 0x12, 0x18, 0x0a, @@ -1471,134 +1479,139 @@ var file_template_manager_proto_rawDesc = []byte{ 0x74, 0x72, 0x79, 0x48, 0x02, 0x52, 0x11, 0x66, 0x72, 0x6f, 0x6d, 0x49, 0x6d, 0x61, 0x67, 0x65, 0x52, 0x65, 0x67, 0x69, 0x73, 0x74, 0x72, 0x79, 0x88, 0x01, 0x01, 0x12, 0x16, 0x0a, 0x06, 0x74, 0x65, 0x61, 0x6d, 0x49, 0x44, 0x18, 0x10, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x74, 0x65, 0x61, - 0x6d, 0x49, 0x44, 0x42, 0x08, 0x0a, 0x06, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x42, 0x08, 0x0a, - 0x06, 0x5f, 0x66, 0x6f, 0x72, 0x63, 0x65, 0x42, 0x14, 0x0a, 0x12, 0x5f, 0x66, 0x72, 0x6f, 0x6d, - 0x49, 0x6d, 0x61, 0x67, 0x65, 0x52, 0x65, 0x67, 0x69, 0x73, 0x74, 0x72, 0x79, 0x22, 0xa3, 0x01, - 0x0a, 0x15, 0x54, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, - 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x2b, 0x0a, 0x08, 0x74, 0x65, 0x6d, 0x70, 0x6c, - 0x61, 0x74, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0f, 0x2e, 0x54, 0x65, 0x6d, 0x70, - 0x6c, 0x61, 0x74, 0x65, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, 0x08, 0x74, 0x65, 0x6d, 0x70, - 0x6c, 0x61, 0x74, 0x65, 0x12, 0x23, 0x0a, 0x0a, 0x63, 0x61, 0x63, 0x68, 0x65, 0x53, 0x63, 0x6f, - 0x70, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x48, 0x00, 0x52, 0x0a, 0x63, 0x61, 0x63, 0x68, - 0x65, 0x53, 0x63, 0x6f, 0x70, 0x65, 0x88, 0x01, 0x01, 0x12, 0x1d, 0x0a, 0x07, 0x76, 0x65, 0x72, - 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x48, 0x01, 0x52, 0x07, 0x76, 0x65, - 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x88, 0x01, 0x01, 0x42, 0x0d, 0x0a, 0x0b, 0x5f, 0x63, 0x61, 0x63, - 0x68, 0x65, 0x53, 0x63, 0x6f, 0x70, 0x65, 0x42, 0x0a, 0x0a, 0x08, 0x5f, 0x76, 0x65, 0x72, 0x73, - 0x69, 0x6f, 0x6e, 0x22, 0x8b, 0x03, 0x0a, 0x15, 0x54, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, - 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1e, 0x0a, - 0x0a, 0x74, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x49, 0x44, 0x18, 0x01, 0x20, 0x01, 0x28, - 0x09, 0x52, 0x0a, 0x74, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x49, 0x44, 0x12, 0x18, 0x0a, - 0x07, 0x62, 0x75, 0x69, 0x6c, 0x64, 0x49, 0x44, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, - 0x62, 0x75, 0x69, 0x6c, 0x64, 0x49, 0x44, 0x12, 0x1b, 0x0a, 0x06, 0x6f, 0x66, 0x66, 0x73, 0x65, - 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, 0x05, 0x48, 0x00, 0x52, 0x06, 0x6f, 0x66, 0x66, 0x73, 0x65, - 0x74, 0x88, 0x01, 0x01, 0x12, 0x24, 0x0a, 0x05, 0x6c, 0x65, 0x76, 0x65, 0x6c, 0x18, 0x04, 0x20, - 0x01, 0x28, 0x0e, 0x32, 0x09, 0x2e, 0x4c, 0x6f, 0x67, 0x4c, 0x65, 0x76, 0x65, 0x6c, 0x48, 0x01, - 0x52, 0x05, 0x6c, 0x65, 0x76, 0x65, 0x6c, 0x88, 0x01, 0x01, 0x12, 0x19, 0x0a, 0x05, 0x6c, 0x69, - 0x6d, 0x69, 0x74, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0d, 0x48, 0x02, 0x52, 0x05, 0x6c, 0x69, 0x6d, - 0x69, 0x74, 0x88, 0x01, 0x01, 0x12, 0x35, 0x0a, 0x05, 0x73, 0x74, 0x61, 0x72, 0x74, 0x18, 0x06, - 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, - 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, - 0x48, 0x03, 0x52, 0x05, 0x73, 0x74, 0x61, 0x72, 0x74, 0x88, 0x01, 0x01, 0x12, 0x31, 0x0a, 0x03, - 0x65, 0x6e, 0x64, 0x18, 0x07, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, - 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, - 0x73, 0x74, 0x61, 0x6d, 0x70, 0x48, 0x04, 0x52, 0x03, 0x65, 0x6e, 0x64, 0x88, 0x01, 0x01, 0x12, - 0x31, 0x0a, 0x09, 0x64, 0x69, 0x72, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x08, 0x20, 0x01, - 0x28, 0x0e, 0x32, 0x0e, 0x2e, 0x4c, 0x6f, 0x67, 0x73, 0x44, 0x69, 0x72, 0x65, 0x63, 0x74, 0x69, - 0x6f, 0x6e, 0x48, 0x05, 0x52, 0x09, 0x64, 0x69, 0x72, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x88, - 0x01, 0x01, 0x42, 0x09, 0x0a, 0x07, 0x5f, 0x6f, 0x66, 0x66, 0x73, 0x65, 0x74, 0x42, 0x08, 0x0a, - 0x06, 0x5f, 0x6c, 0x65, 0x76, 0x65, 0x6c, 0x42, 0x08, 0x0a, 0x06, 0x5f, 0x6c, 0x69, 0x6d, 0x69, - 0x74, 0x42, 0x08, 0x0a, 0x06, 0x5f, 0x73, 0x74, 0x61, 0x72, 0x74, 0x42, 0x06, 0x0a, 0x04, 0x5f, - 0x65, 0x6e, 0x64, 0x42, 0x0c, 0x0a, 0x0a, 0x5f, 0x64, 0x69, 0x72, 0x65, 0x63, 0x74, 0x69, 0x6f, - 0x6e, 0x22, 0x56, 0x0a, 0x1a, 0x54, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x42, 0x75, 0x69, - 0x6c, 0x64, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, - 0x18, 0x0a, 0x07, 0x62, 0x75, 0x69, 0x6c, 0x64, 0x49, 0x44, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, - 0x52, 0x07, 0x62, 0x75, 0x69, 0x6c, 0x64, 0x49, 0x44, 0x12, 0x1e, 0x0a, 0x0a, 0x74, 0x65, 0x6d, - 0x70, 0x6c, 0x61, 0x74, 0x65, 0x49, 0x44, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x74, - 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x49, 0x44, 0x22, 0xbb, 0x01, 0x0a, 0x15, 0x54, 0x65, - 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x42, 0x75, 0x69, 0x6c, 0x64, 0x4d, 0x65, 0x74, 0x61, 0x64, - 0x61, 0x74, 0x61, 0x12, 0x24, 0x0a, 0x0d, 0x72, 0x6f, 0x6f, 0x74, 0x66, 0x73, 0x53, 0x69, 0x7a, - 0x65, 0x4b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x05, 0x52, 0x0d, 0x72, 0x6f, 0x6f, 0x74, - 0x66, 0x73, 0x53, 0x69, 0x7a, 0x65, 0x4b, 0x65, 0x79, 0x12, 0x26, 0x0a, 0x0e, 0x65, 0x6e, 0x76, - 0x64, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x4b, 0x65, 0x79, 0x18, 0x02, 0x20, 0x01, 0x28, - 0x09, 0x52, 0x0e, 0x65, 0x6e, 0x76, 0x64, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x4b, 0x65, - 0x79, 0x12, 0x24, 0x0a, 0x0d, 0x6b, 0x65, 0x72, 0x6e, 0x65, 0x6c, 0x56, 0x65, 0x72, 0x73, 0x69, - 0x6f, 0x6e, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0d, 0x6b, 0x65, 0x72, 0x6e, 0x65, 0x6c, - 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x2e, 0x0a, 0x12, 0x66, 0x69, 0x72, 0x65, 0x63, - 0x72, 0x61, 0x63, 0x6b, 0x65, 0x72, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x04, 0x20, - 0x01, 0x28, 0x09, 0x52, 0x12, 0x66, 0x69, 0x72, 0x65, 0x63, 0x72, 0x61, 0x63, 0x6b, 0x65, 0x72, - 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x22, 0x83, 0x02, 0x0a, 0x15, 0x54, 0x65, 0x6d, 0x70, - 0x6c, 0x61, 0x74, 0x65, 0x42, 0x75, 0x69, 0x6c, 0x64, 0x4c, 0x6f, 0x67, 0x45, 0x6e, 0x74, 0x72, - 0x79, 0x12, 0x38, 0x0a, 0x09, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x18, 0x01, - 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, - 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, - 0x52, 0x09, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x12, 0x18, 0x0a, 0x07, 0x6d, - 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x6d, 0x65, - 0x73, 0x73, 0x61, 0x67, 0x65, 0x12, 0x1f, 0x0a, 0x05, 0x6c, 0x65, 0x76, 0x65, 0x6c, 0x18, 0x03, - 0x20, 0x01, 0x28, 0x0e, 0x32, 0x09, 0x2e, 0x4c, 0x6f, 0x67, 0x4c, 0x65, 0x76, 0x65, 0x6c, 0x52, - 0x05, 0x6c, 0x65, 0x76, 0x65, 0x6c, 0x12, 0x3a, 0x0a, 0x06, 0x66, 0x69, 0x65, 0x6c, 0x64, 0x73, - 0x18, 0x04, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x22, 0x2e, 0x54, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, - 0x65, 0x42, 0x75, 0x69, 0x6c, 0x64, 0x4c, 0x6f, 0x67, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x2e, 0x46, - 0x69, 0x65, 0x6c, 0x64, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x06, 0x66, 0x69, 0x65, 0x6c, - 0x64, 0x73, 0x1a, 0x39, 0x0a, 0x0b, 0x46, 0x69, 0x65, 0x6c, 0x64, 0x73, 0x45, 0x6e, 0x74, 0x72, - 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, - 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, - 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0x57, 0x0a, - 0x19, 0x54, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x42, 0x75, 0x69, 0x6c, 0x64, 0x53, 0x74, - 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x61, 0x73, 0x6f, 0x6e, 0x12, 0x18, 0x0a, 0x07, 0x6d, 0x65, - 0x73, 0x73, 0x61, 0x67, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x6d, 0x65, 0x73, - 0x73, 0x61, 0x67, 0x65, 0x12, 0x17, 0x0a, 0x04, 0x73, 0x74, 0x65, 0x70, 0x18, 0x02, 0x20, 0x01, - 0x28, 0x09, 0x48, 0x00, 0x52, 0x04, 0x73, 0x74, 0x65, 0x70, 0x88, 0x01, 0x01, 0x42, 0x07, 0x0a, - 0x05, 0x5f, 0x73, 0x74, 0x65, 0x70, 0x22, 0x86, 0x02, 0x0a, 0x1b, 0x54, 0x65, 0x6d, 0x70, 0x6c, + 0x6d, 0x49, 0x44, 0x12, 0x31, 0x0a, 0x11, 0x66, 0x72, 0x65, 0x65, 0x50, 0x61, 0x67, 0x65, 0x52, + 0x65, 0x70, 0x6f, 0x72, 0x74, 0x69, 0x6e, 0x67, 0x18, 0x11, 0x20, 0x01, 0x28, 0x08, 0x48, 0x03, + 0x52, 0x11, 0x66, 0x72, 0x65, 0x65, 0x50, 0x61, 0x67, 0x65, 0x52, 0x65, 0x70, 0x6f, 0x72, 0x74, + 0x69, 0x6e, 0x67, 0x88, 0x01, 0x01, 0x42, 0x08, 0x0a, 0x06, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, + 0x42, 0x08, 0x0a, 0x06, 0x5f, 0x66, 0x6f, 0x72, 0x63, 0x65, 0x42, 0x14, 0x0a, 0x12, 0x5f, 0x66, + 0x72, 0x6f, 0x6d, 0x49, 0x6d, 0x61, 0x67, 0x65, 0x52, 0x65, 0x67, 0x69, 0x73, 0x74, 0x72, 0x79, + 0x42, 0x14, 0x0a, 0x12, 0x5f, 0x66, 0x72, 0x65, 0x65, 0x50, 0x61, 0x67, 0x65, 0x52, 0x65, 0x70, + 0x6f, 0x72, 0x74, 0x69, 0x6e, 0x67, 0x22, 0xa3, 0x01, 0x0a, 0x15, 0x54, 0x65, 0x6d, 0x70, 0x6c, + 0x61, 0x74, 0x65, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, + 0x12, 0x2b, 0x0a, 0x08, 0x74, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x18, 0x01, 0x20, 0x01, + 0x28, 0x0b, 0x32, 0x0f, 0x2e, 0x54, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x43, 0x6f, 0x6e, + 0x66, 0x69, 0x67, 0x52, 0x08, 0x74, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x12, 0x23, 0x0a, + 0x0a, 0x63, 0x61, 0x63, 0x68, 0x65, 0x53, 0x63, 0x6f, 0x70, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, + 0x09, 0x48, 0x00, 0x52, 0x0a, 0x63, 0x61, 0x63, 0x68, 0x65, 0x53, 0x63, 0x6f, 0x70, 0x65, 0x88, + 0x01, 0x01, 0x12, 0x1d, 0x0a, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x03, 0x20, + 0x01, 0x28, 0x09, 0x48, 0x01, 0x52, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x88, 0x01, + 0x01, 0x42, 0x0d, 0x0a, 0x0b, 0x5f, 0x63, 0x61, 0x63, 0x68, 0x65, 0x53, 0x63, 0x6f, 0x70, 0x65, + 0x42, 0x0a, 0x0a, 0x08, 0x5f, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x22, 0x8b, 0x03, 0x0a, + 0x15, 0x54, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, + 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1e, 0x0a, 0x0a, 0x74, 0x65, 0x6d, 0x70, 0x6c, 0x61, + 0x74, 0x65, 0x49, 0x44, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x74, 0x65, 0x6d, 0x70, + 0x6c, 0x61, 0x74, 0x65, 0x49, 0x44, 0x12, 0x18, 0x0a, 0x07, 0x62, 0x75, 0x69, 0x6c, 0x64, 0x49, + 0x44, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x62, 0x75, 0x69, 0x6c, 0x64, 0x49, 0x44, + 0x12, 0x1b, 0x0a, 0x06, 0x6f, 0x66, 0x66, 0x73, 0x65, 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, 0x05, + 0x48, 0x00, 0x52, 0x06, 0x6f, 0x66, 0x66, 0x73, 0x65, 0x74, 0x88, 0x01, 0x01, 0x12, 0x24, 0x0a, + 0x05, 0x6c, 0x65, 0x76, 0x65, 0x6c, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x09, 0x2e, 0x4c, + 0x6f, 0x67, 0x4c, 0x65, 0x76, 0x65, 0x6c, 0x48, 0x01, 0x52, 0x05, 0x6c, 0x65, 0x76, 0x65, 0x6c, + 0x88, 0x01, 0x01, 0x12, 0x19, 0x0a, 0x05, 0x6c, 0x69, 0x6d, 0x69, 0x74, 0x18, 0x05, 0x20, 0x01, + 0x28, 0x0d, 0x48, 0x02, 0x52, 0x05, 0x6c, 0x69, 0x6d, 0x69, 0x74, 0x88, 0x01, 0x01, 0x12, 0x35, + 0x0a, 0x05, 0x73, 0x74, 0x61, 0x72, 0x74, 0x18, 0x06, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, + 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, + 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x48, 0x03, 0x52, 0x05, 0x73, 0x74, 0x61, + 0x72, 0x74, 0x88, 0x01, 0x01, 0x12, 0x31, 0x0a, 0x03, 0x65, 0x6e, 0x64, 0x18, 0x07, 0x20, 0x01, + 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, + 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x48, 0x04, + 0x52, 0x03, 0x65, 0x6e, 0x64, 0x88, 0x01, 0x01, 0x12, 0x31, 0x0a, 0x09, 0x64, 0x69, 0x72, 0x65, + 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x08, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x0e, 0x2e, 0x4c, 0x6f, + 0x67, 0x73, 0x44, 0x69, 0x72, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x48, 0x05, 0x52, 0x09, 0x64, + 0x69, 0x72, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x88, 0x01, 0x01, 0x42, 0x09, 0x0a, 0x07, 0x5f, + 0x6f, 0x66, 0x66, 0x73, 0x65, 0x74, 0x42, 0x08, 0x0a, 0x06, 0x5f, 0x6c, 0x65, 0x76, 0x65, 0x6c, + 0x42, 0x08, 0x0a, 0x06, 0x5f, 0x6c, 0x69, 0x6d, 0x69, 0x74, 0x42, 0x08, 0x0a, 0x06, 0x5f, 0x73, + 0x74, 0x61, 0x72, 0x74, 0x42, 0x06, 0x0a, 0x04, 0x5f, 0x65, 0x6e, 0x64, 0x42, 0x0c, 0x0a, 0x0a, + 0x5f, 0x64, 0x69, 0x72, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x22, 0x56, 0x0a, 0x1a, 0x54, 0x65, + 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x42, 0x75, 0x69, 0x6c, 0x64, 0x44, 0x65, 0x6c, 0x65, 0x74, + 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x18, 0x0a, 0x07, 0x62, 0x75, 0x69, 0x6c, + 0x64, 0x49, 0x44, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x62, 0x75, 0x69, 0x6c, 0x64, + 0x49, 0x44, 0x12, 0x1e, 0x0a, 0x0a, 0x74, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x49, 0x44, + 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x74, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, + 0x49, 0x44, 0x22, 0xbb, 0x01, 0x0a, 0x15, 0x54, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x42, + 0x75, 0x69, 0x6c, 0x64, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x12, 0x24, 0x0a, 0x0d, + 0x72, 0x6f, 0x6f, 0x74, 0x66, 0x73, 0x53, 0x69, 0x7a, 0x65, 0x4b, 0x65, 0x79, 0x18, 0x01, 0x20, + 0x01, 0x28, 0x05, 0x52, 0x0d, 0x72, 0x6f, 0x6f, 0x74, 0x66, 0x73, 0x53, 0x69, 0x7a, 0x65, 0x4b, + 0x65, 0x79, 0x12, 0x26, 0x0a, 0x0e, 0x65, 0x6e, 0x76, 0x64, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, + 0x6e, 0x4b, 0x65, 0x79, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0e, 0x65, 0x6e, 0x76, 0x64, + 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x4b, 0x65, 0x79, 0x12, 0x24, 0x0a, 0x0d, 0x6b, 0x65, + 0x72, 0x6e, 0x65, 0x6c, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x03, 0x20, 0x01, 0x28, + 0x09, 0x52, 0x0d, 0x6b, 0x65, 0x72, 0x6e, 0x65, 0x6c, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, + 0x12, 0x2e, 0x0a, 0x12, 0x66, 0x69, 0x72, 0x65, 0x63, 0x72, 0x61, 0x63, 0x6b, 0x65, 0x72, 0x56, + 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x12, 0x66, 0x69, + 0x72, 0x65, 0x63, 0x72, 0x61, 0x63, 0x6b, 0x65, 0x72, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, + 0x22, 0x83, 0x02, 0x0a, 0x15, 0x54, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x42, 0x75, 0x69, + 0x6c, 0x64, 0x4c, 0x6f, 0x67, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x38, 0x0a, 0x09, 0x74, 0x69, + 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, + 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, + 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x09, 0x74, 0x69, 0x6d, 0x65, 0x73, + 0x74, 0x61, 0x6d, 0x70, 0x12, 0x18, 0x0a, 0x07, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x18, + 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x12, 0x1f, + 0x0a, 0x05, 0x6c, 0x65, 0x76, 0x65, 0x6c, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x09, 0x2e, + 0x4c, 0x6f, 0x67, 0x4c, 0x65, 0x76, 0x65, 0x6c, 0x52, 0x05, 0x6c, 0x65, 0x76, 0x65, 0x6c, 0x12, + 0x3a, 0x0a, 0x06, 0x66, 0x69, 0x65, 0x6c, 0x64, 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, 0x0b, 0x32, + 0x22, 0x2e, 0x54, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x42, 0x75, 0x69, 0x6c, 0x64, 0x4c, + 0x6f, 0x67, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x2e, 0x46, 0x69, 0x65, 0x6c, 0x64, 0x73, 0x45, 0x6e, + 0x74, 0x72, 0x79, 0x52, 0x06, 0x66, 0x69, 0x65, 0x6c, 0x64, 0x73, 0x1a, 0x39, 0x0a, 0x0b, 0x46, + 0x69, 0x65, 0x6c, 0x64, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, + 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, + 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, + 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0x57, 0x0a, 0x19, 0x54, 0x65, 0x6d, 0x70, 0x6c, 0x61, + 0x74, 0x65, 0x42, 0x75, 0x69, 0x6c, 0x64, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x61, + 0x73, 0x6f, 0x6e, 0x12, 0x18, 0x0a, 0x07, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x18, 0x01, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x12, 0x17, 0x0a, + 0x04, 0x73, 0x74, 0x65, 0x70, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x48, 0x00, 0x52, 0x04, 0x73, + 0x74, 0x65, 0x70, 0x88, 0x01, 0x01, 0x42, 0x07, 0x0a, 0x05, 0x5f, 0x73, 0x74, 0x65, 0x70, 0x22, + 0x86, 0x02, 0x0a, 0x1b, 0x54, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x42, 0x75, 0x69, 0x6c, + 0x64, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, + 0x2b, 0x0a, 0x06, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0e, 0x32, + 0x13, 0x2e, 0x54, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x42, 0x75, 0x69, 0x6c, 0x64, 0x53, + 0x74, 0x61, 0x74, 0x65, 0x52, 0x06, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x32, 0x0a, 0x08, + 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x16, + 0x2e, 0x54, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x42, 0x75, 0x69, 0x6c, 0x64, 0x4d, 0x65, + 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x52, 0x08, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, + 0x12, 0x36, 0x0a, 0x0a, 0x6c, 0x6f, 0x67, 0x45, 0x6e, 0x74, 0x72, 0x69, 0x65, 0x73, 0x18, 0x05, + 0x20, 0x03, 0x28, 0x0b, 0x32, 0x16, 0x2e, 0x54, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x42, + 0x75, 0x69, 0x6c, 0x64, 0x4c, 0x6f, 0x67, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x0a, 0x6c, 0x6f, + 0x67, 0x45, 0x6e, 0x74, 0x72, 0x69, 0x65, 0x73, 0x12, 0x37, 0x0a, 0x06, 0x72, 0x65, 0x61, 0x73, + 0x6f, 0x6e, 0x18, 0x06, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x54, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x42, 0x75, 0x69, 0x6c, 0x64, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, - 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x2b, 0x0a, 0x06, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, - 0x18, 0x01, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x13, 0x2e, 0x54, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, - 0x65, 0x42, 0x75, 0x69, 0x6c, 0x64, 0x53, 0x74, 0x61, 0x74, 0x65, 0x52, 0x06, 0x73, 0x74, 0x61, - 0x74, 0x75, 0x73, 0x12, 0x32, 0x0a, 0x08, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x18, - 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x16, 0x2e, 0x54, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, - 0x42, 0x75, 0x69, 0x6c, 0x64, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x52, 0x08, 0x6d, - 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x12, 0x36, 0x0a, 0x0a, 0x6c, 0x6f, 0x67, 0x45, 0x6e, - 0x74, 0x72, 0x69, 0x65, 0x73, 0x18, 0x05, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x16, 0x2e, 0x54, 0x65, - 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x42, 0x75, 0x69, 0x6c, 0x64, 0x4c, 0x6f, 0x67, 0x45, 0x6e, - 0x74, 0x72, 0x79, 0x52, 0x0a, 0x6c, 0x6f, 0x67, 0x45, 0x6e, 0x74, 0x72, 0x69, 0x65, 0x73, 0x12, - 0x37, 0x0a, 0x06, 0x72, 0x65, 0x61, 0x73, 0x6f, 0x6e, 0x18, 0x06, 0x20, 0x01, 0x28, 0x0b, 0x32, - 0x1a, 0x2e, 0x54, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x42, 0x75, 0x69, 0x6c, 0x64, 0x53, - 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x61, 0x73, 0x6f, 0x6e, 0x48, 0x00, 0x52, 0x06, 0x72, - 0x65, 0x61, 0x73, 0x6f, 0x6e, 0x88, 0x01, 0x01, 0x42, 0x09, 0x0a, 0x07, 0x5f, 0x72, 0x65, 0x61, - 0x73, 0x6f, 0x6e, 0x4a, 0x04, 0x08, 0x03, 0x10, 0x04, 0x4a, 0x04, 0x08, 0x04, 0x10, 0x05, 0x2a, - 0x34, 0x0a, 0x08, 0x4c, 0x6f, 0x67, 0x4c, 0x65, 0x76, 0x65, 0x6c, 0x12, 0x09, 0x0a, 0x05, 0x44, - 0x65, 0x62, 0x75, 0x67, 0x10, 0x00, 0x12, 0x08, 0x0a, 0x04, 0x49, 0x6e, 0x66, 0x6f, 0x10, 0x01, - 0x12, 0x08, 0x0a, 0x04, 0x57, 0x61, 0x72, 0x6e, 0x10, 0x02, 0x12, 0x09, 0x0a, 0x05, 0x45, 0x72, - 0x72, 0x6f, 0x72, 0x10, 0x03, 0x2a, 0x2a, 0x0a, 0x0d, 0x4c, 0x6f, 0x67, 0x73, 0x44, 0x69, 0x72, - 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x0b, 0x0a, 0x07, 0x46, 0x6f, 0x72, 0x77, 0x61, 0x72, - 0x64, 0x10, 0x00, 0x12, 0x0c, 0x0a, 0x08, 0x42, 0x61, 0x63, 0x6b, 0x77, 0x61, 0x72, 0x64, 0x10, - 0x01, 0x2a, 0x3d, 0x0a, 0x12, 0x54, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x42, 0x75, 0x69, - 0x6c, 0x64, 0x53, 0x74, 0x61, 0x74, 0x65, 0x12, 0x0c, 0x0a, 0x08, 0x42, 0x75, 0x69, 0x6c, 0x64, - 0x69, 0x6e, 0x67, 0x10, 0x00, 0x12, 0x0a, 0x0a, 0x06, 0x46, 0x61, 0x69, 0x6c, 0x65, 0x64, 0x10, - 0x01, 0x12, 0x0d, 0x0a, 0x09, 0x43, 0x6f, 0x6d, 0x70, 0x6c, 0x65, 0x74, 0x65, 0x64, 0x10, 0x02, - 0x32, 0xbe, 0x02, 0x0a, 0x0f, 0x54, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x53, 0x65, 0x72, - 0x76, 0x69, 0x63, 0x65, 0x12, 0x40, 0x0a, 0x0e, 0x54, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, - 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x12, 0x16, 0x2e, 0x54, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, - 0x65, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x16, - 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, - 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x12, 0x4b, 0x0a, 0x13, 0x54, 0x65, 0x6d, 0x70, 0x6c, 0x61, - 0x74, 0x65, 0x42, 0x75, 0x69, 0x6c, 0x64, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x16, 0x2e, - 0x54, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, - 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1c, 0x2e, 0x54, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, - 0x42, 0x75, 0x69, 0x6c, 0x64, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, - 0x6e, 0x73, 0x65, 0x12, 0x4a, 0x0a, 0x13, 0x54, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x42, - 0x75, 0x69, 0x6c, 0x64, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x12, 0x1b, 0x2e, 0x54, 0x65, 0x6d, - 0x70, 0x6c, 0x61, 0x74, 0x65, 0x42, 0x75, 0x69, 0x6c, 0x64, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, + 0x61, 0x73, 0x6f, 0x6e, 0x48, 0x00, 0x52, 0x06, 0x72, 0x65, 0x61, 0x73, 0x6f, 0x6e, 0x88, 0x01, + 0x01, 0x42, 0x09, 0x0a, 0x07, 0x5f, 0x72, 0x65, 0x61, 0x73, 0x6f, 0x6e, 0x4a, 0x04, 0x08, 0x03, + 0x10, 0x04, 0x4a, 0x04, 0x08, 0x04, 0x10, 0x05, 0x2a, 0x34, 0x0a, 0x08, 0x4c, 0x6f, 0x67, 0x4c, + 0x65, 0x76, 0x65, 0x6c, 0x12, 0x09, 0x0a, 0x05, 0x44, 0x65, 0x62, 0x75, 0x67, 0x10, 0x00, 0x12, + 0x08, 0x0a, 0x04, 0x49, 0x6e, 0x66, 0x6f, 0x10, 0x01, 0x12, 0x08, 0x0a, 0x04, 0x57, 0x61, 0x72, + 0x6e, 0x10, 0x02, 0x12, 0x09, 0x0a, 0x05, 0x45, 0x72, 0x72, 0x6f, 0x72, 0x10, 0x03, 0x2a, 0x2a, + 0x0a, 0x0d, 0x4c, 0x6f, 0x67, 0x73, 0x44, 0x69, 0x72, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x12, + 0x0b, 0x0a, 0x07, 0x46, 0x6f, 0x72, 0x77, 0x61, 0x72, 0x64, 0x10, 0x00, 0x12, 0x0c, 0x0a, 0x08, + 0x42, 0x61, 0x63, 0x6b, 0x77, 0x61, 0x72, 0x64, 0x10, 0x01, 0x2a, 0x3d, 0x0a, 0x12, 0x54, 0x65, + 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x42, 0x75, 0x69, 0x6c, 0x64, 0x53, 0x74, 0x61, 0x74, 0x65, + 0x12, 0x0c, 0x0a, 0x08, 0x42, 0x75, 0x69, 0x6c, 0x64, 0x69, 0x6e, 0x67, 0x10, 0x00, 0x12, 0x0a, + 0x0a, 0x06, 0x46, 0x61, 0x69, 0x6c, 0x65, 0x64, 0x10, 0x01, 0x12, 0x0d, 0x0a, 0x09, 0x43, 0x6f, + 0x6d, 0x70, 0x6c, 0x65, 0x74, 0x65, 0x64, 0x10, 0x02, 0x32, 0xbe, 0x02, 0x0a, 0x0f, 0x54, 0x65, + 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0x40, 0x0a, + 0x0e, 0x54, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x12, + 0x16, 0x2e, 0x54, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x12, - 0x50, 0x0a, 0x13, 0x49, 0x6e, 0x69, 0x74, 0x4c, 0x61, 0x79, 0x65, 0x72, 0x46, 0x69, 0x6c, 0x65, - 0x55, 0x70, 0x6c, 0x6f, 0x61, 0x64, 0x12, 0x1b, 0x2e, 0x49, 0x6e, 0x69, 0x74, 0x4c, 0x61, 0x79, - 0x65, 0x72, 0x46, 0x69, 0x6c, 0x65, 0x55, 0x70, 0x6c, 0x6f, 0x61, 0x64, 0x52, 0x65, 0x71, 0x75, - 0x65, 0x73, 0x74, 0x1a, 0x1c, 0x2e, 0x49, 0x6e, 0x69, 0x74, 0x4c, 0x61, 0x79, 0x65, 0x72, 0x46, - 0x69, 0x6c, 0x65, 0x55, 0x70, 0x6c, 0x6f, 0x61, 0x64, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, - 0x65, 0x42, 0x33, 0x5a, 0x31, 0x68, 0x74, 0x74, 0x70, 0x73, 0x3a, 0x2f, 0x2f, 0x67, 0x69, 0x74, - 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x65, 0x32, 0x62, 0x2d, 0x64, 0x65, 0x76, 0x2f, - 0x69, 0x6e, 0x66, 0x72, 0x61, 0x2f, 0x74, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x2d, 0x6d, - 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x4b, 0x0a, 0x13, 0x54, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x42, 0x75, 0x69, 0x6c, 0x64, + 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x16, 0x2e, 0x54, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, + 0x65, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1c, + 0x2e, 0x54, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x42, 0x75, 0x69, 0x6c, 0x64, 0x53, 0x74, + 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x4a, 0x0a, 0x13, + 0x54, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x42, 0x75, 0x69, 0x6c, 0x64, 0x44, 0x65, 0x6c, + 0x65, 0x74, 0x65, 0x12, 0x1b, 0x2e, 0x54, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x42, 0x75, + 0x69, 0x6c, 0x64, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, + 0x1a, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, + 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x12, 0x50, 0x0a, 0x13, 0x49, 0x6e, 0x69, 0x74, + 0x4c, 0x61, 0x79, 0x65, 0x72, 0x46, 0x69, 0x6c, 0x65, 0x55, 0x70, 0x6c, 0x6f, 0x61, 0x64, 0x12, + 0x1b, 0x2e, 0x49, 0x6e, 0x69, 0x74, 0x4c, 0x61, 0x79, 0x65, 0x72, 0x46, 0x69, 0x6c, 0x65, 0x55, + 0x70, 0x6c, 0x6f, 0x61, 0x64, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1c, 0x2e, 0x49, + 0x6e, 0x69, 0x74, 0x4c, 0x61, 0x79, 0x65, 0x72, 0x46, 0x69, 0x6c, 0x65, 0x55, 0x70, 0x6c, 0x6f, + 0x61, 0x64, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x42, 0x33, 0x5a, 0x31, 0x68, 0x74, + 0x74, 0x70, 0x73, 0x3a, 0x2f, 0x2f, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, + 0x2f, 0x65, 0x32, 0x62, 0x2d, 0x64, 0x65, 0x76, 0x2f, 0x69, 0x6e, 0x66, 0x72, 0x61, 0x2f, 0x74, + 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x2d, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x62, + 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( From 89073662a7eebfb9b858ec55e5303a3b83246ba4 Mon Sep 17 00:00:00 2001 From: ValentaTomas Date: Sat, 2 May 2026 18:41:40 -0700 Subject: [PATCH 02/10] fix(template-build): use fcInfo to auto-detect FreePageReporting in orchestrator HasFreePageReporting() was added to fcversion.Info but had zero callers in the production path. Mirror the HasHugePages() pattern: let the orchestrator derive the value from the FC version (authoritative), gated by the FreePageReportingFlag LaunchDarkly flag (default false). Also emit an env.free_page_reporting span attribute alongside the existing env.huge_pages one. --- packages/orchestrator/pkg/template/server/create_template.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/packages/orchestrator/pkg/template/server/create_template.go b/packages/orchestrator/pkg/template/server/create_template.go index 2aa4dd8588..a049548001 100644 --- a/packages/orchestrator/pkg/template/server/create_template.go +++ b/packages/orchestrator/pkg/template/server/create_template.go @@ -65,6 +65,7 @@ func (s *ServerStore) TemplateCreate(ctx context.Context, templateRequest *templ return nil, fmt.Errorf("invalid resolved firecracker version %q: %w", firecrackerVersion, err) } hugePages := fcInfo.HasHugePages() + freePageReporting := fcInfo.HasFreePageReporting() && s.featureFlags.BoolFlag(ctx, featureflags.FreePageReportingFlag) childSpan.SetAttributes( telemetry.WithTemplateID(cfg.GetTemplateID()), @@ -75,6 +76,7 @@ func (s *ServerStore) TemplateCreate(ctx context.Context, templateRequest *templ attribute.Int64("env.memory_mb", int64(cfg.GetMemoryMB())), attribute.Int64("env.vcpu_count", int64(cfg.GetVCpuCount())), attribute.Bool("env.huge_pages", hugePages), + attribute.Bool("env.free_page_reporting", freePageReporting), ) template := config.TemplateConfig{ @@ -88,7 +90,7 @@ func (s *ServerStore) TemplateCreate(ctx context.Context, templateRequest *templ ReadyCmd: cfg.GetReadyCommand(), DiskSizeMB: int64(cfg.GetDiskSizeMB()), HugePages: hugePages, - FreePageReporting: cfg.GetFreePageReporting(), + FreePageReporting: freePageReporting, FromImage: cfg.GetFromImage(), FromTemplate: cfg.GetFromTemplate(), RegistryAuthProvider: authProvider, From a194833fa3e8cc0f579b8e09cf2d187c52bf1272 Mon Sep 17 00:00:00 2001 From: ValentaTomas Date: Sat, 2 May 2026 19:56:58 -0700 Subject: [PATCH 03/10] chore(uffd): trim comments for review --- packages/orchestrator/pkg/sandbox/sandbox.go | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/packages/orchestrator/pkg/sandbox/sandbox.go b/packages/orchestrator/pkg/sandbox/sandbox.go index 87c83c9f2e..23ca0bda82 100644 --- a/packages/orchestrator/pkg/sandbox/sandbox.go +++ b/packages/orchestrator/pkg/sandbox/sandbox.go @@ -68,10 +68,8 @@ type Config struct { RamMB int64 // TotalDiskSizeMB optional, now used only for metrics. - TotalDiskSizeMB int64 - HugePages bool - - // Enable free page reporting + TotalDiskSizeMB int64 + HugePages bool FreePageReporting bool Envd EnvdMetadata From 8652309ec5270d1dac4f598e33f2acb06b64008b Mon Sep 17 00:00:00 2001 From: ValentaTomas Date: Sat, 2 May 2026 21:52:34 -0700 Subject: [PATCH 04/10] chore(template-build): trim narrating comments --- packages/orchestrator/pkg/template/build/config/config.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/orchestrator/pkg/template/build/config/config.go b/packages/orchestrator/pkg/template/build/config/config.go index f780b1ca3d..4c28adff2e 100644 --- a/packages/orchestrator/pkg/template/build/config/config.go +++ b/packages/orchestrator/pkg/template/build/config/config.go @@ -41,7 +41,8 @@ type TemplateConfig struct { // HugePages sets whether the VM use huge pages. HugePages bool - // FreePageReporting enables the corresponding feature in Firecracker + // FreePageReporting enables Firecracker's balloon free-page-reporting, + // which madvise(MADV_DONTNEED)s freed guest pages back to the host. FreePageReporting bool // Command to run to check if the template is ready. From 7f2270986d016e285adf78f735956a54873d16a6 Mon Sep 17 00:00:00 2001 From: ValentaTomas Date: Sat, 2 May 2026 23:28:24 -0700 Subject: [PATCH 05/10] fix(uffd): propagate Removed pages into DiffMetadata.Empty MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Read the Removed bitmap from PageTracker and emit it as DiffMetadata.Empty so REMOVE'd pages become uuid.Nil mappings in the snapshot header (read as zero on resume). Defensively AndNot the empty set out of dirty: settle drains make these disjoint in practice (Removed pages have no PTE, WP-async only sees present pages with WP cleared), but if the invariant ever breaks the guest's last intent for a Removed page is "free, read zero on restore" — so empty must win, not stale dirty content. --- .../orchestrator/pkg/sandbox/uffd/uffd.go | 28 ++++++++++++++++++- .../sandbox/uffd/userfaultfd/userfaultfd.go | 19 +++++++++++++ 2 files changed, 46 insertions(+), 1 deletion(-) diff --git a/packages/orchestrator/pkg/sandbox/uffd/uffd.go b/packages/orchestrator/pkg/sandbox/uffd/uffd.go index 5fe3e5370b..e5570eb919 100644 --- a/packages/orchestrator/pkg/sandbox/uffd/uffd.go +++ b/packages/orchestrator/pkg/sandbox/uffd/uffd.go @@ -217,7 +217,33 @@ func (u *Uffd) Exit() *utils.ErrorOnce { // // It *MUST* be only called after the sandbox was successfully paused via API and after the snapshot endpoint was called. func (u *Uffd) DiffMetadata(ctx context.Context, f *fc.Process) (*header.DiffMetadata, error) { - return f.DirtyMemory(ctx, u.memfile.BlockSize()) + handler, err := u.handler.WaitWithContext(ctx) + if err != nil { + return nil, fmt.Errorf("failed to get uffd: %w", err) + } + + diff, err := f.DirtyMemory(ctx, u.memfile.BlockSize()) + if err != nil { + return nil, fmt.Errorf("failed to get dirty memory: %w", err) + } + + // Without this, REMOVE'd pages stay attributed to the previous layer's BuildId in + // the merged mapping and leak their stale contents on resume; with it, those ranges + // become uuid.Nil sentinels (read as zero, matching guest free-page intent). + _, empty := handler.ExportPageStates() + + // dirty ∩ empty should be empty after settle (Removed pages have no PTE, + // FC's WP-async dirty scan only sees present pages with WP cleared), but + // AndNot here means "empty wins" if the invariant ever breaks: the guest's + // most recent intent for a Removed page is "free, read zero on restore", + // so we must not let stale dirty content shadow it in this diff layer. + diff.Dirty.AndNot(empty) + + return &header.DiffMetadata{ + BlockSize: diff.BlockSize, + Dirty: diff.Dirty, + Empty: empty, + }, nil } // PrefetchData returns page fault data for prefetch mapping. diff --git a/packages/orchestrator/pkg/sandbox/uffd/userfaultfd/userfaultfd.go b/packages/orchestrator/pkg/sandbox/uffd/userfaultfd/userfaultfd.go index 2dcb7534a1..48ab3d8a1b 100644 --- a/packages/orchestrator/pkg/sandbox/uffd/userfaultfd/userfaultfd.go +++ b/packages/orchestrator/pkg/sandbox/uffd/userfaultfd/userfaultfd.go @@ -11,6 +11,7 @@ import ( "time" "unsafe" + "github.com/RoaringBitmap/roaring/v2" "go.opentelemetry.io/otel" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/trace" @@ -136,6 +137,24 @@ func NewUserfaultfdFromFd(fd uintptr, src block.Slicer, m *memory.Mapping, logge return u, nil } +// ExportPageStates returns snapshots of the faulted and removed +// page-index bitmaps. The diff-snapshot writer uses the removed +// bitmap to mark freed pages as DiffMetadata.Empty so they resolve +// to uuid.Nil (zero-on-restore) in the snapshot header. +func (u *Userfaultfd) ExportPageStates() (faulted, removed *roaring.Bitmap) { + // Take readSerial first to wait for the serve loop's current + // iteration (read+apply) to complete, then settleRequests to wait + // for in-flight workers. Same lock order as serve loop avoids any + // AB-BA inversion. + u.readSerial.Lock() + defer u.readSerial.Unlock() + + u.settleRequests.Lock() + defer u.settleRequests.Unlock() + + return u.pageTracker.Export() +} + func (u *Userfaultfd) readEvents(ctx context.Context) ([]*UffdRemove, []*UffdPagefault, error) { buf := make([]byte, unsafe.Sizeof(UffdMsg{})) From 9f57128ab5e880ccc56e2ec9a3c3f564add2979b Mon Sep 17 00:00:00 2001 From: Tomas Valenta Date: Tue, 5 May 2026 01:45:42 -0700 Subject: [PATCH 06/10] fix(template): drop unused freePageReporting proto field, trim verbose comments Receiver derives freePageReporting from FC version + LD flag and never reads cfg.GetFreePageReporting(), so the proto field was dead. Removing it also kills the noisy generated pb.go diff. --- .../orchestrator/pkg/sandbox/uffd/uffd.go | 11 +- .../sandbox/uffd/userfaultfd/userfaultfd.go | 11 +- .../pkg/template/build/config/config.go | 3 +- packages/orchestrator/template-manager.proto | 2 - .../template-manager/template-manager.pb.go | 265 +++++++++--------- 5 files changed, 132 insertions(+), 160 deletions(-) diff --git a/packages/orchestrator/pkg/sandbox/uffd/uffd.go b/packages/orchestrator/pkg/sandbox/uffd/uffd.go index e5570eb919..c69dd4a607 100644 --- a/packages/orchestrator/pkg/sandbox/uffd/uffd.go +++ b/packages/orchestrator/pkg/sandbox/uffd/uffd.go @@ -227,16 +227,9 @@ func (u *Uffd) DiffMetadata(ctx context.Context, f *fc.Process) (*header.DiffMet return nil, fmt.Errorf("failed to get dirty memory: %w", err) } - // Without this, REMOVE'd pages stay attributed to the previous layer's BuildId in - // the merged mapping and leak their stale contents on resume; with it, those ranges - // become uuid.Nil sentinels (read as zero, matching guest free-page intent). + // Mark REMOVE'd pages as empty so they resolve to uuid.Nil (zero on restore) + // instead of leaking stale contents from the previous layer. _, empty := handler.ExportPageStates() - - // dirty ∩ empty should be empty after settle (Removed pages have no PTE, - // FC's WP-async dirty scan only sees present pages with WP cleared), but - // AndNot here means "empty wins" if the invariant ever breaks: the guest's - // most recent intent for a Removed page is "free, read zero on restore", - // so we must not let stale dirty content shadow it in this diff layer. diff.Dirty.AndNot(empty) return &header.DiffMetadata{ diff --git a/packages/orchestrator/pkg/sandbox/uffd/userfaultfd/userfaultfd.go b/packages/orchestrator/pkg/sandbox/uffd/userfaultfd/userfaultfd.go index 48ab3d8a1b..b447b15d4a 100644 --- a/packages/orchestrator/pkg/sandbox/uffd/userfaultfd/userfaultfd.go +++ b/packages/orchestrator/pkg/sandbox/uffd/userfaultfd/userfaultfd.go @@ -137,15 +137,10 @@ func NewUserfaultfdFromFd(fd uintptr, src block.Slicer, m *memory.Mapping, logge return u, nil } -// ExportPageStates returns snapshots of the faulted and removed -// page-index bitmaps. The diff-snapshot writer uses the removed -// bitmap to mark freed pages as DiffMetadata.Empty so they resolve -// to uuid.Nil (zero-on-restore) in the snapshot header. +// ExportPageStates returns snapshots of the faulted and removed page-index +// bitmaps after draining in-flight serve-loop iterations and workers. +// Lock order matches the serve loop to avoid AB-BA inversion. func (u *Userfaultfd) ExportPageStates() (faulted, removed *roaring.Bitmap) { - // Take readSerial first to wait for the serve loop's current - // iteration (read+apply) to complete, then settleRequests to wait - // for in-flight workers. Same lock order as serve loop avoids any - // AB-BA inversion. u.readSerial.Lock() defer u.readSerial.Unlock() diff --git a/packages/orchestrator/pkg/template/build/config/config.go b/packages/orchestrator/pkg/template/build/config/config.go index 4c28adff2e..825f9021bf 100644 --- a/packages/orchestrator/pkg/template/build/config/config.go +++ b/packages/orchestrator/pkg/template/build/config/config.go @@ -41,8 +41,7 @@ type TemplateConfig struct { // HugePages sets whether the VM use huge pages. HugePages bool - // FreePageReporting enables Firecracker's balloon free-page-reporting, - // which madvise(MADV_DONTNEED)s freed guest pages back to the host. + // FreePageReporting enables Firecracker's balloon free-page-reporting. FreePageReporting bool // Command to run to check if the template is ready. diff --git a/packages/orchestrator/template-manager.proto b/packages/orchestrator/template-manager.proto index 3187950566..1e70950637 100644 --- a/packages/orchestrator/template-manager.proto +++ b/packages/orchestrator/template-manager.proto @@ -86,8 +86,6 @@ message TemplateConfig { optional FromImageRegistry fromImageRegistry = 15; string teamID = 16; - - optional bool freePageReporting = 17; } message TemplateCreateRequest { diff --git a/packages/shared/pkg/grpc/template-manager/template-manager.pb.go b/packages/shared/pkg/grpc/template-manager/template-manager.pb.go index 8c6ceae914..131823fd24 100644 --- a/packages/shared/pkg/grpc/template-manager/template-manager.pb.go +++ b/packages/shared/pkg/grpc/template-manager/template-manager.pb.go @@ -706,7 +706,6 @@ type TemplateConfig struct { Source isTemplateConfig_Source `protobuf_oneof:"source"` FromImageRegistry *FromImageRegistry `protobuf:"bytes,15,opt,name=fromImageRegistry,proto3,oneof" json:"fromImageRegistry,omitempty"` TeamID string `protobuf:"bytes,16,opt,name=teamID,proto3" json:"teamID,omitempty"` - FreePageReporting *bool `protobuf:"varint,17,opt,name=freePageReporting,proto3,oneof" json:"freePageReporting,omitempty"` } func (x *TemplateConfig) Reset() { @@ -863,13 +862,6 @@ func (x *TemplateConfig) GetTeamID() string { return "" } -func (x *TemplateConfig) GetFreePageReporting() bool { - if x != nil && x.FreePageReporting != nil { - return *x.FreePageReporting - } - return false -} - type isTemplateConfig_Source interface { isTemplateConfig_Source() } @@ -1440,7 +1432,7 @@ var file_template_manager_proto_rawDesc = []byte{ 0x03, 0x67, 0x63, 0x70, 0x12, 0x2c, 0x0a, 0x07, 0x67, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x6c, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x10, 0x2e, 0x47, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x6c, 0x52, 0x65, 0x67, 0x69, 0x73, 0x74, 0x72, 0x79, 0x48, 0x00, 0x52, 0x07, 0x67, 0x65, 0x6e, 0x65, 0x72, - 0x61, 0x6c, 0x42, 0x06, 0x0a, 0x04, 0x74, 0x79, 0x70, 0x65, 0x22, 0xd9, 0x05, 0x0a, 0x0e, 0x54, + 0x61, 0x6c, 0x42, 0x06, 0x0a, 0x04, 0x74, 0x79, 0x70, 0x65, 0x22, 0x90, 0x05, 0x0a, 0x0e, 0x54, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x1e, 0x0a, 0x0a, 0x74, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x49, 0x44, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x74, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x49, 0x44, 0x12, 0x18, 0x0a, @@ -1479,139 +1471,134 @@ var file_template_manager_proto_rawDesc = []byte{ 0x74, 0x72, 0x79, 0x48, 0x02, 0x52, 0x11, 0x66, 0x72, 0x6f, 0x6d, 0x49, 0x6d, 0x61, 0x67, 0x65, 0x52, 0x65, 0x67, 0x69, 0x73, 0x74, 0x72, 0x79, 0x88, 0x01, 0x01, 0x12, 0x16, 0x0a, 0x06, 0x74, 0x65, 0x61, 0x6d, 0x49, 0x44, 0x18, 0x10, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x74, 0x65, 0x61, - 0x6d, 0x49, 0x44, 0x12, 0x31, 0x0a, 0x11, 0x66, 0x72, 0x65, 0x65, 0x50, 0x61, 0x67, 0x65, 0x52, - 0x65, 0x70, 0x6f, 0x72, 0x74, 0x69, 0x6e, 0x67, 0x18, 0x11, 0x20, 0x01, 0x28, 0x08, 0x48, 0x03, - 0x52, 0x11, 0x66, 0x72, 0x65, 0x65, 0x50, 0x61, 0x67, 0x65, 0x52, 0x65, 0x70, 0x6f, 0x72, 0x74, - 0x69, 0x6e, 0x67, 0x88, 0x01, 0x01, 0x42, 0x08, 0x0a, 0x06, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, - 0x42, 0x08, 0x0a, 0x06, 0x5f, 0x66, 0x6f, 0x72, 0x63, 0x65, 0x42, 0x14, 0x0a, 0x12, 0x5f, 0x66, - 0x72, 0x6f, 0x6d, 0x49, 0x6d, 0x61, 0x67, 0x65, 0x52, 0x65, 0x67, 0x69, 0x73, 0x74, 0x72, 0x79, - 0x42, 0x14, 0x0a, 0x12, 0x5f, 0x66, 0x72, 0x65, 0x65, 0x50, 0x61, 0x67, 0x65, 0x52, 0x65, 0x70, - 0x6f, 0x72, 0x74, 0x69, 0x6e, 0x67, 0x22, 0xa3, 0x01, 0x0a, 0x15, 0x54, 0x65, 0x6d, 0x70, 0x6c, - 0x61, 0x74, 0x65, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, - 0x12, 0x2b, 0x0a, 0x08, 0x74, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x18, 0x01, 0x20, 0x01, - 0x28, 0x0b, 0x32, 0x0f, 0x2e, 0x54, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x43, 0x6f, 0x6e, - 0x66, 0x69, 0x67, 0x52, 0x08, 0x74, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x12, 0x23, 0x0a, - 0x0a, 0x63, 0x61, 0x63, 0x68, 0x65, 0x53, 0x63, 0x6f, 0x70, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, - 0x09, 0x48, 0x00, 0x52, 0x0a, 0x63, 0x61, 0x63, 0x68, 0x65, 0x53, 0x63, 0x6f, 0x70, 0x65, 0x88, - 0x01, 0x01, 0x12, 0x1d, 0x0a, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x03, 0x20, - 0x01, 0x28, 0x09, 0x48, 0x01, 0x52, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x88, 0x01, - 0x01, 0x42, 0x0d, 0x0a, 0x0b, 0x5f, 0x63, 0x61, 0x63, 0x68, 0x65, 0x53, 0x63, 0x6f, 0x70, 0x65, - 0x42, 0x0a, 0x0a, 0x08, 0x5f, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x22, 0x8b, 0x03, 0x0a, - 0x15, 0x54, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, - 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1e, 0x0a, 0x0a, 0x74, 0x65, 0x6d, 0x70, 0x6c, 0x61, - 0x74, 0x65, 0x49, 0x44, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x74, 0x65, 0x6d, 0x70, - 0x6c, 0x61, 0x74, 0x65, 0x49, 0x44, 0x12, 0x18, 0x0a, 0x07, 0x62, 0x75, 0x69, 0x6c, 0x64, 0x49, - 0x44, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x62, 0x75, 0x69, 0x6c, 0x64, 0x49, 0x44, - 0x12, 0x1b, 0x0a, 0x06, 0x6f, 0x66, 0x66, 0x73, 0x65, 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, 0x05, - 0x48, 0x00, 0x52, 0x06, 0x6f, 0x66, 0x66, 0x73, 0x65, 0x74, 0x88, 0x01, 0x01, 0x12, 0x24, 0x0a, - 0x05, 0x6c, 0x65, 0x76, 0x65, 0x6c, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x09, 0x2e, 0x4c, - 0x6f, 0x67, 0x4c, 0x65, 0x76, 0x65, 0x6c, 0x48, 0x01, 0x52, 0x05, 0x6c, 0x65, 0x76, 0x65, 0x6c, - 0x88, 0x01, 0x01, 0x12, 0x19, 0x0a, 0x05, 0x6c, 0x69, 0x6d, 0x69, 0x74, 0x18, 0x05, 0x20, 0x01, - 0x28, 0x0d, 0x48, 0x02, 0x52, 0x05, 0x6c, 0x69, 0x6d, 0x69, 0x74, 0x88, 0x01, 0x01, 0x12, 0x35, - 0x0a, 0x05, 0x73, 0x74, 0x61, 0x72, 0x74, 0x18, 0x06, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, - 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, - 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x48, 0x03, 0x52, 0x05, 0x73, 0x74, 0x61, - 0x72, 0x74, 0x88, 0x01, 0x01, 0x12, 0x31, 0x0a, 0x03, 0x65, 0x6e, 0x64, 0x18, 0x07, 0x20, 0x01, - 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, - 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x48, 0x04, - 0x52, 0x03, 0x65, 0x6e, 0x64, 0x88, 0x01, 0x01, 0x12, 0x31, 0x0a, 0x09, 0x64, 0x69, 0x72, 0x65, - 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x08, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x0e, 0x2e, 0x4c, 0x6f, - 0x67, 0x73, 0x44, 0x69, 0x72, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x48, 0x05, 0x52, 0x09, 0x64, - 0x69, 0x72, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x88, 0x01, 0x01, 0x42, 0x09, 0x0a, 0x07, 0x5f, - 0x6f, 0x66, 0x66, 0x73, 0x65, 0x74, 0x42, 0x08, 0x0a, 0x06, 0x5f, 0x6c, 0x65, 0x76, 0x65, 0x6c, - 0x42, 0x08, 0x0a, 0x06, 0x5f, 0x6c, 0x69, 0x6d, 0x69, 0x74, 0x42, 0x08, 0x0a, 0x06, 0x5f, 0x73, - 0x74, 0x61, 0x72, 0x74, 0x42, 0x06, 0x0a, 0x04, 0x5f, 0x65, 0x6e, 0x64, 0x42, 0x0c, 0x0a, 0x0a, - 0x5f, 0x64, 0x69, 0x72, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x22, 0x56, 0x0a, 0x1a, 0x54, 0x65, - 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x42, 0x75, 0x69, 0x6c, 0x64, 0x44, 0x65, 0x6c, 0x65, 0x74, - 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x18, 0x0a, 0x07, 0x62, 0x75, 0x69, 0x6c, - 0x64, 0x49, 0x44, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x62, 0x75, 0x69, 0x6c, 0x64, - 0x49, 0x44, 0x12, 0x1e, 0x0a, 0x0a, 0x74, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x49, 0x44, - 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x74, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, - 0x49, 0x44, 0x22, 0xbb, 0x01, 0x0a, 0x15, 0x54, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x42, - 0x75, 0x69, 0x6c, 0x64, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x12, 0x24, 0x0a, 0x0d, - 0x72, 0x6f, 0x6f, 0x74, 0x66, 0x73, 0x53, 0x69, 0x7a, 0x65, 0x4b, 0x65, 0x79, 0x18, 0x01, 0x20, - 0x01, 0x28, 0x05, 0x52, 0x0d, 0x72, 0x6f, 0x6f, 0x74, 0x66, 0x73, 0x53, 0x69, 0x7a, 0x65, 0x4b, - 0x65, 0x79, 0x12, 0x26, 0x0a, 0x0e, 0x65, 0x6e, 0x76, 0x64, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, - 0x6e, 0x4b, 0x65, 0x79, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0e, 0x65, 0x6e, 0x76, 0x64, - 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x4b, 0x65, 0x79, 0x12, 0x24, 0x0a, 0x0d, 0x6b, 0x65, - 0x72, 0x6e, 0x65, 0x6c, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x03, 0x20, 0x01, 0x28, - 0x09, 0x52, 0x0d, 0x6b, 0x65, 0x72, 0x6e, 0x65, 0x6c, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, - 0x12, 0x2e, 0x0a, 0x12, 0x66, 0x69, 0x72, 0x65, 0x63, 0x72, 0x61, 0x63, 0x6b, 0x65, 0x72, 0x56, - 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x12, 0x66, 0x69, - 0x72, 0x65, 0x63, 0x72, 0x61, 0x63, 0x6b, 0x65, 0x72, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, - 0x22, 0x83, 0x02, 0x0a, 0x15, 0x54, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x42, 0x75, 0x69, - 0x6c, 0x64, 0x4c, 0x6f, 0x67, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x38, 0x0a, 0x09, 0x74, 0x69, - 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, - 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, - 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x09, 0x74, 0x69, 0x6d, 0x65, 0x73, - 0x74, 0x61, 0x6d, 0x70, 0x12, 0x18, 0x0a, 0x07, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x18, - 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x12, 0x1f, - 0x0a, 0x05, 0x6c, 0x65, 0x76, 0x65, 0x6c, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x09, 0x2e, - 0x4c, 0x6f, 0x67, 0x4c, 0x65, 0x76, 0x65, 0x6c, 0x52, 0x05, 0x6c, 0x65, 0x76, 0x65, 0x6c, 0x12, - 0x3a, 0x0a, 0x06, 0x66, 0x69, 0x65, 0x6c, 0x64, 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, 0x0b, 0x32, - 0x22, 0x2e, 0x54, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x42, 0x75, 0x69, 0x6c, 0x64, 0x4c, - 0x6f, 0x67, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x2e, 0x46, 0x69, 0x65, 0x6c, 0x64, 0x73, 0x45, 0x6e, - 0x74, 0x72, 0x79, 0x52, 0x06, 0x66, 0x69, 0x65, 0x6c, 0x64, 0x73, 0x1a, 0x39, 0x0a, 0x0b, 0x46, - 0x69, 0x65, 0x6c, 0x64, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, - 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, - 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, - 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0x57, 0x0a, 0x19, 0x54, 0x65, 0x6d, 0x70, 0x6c, 0x61, - 0x74, 0x65, 0x42, 0x75, 0x69, 0x6c, 0x64, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x61, - 0x73, 0x6f, 0x6e, 0x12, 0x18, 0x0a, 0x07, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x18, 0x01, - 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x12, 0x17, 0x0a, - 0x04, 0x73, 0x74, 0x65, 0x70, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x48, 0x00, 0x52, 0x04, 0x73, - 0x74, 0x65, 0x70, 0x88, 0x01, 0x01, 0x42, 0x07, 0x0a, 0x05, 0x5f, 0x73, 0x74, 0x65, 0x70, 0x22, - 0x86, 0x02, 0x0a, 0x1b, 0x54, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x42, 0x75, 0x69, 0x6c, - 0x64, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, - 0x2b, 0x0a, 0x06, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0e, 0x32, - 0x13, 0x2e, 0x54, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x42, 0x75, 0x69, 0x6c, 0x64, 0x53, - 0x74, 0x61, 0x74, 0x65, 0x52, 0x06, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x32, 0x0a, 0x08, - 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x16, - 0x2e, 0x54, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x42, 0x75, 0x69, 0x6c, 0x64, 0x4d, 0x65, - 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x52, 0x08, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, - 0x12, 0x36, 0x0a, 0x0a, 0x6c, 0x6f, 0x67, 0x45, 0x6e, 0x74, 0x72, 0x69, 0x65, 0x73, 0x18, 0x05, - 0x20, 0x03, 0x28, 0x0b, 0x32, 0x16, 0x2e, 0x54, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x42, - 0x75, 0x69, 0x6c, 0x64, 0x4c, 0x6f, 0x67, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x0a, 0x6c, 0x6f, - 0x67, 0x45, 0x6e, 0x74, 0x72, 0x69, 0x65, 0x73, 0x12, 0x37, 0x0a, 0x06, 0x72, 0x65, 0x61, 0x73, - 0x6f, 0x6e, 0x18, 0x06, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x54, 0x65, 0x6d, 0x70, 0x6c, + 0x6d, 0x49, 0x44, 0x42, 0x08, 0x0a, 0x06, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x42, 0x08, 0x0a, + 0x06, 0x5f, 0x66, 0x6f, 0x72, 0x63, 0x65, 0x42, 0x14, 0x0a, 0x12, 0x5f, 0x66, 0x72, 0x6f, 0x6d, + 0x49, 0x6d, 0x61, 0x67, 0x65, 0x52, 0x65, 0x67, 0x69, 0x73, 0x74, 0x72, 0x79, 0x22, 0xa3, 0x01, + 0x0a, 0x15, 0x54, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, + 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x2b, 0x0a, 0x08, 0x74, 0x65, 0x6d, 0x70, 0x6c, + 0x61, 0x74, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0f, 0x2e, 0x54, 0x65, 0x6d, 0x70, + 0x6c, 0x61, 0x74, 0x65, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, 0x08, 0x74, 0x65, 0x6d, 0x70, + 0x6c, 0x61, 0x74, 0x65, 0x12, 0x23, 0x0a, 0x0a, 0x63, 0x61, 0x63, 0x68, 0x65, 0x53, 0x63, 0x6f, + 0x70, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x48, 0x00, 0x52, 0x0a, 0x63, 0x61, 0x63, 0x68, + 0x65, 0x53, 0x63, 0x6f, 0x70, 0x65, 0x88, 0x01, 0x01, 0x12, 0x1d, 0x0a, 0x07, 0x76, 0x65, 0x72, + 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x48, 0x01, 0x52, 0x07, 0x76, 0x65, + 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x88, 0x01, 0x01, 0x42, 0x0d, 0x0a, 0x0b, 0x5f, 0x63, 0x61, 0x63, + 0x68, 0x65, 0x53, 0x63, 0x6f, 0x70, 0x65, 0x42, 0x0a, 0x0a, 0x08, 0x5f, 0x76, 0x65, 0x72, 0x73, + 0x69, 0x6f, 0x6e, 0x22, 0x8b, 0x03, 0x0a, 0x15, 0x54, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, + 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1e, 0x0a, + 0x0a, 0x74, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x49, 0x44, 0x18, 0x01, 0x20, 0x01, 0x28, + 0x09, 0x52, 0x0a, 0x74, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x49, 0x44, 0x12, 0x18, 0x0a, + 0x07, 0x62, 0x75, 0x69, 0x6c, 0x64, 0x49, 0x44, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, + 0x62, 0x75, 0x69, 0x6c, 0x64, 0x49, 0x44, 0x12, 0x1b, 0x0a, 0x06, 0x6f, 0x66, 0x66, 0x73, 0x65, + 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, 0x05, 0x48, 0x00, 0x52, 0x06, 0x6f, 0x66, 0x66, 0x73, 0x65, + 0x74, 0x88, 0x01, 0x01, 0x12, 0x24, 0x0a, 0x05, 0x6c, 0x65, 0x76, 0x65, 0x6c, 0x18, 0x04, 0x20, + 0x01, 0x28, 0x0e, 0x32, 0x09, 0x2e, 0x4c, 0x6f, 0x67, 0x4c, 0x65, 0x76, 0x65, 0x6c, 0x48, 0x01, + 0x52, 0x05, 0x6c, 0x65, 0x76, 0x65, 0x6c, 0x88, 0x01, 0x01, 0x12, 0x19, 0x0a, 0x05, 0x6c, 0x69, + 0x6d, 0x69, 0x74, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0d, 0x48, 0x02, 0x52, 0x05, 0x6c, 0x69, 0x6d, + 0x69, 0x74, 0x88, 0x01, 0x01, 0x12, 0x35, 0x0a, 0x05, 0x73, 0x74, 0x61, 0x72, 0x74, 0x18, 0x06, + 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, + 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, + 0x48, 0x03, 0x52, 0x05, 0x73, 0x74, 0x61, 0x72, 0x74, 0x88, 0x01, 0x01, 0x12, 0x31, 0x0a, 0x03, + 0x65, 0x6e, 0x64, 0x18, 0x07, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, + 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, + 0x73, 0x74, 0x61, 0x6d, 0x70, 0x48, 0x04, 0x52, 0x03, 0x65, 0x6e, 0x64, 0x88, 0x01, 0x01, 0x12, + 0x31, 0x0a, 0x09, 0x64, 0x69, 0x72, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x08, 0x20, 0x01, + 0x28, 0x0e, 0x32, 0x0e, 0x2e, 0x4c, 0x6f, 0x67, 0x73, 0x44, 0x69, 0x72, 0x65, 0x63, 0x74, 0x69, + 0x6f, 0x6e, 0x48, 0x05, 0x52, 0x09, 0x64, 0x69, 0x72, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x88, + 0x01, 0x01, 0x42, 0x09, 0x0a, 0x07, 0x5f, 0x6f, 0x66, 0x66, 0x73, 0x65, 0x74, 0x42, 0x08, 0x0a, + 0x06, 0x5f, 0x6c, 0x65, 0x76, 0x65, 0x6c, 0x42, 0x08, 0x0a, 0x06, 0x5f, 0x6c, 0x69, 0x6d, 0x69, + 0x74, 0x42, 0x08, 0x0a, 0x06, 0x5f, 0x73, 0x74, 0x61, 0x72, 0x74, 0x42, 0x06, 0x0a, 0x04, 0x5f, + 0x65, 0x6e, 0x64, 0x42, 0x0c, 0x0a, 0x0a, 0x5f, 0x64, 0x69, 0x72, 0x65, 0x63, 0x74, 0x69, 0x6f, + 0x6e, 0x22, 0x56, 0x0a, 0x1a, 0x54, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x42, 0x75, 0x69, + 0x6c, 0x64, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, + 0x18, 0x0a, 0x07, 0x62, 0x75, 0x69, 0x6c, 0x64, 0x49, 0x44, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, + 0x52, 0x07, 0x62, 0x75, 0x69, 0x6c, 0x64, 0x49, 0x44, 0x12, 0x1e, 0x0a, 0x0a, 0x74, 0x65, 0x6d, + 0x70, 0x6c, 0x61, 0x74, 0x65, 0x49, 0x44, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x74, + 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x49, 0x44, 0x22, 0xbb, 0x01, 0x0a, 0x15, 0x54, 0x65, + 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x42, 0x75, 0x69, 0x6c, 0x64, 0x4d, 0x65, 0x74, 0x61, 0x64, + 0x61, 0x74, 0x61, 0x12, 0x24, 0x0a, 0x0d, 0x72, 0x6f, 0x6f, 0x74, 0x66, 0x73, 0x53, 0x69, 0x7a, + 0x65, 0x4b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x05, 0x52, 0x0d, 0x72, 0x6f, 0x6f, 0x74, + 0x66, 0x73, 0x53, 0x69, 0x7a, 0x65, 0x4b, 0x65, 0x79, 0x12, 0x26, 0x0a, 0x0e, 0x65, 0x6e, 0x76, + 0x64, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x4b, 0x65, 0x79, 0x18, 0x02, 0x20, 0x01, 0x28, + 0x09, 0x52, 0x0e, 0x65, 0x6e, 0x76, 0x64, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x4b, 0x65, + 0x79, 0x12, 0x24, 0x0a, 0x0d, 0x6b, 0x65, 0x72, 0x6e, 0x65, 0x6c, 0x56, 0x65, 0x72, 0x73, 0x69, + 0x6f, 0x6e, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0d, 0x6b, 0x65, 0x72, 0x6e, 0x65, 0x6c, + 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x2e, 0x0a, 0x12, 0x66, 0x69, 0x72, 0x65, 0x63, + 0x72, 0x61, 0x63, 0x6b, 0x65, 0x72, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x04, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x12, 0x66, 0x69, 0x72, 0x65, 0x63, 0x72, 0x61, 0x63, 0x6b, 0x65, 0x72, + 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x22, 0x83, 0x02, 0x0a, 0x15, 0x54, 0x65, 0x6d, 0x70, + 0x6c, 0x61, 0x74, 0x65, 0x42, 0x75, 0x69, 0x6c, 0x64, 0x4c, 0x6f, 0x67, 0x45, 0x6e, 0x74, 0x72, + 0x79, 0x12, 0x38, 0x0a, 0x09, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x18, 0x01, + 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, + 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, + 0x52, 0x09, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x12, 0x18, 0x0a, 0x07, 0x6d, + 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x6d, 0x65, + 0x73, 0x73, 0x61, 0x67, 0x65, 0x12, 0x1f, 0x0a, 0x05, 0x6c, 0x65, 0x76, 0x65, 0x6c, 0x18, 0x03, + 0x20, 0x01, 0x28, 0x0e, 0x32, 0x09, 0x2e, 0x4c, 0x6f, 0x67, 0x4c, 0x65, 0x76, 0x65, 0x6c, 0x52, + 0x05, 0x6c, 0x65, 0x76, 0x65, 0x6c, 0x12, 0x3a, 0x0a, 0x06, 0x66, 0x69, 0x65, 0x6c, 0x64, 0x73, + 0x18, 0x04, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x22, 0x2e, 0x54, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, + 0x65, 0x42, 0x75, 0x69, 0x6c, 0x64, 0x4c, 0x6f, 0x67, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x2e, 0x46, + 0x69, 0x65, 0x6c, 0x64, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x06, 0x66, 0x69, 0x65, 0x6c, + 0x64, 0x73, 0x1a, 0x39, 0x0a, 0x0b, 0x46, 0x69, 0x65, 0x6c, 0x64, 0x73, 0x45, 0x6e, 0x74, 0x72, + 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, + 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, + 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0x57, 0x0a, + 0x19, 0x54, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x42, 0x75, 0x69, 0x6c, 0x64, 0x53, 0x74, + 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x61, 0x73, 0x6f, 0x6e, 0x12, 0x18, 0x0a, 0x07, 0x6d, 0x65, + 0x73, 0x73, 0x61, 0x67, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x6d, 0x65, 0x73, + 0x73, 0x61, 0x67, 0x65, 0x12, 0x17, 0x0a, 0x04, 0x73, 0x74, 0x65, 0x70, 0x18, 0x02, 0x20, 0x01, + 0x28, 0x09, 0x48, 0x00, 0x52, 0x04, 0x73, 0x74, 0x65, 0x70, 0x88, 0x01, 0x01, 0x42, 0x07, 0x0a, + 0x05, 0x5f, 0x73, 0x74, 0x65, 0x70, 0x22, 0x86, 0x02, 0x0a, 0x1b, 0x54, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x42, 0x75, 0x69, 0x6c, 0x64, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, - 0x61, 0x73, 0x6f, 0x6e, 0x48, 0x00, 0x52, 0x06, 0x72, 0x65, 0x61, 0x73, 0x6f, 0x6e, 0x88, 0x01, - 0x01, 0x42, 0x09, 0x0a, 0x07, 0x5f, 0x72, 0x65, 0x61, 0x73, 0x6f, 0x6e, 0x4a, 0x04, 0x08, 0x03, - 0x10, 0x04, 0x4a, 0x04, 0x08, 0x04, 0x10, 0x05, 0x2a, 0x34, 0x0a, 0x08, 0x4c, 0x6f, 0x67, 0x4c, - 0x65, 0x76, 0x65, 0x6c, 0x12, 0x09, 0x0a, 0x05, 0x44, 0x65, 0x62, 0x75, 0x67, 0x10, 0x00, 0x12, - 0x08, 0x0a, 0x04, 0x49, 0x6e, 0x66, 0x6f, 0x10, 0x01, 0x12, 0x08, 0x0a, 0x04, 0x57, 0x61, 0x72, - 0x6e, 0x10, 0x02, 0x12, 0x09, 0x0a, 0x05, 0x45, 0x72, 0x72, 0x6f, 0x72, 0x10, 0x03, 0x2a, 0x2a, - 0x0a, 0x0d, 0x4c, 0x6f, 0x67, 0x73, 0x44, 0x69, 0x72, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x12, - 0x0b, 0x0a, 0x07, 0x46, 0x6f, 0x72, 0x77, 0x61, 0x72, 0x64, 0x10, 0x00, 0x12, 0x0c, 0x0a, 0x08, - 0x42, 0x61, 0x63, 0x6b, 0x77, 0x61, 0x72, 0x64, 0x10, 0x01, 0x2a, 0x3d, 0x0a, 0x12, 0x54, 0x65, - 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x42, 0x75, 0x69, 0x6c, 0x64, 0x53, 0x74, 0x61, 0x74, 0x65, - 0x12, 0x0c, 0x0a, 0x08, 0x42, 0x75, 0x69, 0x6c, 0x64, 0x69, 0x6e, 0x67, 0x10, 0x00, 0x12, 0x0a, - 0x0a, 0x06, 0x46, 0x61, 0x69, 0x6c, 0x65, 0x64, 0x10, 0x01, 0x12, 0x0d, 0x0a, 0x09, 0x43, 0x6f, - 0x6d, 0x70, 0x6c, 0x65, 0x74, 0x65, 0x64, 0x10, 0x02, 0x32, 0xbe, 0x02, 0x0a, 0x0f, 0x54, 0x65, - 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0x40, 0x0a, - 0x0e, 0x54, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x12, - 0x16, 0x2e, 0x54, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, + 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x2b, 0x0a, 0x06, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, + 0x18, 0x01, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x13, 0x2e, 0x54, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, + 0x65, 0x42, 0x75, 0x69, 0x6c, 0x64, 0x53, 0x74, 0x61, 0x74, 0x65, 0x52, 0x06, 0x73, 0x74, 0x61, + 0x74, 0x75, 0x73, 0x12, 0x32, 0x0a, 0x08, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x18, + 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x16, 0x2e, 0x54, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, + 0x42, 0x75, 0x69, 0x6c, 0x64, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x52, 0x08, 0x6d, + 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x12, 0x36, 0x0a, 0x0a, 0x6c, 0x6f, 0x67, 0x45, 0x6e, + 0x74, 0x72, 0x69, 0x65, 0x73, 0x18, 0x05, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x16, 0x2e, 0x54, 0x65, + 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x42, 0x75, 0x69, 0x6c, 0x64, 0x4c, 0x6f, 0x67, 0x45, 0x6e, + 0x74, 0x72, 0x79, 0x52, 0x0a, 0x6c, 0x6f, 0x67, 0x45, 0x6e, 0x74, 0x72, 0x69, 0x65, 0x73, 0x12, + 0x37, 0x0a, 0x06, 0x72, 0x65, 0x61, 0x73, 0x6f, 0x6e, 0x18, 0x06, 0x20, 0x01, 0x28, 0x0b, 0x32, + 0x1a, 0x2e, 0x54, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x42, 0x75, 0x69, 0x6c, 0x64, 0x53, + 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x61, 0x73, 0x6f, 0x6e, 0x48, 0x00, 0x52, 0x06, 0x72, + 0x65, 0x61, 0x73, 0x6f, 0x6e, 0x88, 0x01, 0x01, 0x42, 0x09, 0x0a, 0x07, 0x5f, 0x72, 0x65, 0x61, + 0x73, 0x6f, 0x6e, 0x4a, 0x04, 0x08, 0x03, 0x10, 0x04, 0x4a, 0x04, 0x08, 0x04, 0x10, 0x05, 0x2a, + 0x34, 0x0a, 0x08, 0x4c, 0x6f, 0x67, 0x4c, 0x65, 0x76, 0x65, 0x6c, 0x12, 0x09, 0x0a, 0x05, 0x44, + 0x65, 0x62, 0x75, 0x67, 0x10, 0x00, 0x12, 0x08, 0x0a, 0x04, 0x49, 0x6e, 0x66, 0x6f, 0x10, 0x01, + 0x12, 0x08, 0x0a, 0x04, 0x57, 0x61, 0x72, 0x6e, 0x10, 0x02, 0x12, 0x09, 0x0a, 0x05, 0x45, 0x72, + 0x72, 0x6f, 0x72, 0x10, 0x03, 0x2a, 0x2a, 0x0a, 0x0d, 0x4c, 0x6f, 0x67, 0x73, 0x44, 0x69, 0x72, + 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x0b, 0x0a, 0x07, 0x46, 0x6f, 0x72, 0x77, 0x61, 0x72, + 0x64, 0x10, 0x00, 0x12, 0x0c, 0x0a, 0x08, 0x42, 0x61, 0x63, 0x6b, 0x77, 0x61, 0x72, 0x64, 0x10, + 0x01, 0x2a, 0x3d, 0x0a, 0x12, 0x54, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x42, 0x75, 0x69, + 0x6c, 0x64, 0x53, 0x74, 0x61, 0x74, 0x65, 0x12, 0x0c, 0x0a, 0x08, 0x42, 0x75, 0x69, 0x6c, 0x64, + 0x69, 0x6e, 0x67, 0x10, 0x00, 0x12, 0x0a, 0x0a, 0x06, 0x46, 0x61, 0x69, 0x6c, 0x65, 0x64, 0x10, + 0x01, 0x12, 0x0d, 0x0a, 0x09, 0x43, 0x6f, 0x6d, 0x70, 0x6c, 0x65, 0x74, 0x65, 0x64, 0x10, 0x02, + 0x32, 0xbe, 0x02, 0x0a, 0x0f, 0x54, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x53, 0x65, 0x72, + 0x76, 0x69, 0x63, 0x65, 0x12, 0x40, 0x0a, 0x0e, 0x54, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, + 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x12, 0x16, 0x2e, 0x54, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, + 0x65, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x16, + 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, + 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x12, 0x4b, 0x0a, 0x13, 0x54, 0x65, 0x6d, 0x70, 0x6c, 0x61, + 0x74, 0x65, 0x42, 0x75, 0x69, 0x6c, 0x64, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x16, 0x2e, + 0x54, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, + 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1c, 0x2e, 0x54, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, + 0x42, 0x75, 0x69, 0x6c, 0x64, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, + 0x6e, 0x73, 0x65, 0x12, 0x4a, 0x0a, 0x13, 0x54, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x42, + 0x75, 0x69, 0x6c, 0x64, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x12, 0x1b, 0x2e, 0x54, 0x65, 0x6d, + 0x70, 0x6c, 0x61, 0x74, 0x65, 0x42, 0x75, 0x69, 0x6c, 0x64, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x12, - 0x4b, 0x0a, 0x13, 0x54, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x42, 0x75, 0x69, 0x6c, 0x64, - 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x16, 0x2e, 0x54, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, - 0x65, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1c, - 0x2e, 0x54, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x42, 0x75, 0x69, 0x6c, 0x64, 0x53, 0x74, - 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x4a, 0x0a, 0x13, - 0x54, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x42, 0x75, 0x69, 0x6c, 0x64, 0x44, 0x65, 0x6c, - 0x65, 0x74, 0x65, 0x12, 0x1b, 0x2e, 0x54, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x42, 0x75, - 0x69, 0x6c, 0x64, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, - 0x1a, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, - 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x12, 0x50, 0x0a, 0x13, 0x49, 0x6e, 0x69, 0x74, - 0x4c, 0x61, 0x79, 0x65, 0x72, 0x46, 0x69, 0x6c, 0x65, 0x55, 0x70, 0x6c, 0x6f, 0x61, 0x64, 0x12, - 0x1b, 0x2e, 0x49, 0x6e, 0x69, 0x74, 0x4c, 0x61, 0x79, 0x65, 0x72, 0x46, 0x69, 0x6c, 0x65, 0x55, - 0x70, 0x6c, 0x6f, 0x61, 0x64, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1c, 0x2e, 0x49, - 0x6e, 0x69, 0x74, 0x4c, 0x61, 0x79, 0x65, 0x72, 0x46, 0x69, 0x6c, 0x65, 0x55, 0x70, 0x6c, 0x6f, - 0x61, 0x64, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x42, 0x33, 0x5a, 0x31, 0x68, 0x74, - 0x74, 0x70, 0x73, 0x3a, 0x2f, 0x2f, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, - 0x2f, 0x65, 0x32, 0x62, 0x2d, 0x64, 0x65, 0x76, 0x2f, 0x69, 0x6e, 0x66, 0x72, 0x61, 0x2f, 0x74, - 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x2d, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x62, - 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x50, 0x0a, 0x13, 0x49, 0x6e, 0x69, 0x74, 0x4c, 0x61, 0x79, 0x65, 0x72, 0x46, 0x69, 0x6c, 0x65, + 0x55, 0x70, 0x6c, 0x6f, 0x61, 0x64, 0x12, 0x1b, 0x2e, 0x49, 0x6e, 0x69, 0x74, 0x4c, 0x61, 0x79, + 0x65, 0x72, 0x46, 0x69, 0x6c, 0x65, 0x55, 0x70, 0x6c, 0x6f, 0x61, 0x64, 0x52, 0x65, 0x71, 0x75, + 0x65, 0x73, 0x74, 0x1a, 0x1c, 0x2e, 0x49, 0x6e, 0x69, 0x74, 0x4c, 0x61, 0x79, 0x65, 0x72, 0x46, + 0x69, 0x6c, 0x65, 0x55, 0x70, 0x6c, 0x6f, 0x61, 0x64, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, + 0x65, 0x42, 0x33, 0x5a, 0x31, 0x68, 0x74, 0x74, 0x70, 0x73, 0x3a, 0x2f, 0x2f, 0x67, 0x69, 0x74, + 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x65, 0x32, 0x62, 0x2d, 0x64, 0x65, 0x76, 0x2f, + 0x69, 0x6e, 0x66, 0x72, 0x61, 0x2f, 0x74, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x2d, 0x6d, + 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( From 3ae7f06e8773deff9ffa4e82c2e41e3e36098d59 Mon Sep 17 00:00:00 2001 From: Tomas Valenta Date: Tue, 5 May 2026 02:14:37 -0700 Subject: [PATCH 07/10] fix(uffd): keep tracker Zero on read-fault zero install A REMOVE'd page that gets a read fault before snapshot installs zero+WP+wake. The unconditional Zero->Dirty promotion stripped it from the empty bitmap while WP-async dirty scan still excludes it (WP set, no write yet), so the page fell back to the parent layer's stale content. Keep the tracker entry as Zero for that case and let WP-async surface any later write; flip the AndNot so dirty wins over empty for written pages. --- packages/orchestrator/pkg/sandbox/uffd/uffd.go | 6 ++++-- .../pkg/sandbox/uffd/userfaultfd/userfaultfd.go | 8 +++++++- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/packages/orchestrator/pkg/sandbox/uffd/uffd.go b/packages/orchestrator/pkg/sandbox/uffd/uffd.go index c69dd4a607..842db41dab 100644 --- a/packages/orchestrator/pkg/sandbox/uffd/uffd.go +++ b/packages/orchestrator/pkg/sandbox/uffd/uffd.go @@ -228,9 +228,11 @@ func (u *Uffd) DiffMetadata(ctx context.Context, f *fc.Process) (*header.DiffMet } // Mark REMOVE'd pages as empty so they resolve to uuid.Nil (zero on restore) - // instead of leaking stale contents from the previous layer. + // instead of leaking stale contents from the previous layer. A page that + // was zero-installed and later written shows up in diff.Dirty via WP-async, + // so dirty wins over empty for those. _, empty := handler.ExportPageStates() - diff.Dirty.AndNot(empty) + empty.AndNot(diff.Dirty) return &header.DiffMetadata{ BlockSize: diff.BlockSize, diff --git a/packages/orchestrator/pkg/sandbox/uffd/userfaultfd/userfaultfd.go b/packages/orchestrator/pkg/sandbox/uffd/userfaultfd/userfaultfd.go index b447b15d4a..1fbc0182fe 100644 --- a/packages/orchestrator/pkg/sandbox/uffd/userfaultfd/userfaultfd.go +++ b/packages/orchestrator/pkg/sandbox/uffd/userfaultfd/userfaultfd.go @@ -420,7 +420,13 @@ func (u *Userfaultfd) Serve( switch outcome { case faultInstalled: - u.pageTracker.SetRange(idx, idx+1, block.Dirty) + // Zero-fill on a read fault installs zero+WP; the page still + // reads as zero, so keep the tracker entry as Zero so the + // snapshot diff marks it Empty. WP-async will catch any + // later write and surface it via DirtyMemory. + if source != nil || accessType == block.Write { + u.pageTracker.SetRange(idx, idx+1, block.Dirty) + } u.prefetchTracker.Add(offset, accessType) case faultDeferred: deferred.push(pf) From a092da952a100a00242d203b0ccc496765ec280a Mon Sep 17 00:00:00 2001 From: ValentaTomas Date: Tue, 5 May 2026 13:23:20 -0700 Subject: [PATCH 08/10] fix(uffd,template): address PR review - DiffMetadata: settle UFFD workers before sampling FC WP-async pagemap so a Zero->Write install in flight can't escape both Dirty and Empty bitmaps. - TemplateCreate: gate freePageReporting on !hugePages. Firecracker rejects balloon device + hugepages combinations, and the v1.14 rollout would otherwise auto-enable both and abort VM creation. --- packages/orchestrator/pkg/sandbox/uffd/uffd.go | 12 +++++++----- .../pkg/template/server/create_template.go | 5 ++++- 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/packages/orchestrator/pkg/sandbox/uffd/uffd.go b/packages/orchestrator/pkg/sandbox/uffd/uffd.go index 842db41dab..2c0ee0fc63 100644 --- a/packages/orchestrator/pkg/sandbox/uffd/uffd.go +++ b/packages/orchestrator/pkg/sandbox/uffd/uffd.go @@ -222,16 +222,18 @@ func (u *Uffd) DiffMetadata(ctx context.Context, f *fc.Process) (*header.DiffMet return nil, fmt.Errorf("failed to get uffd: %w", err) } + // Settle in-flight UFFD workers (and the REMOVE batch) before sampling + // FC's WP-async pagemap, so a Zero→Write install can't slip in between + // and escape both bitmaps. + _, empty := handler.ExportPageStates() + diff, err := f.DirtyMemory(ctx, u.memfile.BlockSize()) if err != nil { return nil, fmt.Errorf("failed to get dirty memory: %w", err) } - // Mark REMOVE'd pages as empty so they resolve to uuid.Nil (zero on restore) - // instead of leaking stale contents from the previous layer. A page that - // was zero-installed and later written shows up in diff.Dirty via WP-async, - // so dirty wins over empty for those. - _, empty := handler.ExportPageStates() + // Pages that were zero-installed and later written show up in diff.Dirty + // via WP-async, so dirty wins over empty for those. empty.AndNot(diff.Dirty) return &header.DiffMetadata{ diff --git a/packages/orchestrator/pkg/template/server/create_template.go b/packages/orchestrator/pkg/template/server/create_template.go index a049548001..944edec74b 100644 --- a/packages/orchestrator/pkg/template/server/create_template.go +++ b/packages/orchestrator/pkg/template/server/create_template.go @@ -65,7 +65,10 @@ func (s *ServerStore) TemplateCreate(ctx context.Context, templateRequest *templ return nil, fmt.Errorf("invalid resolved firecracker version %q: %w", firecrackerVersion, err) } hugePages := fcInfo.HasHugePages() - freePageReporting := fcInfo.HasFreePageReporting() && s.featureFlags.BoolFlag(ctx, featureflags.FreePageReportingFlag) + // Firecracker rejects balloon (free-page-reporting) together with hugepages, + // so hugepages wins. Once the flag is enabled and we want FPR over hugepages, + // gate hugepages on the FC version that supports both — which doesn't exist yet. + freePageReporting := !hugePages && fcInfo.HasFreePageReporting() && s.featureFlags.BoolFlag(ctx, featureflags.FreePageReportingFlag) childSpan.SetAttributes( telemetry.WithTemplateID(cfg.GetTemplateID()), From 14f995fba71f2d2206fb1561ca9a5200f4d81e21 Mon Sep 17 00:00:00 2001 From: ValentaTomas Date: Wed, 6 May 2026 13:28:47 -0700 Subject: [PATCH 09/10] fix(template): allow FPR with hugepages per FC 1.14 support MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Remove the !hugePages guard — hugepages and balloon FPR are not mutually exclusive in Firecracker. --- packages/orchestrator/pkg/template/server/create_template.go | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/packages/orchestrator/pkg/template/server/create_template.go b/packages/orchestrator/pkg/template/server/create_template.go index 944edec74b..a049548001 100644 --- a/packages/orchestrator/pkg/template/server/create_template.go +++ b/packages/orchestrator/pkg/template/server/create_template.go @@ -65,10 +65,7 @@ func (s *ServerStore) TemplateCreate(ctx context.Context, templateRequest *templ return nil, fmt.Errorf("invalid resolved firecracker version %q: %w", firecrackerVersion, err) } hugePages := fcInfo.HasHugePages() - // Firecracker rejects balloon (free-page-reporting) together with hugepages, - // so hugepages wins. Once the flag is enabled and we want FPR over hugepages, - // gate hugepages on the FC version that supports both — which doesn't exist yet. - freePageReporting := !hugePages && fcInfo.HasFreePageReporting() && s.featureFlags.BoolFlag(ctx, featureflags.FreePageReportingFlag) + freePageReporting := fcInfo.HasFreePageReporting() && s.featureFlags.BoolFlag(ctx, featureflags.FreePageReportingFlag) childSpan.SetAttributes( telemetry.WithTemplateID(cfg.GetTemplateID()), From 7df828bbd0252080b1f55b561fe5ca0152313820 Mon Sep 17 00:00:00 2001 From: ValentaTomas Date: Fri, 8 May 2026 01:21:51 -0700 Subject: [PATCH 10/10] =?UTF-8?q?refactor(fc):=20rename=20enableFreePageRe?= =?UTF-8?q?porting=20=E2=86=92=20installBalloon?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Generic name and parameterized FreePageReporting so individual balloon features (FPR today, FPH next) can be opted in independently from the caller without renaming the helper again. --- packages/orchestrator/pkg/sandbox/fc/client.go | 11 +++++++---- packages/orchestrator/pkg/sandbox/fc/process.go | 6 +++--- 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/packages/orchestrator/pkg/sandbox/fc/client.go b/packages/orchestrator/pkg/sandbox/fc/client.go index d35d28040c..b6cf4ff34e 100644 --- a/packages/orchestrator/pkg/sandbox/fc/client.go +++ b/packages/orchestrator/pkg/sandbox/fc/client.go @@ -427,8 +427,11 @@ func (c *apiClient) startVM(ctx context.Context) error { return nil } -func (c *apiClient) enableFreePageReporting(ctx context.Context) error { - ctx, span := tracer.Start(ctx, "enable-free-page-reporting") +// installBalloon attaches a zero-MiB balloon device. Individual balloon +// features (free-page-reporting today, free-page-hinting next) are toggled +// via parameters so callers can opt in to any subset independently. +func (c *apiClient) installBalloon(ctx context.Context, freePageReporting bool) error { + ctx, span := tracer.Start(ctx, "install-balloon") defer span.End() amountMib := int64(0) @@ -439,13 +442,13 @@ func (c *apiClient) enableFreePageReporting(ctx context.Context) error { Body: &models.Balloon{ AmountMib: &amountMib, DeflateOnOom: &deflateOnOom, - FreePageReporting: true, + FreePageReporting: freePageReporting, }, } _, err := c.client.Operations.PutBalloon(&balloonConfig) if err != nil { - return fmt.Errorf("error setting up balloon device: %w", err) + return fmt.Errorf("error installing balloon device: %w", err) } return nil diff --git a/packages/orchestrator/pkg/sandbox/fc/process.go b/packages/orchestrator/pkg/sandbox/fc/process.go index 004de38450..c2bbb7caf7 100644 --- a/packages/orchestrator/pkg/sandbox/fc/process.go +++ b/packages/orchestrator/pkg/sandbox/fc/process.go @@ -443,13 +443,13 @@ func (p *Process) Create( telemetry.ReportEvent(ctx, "set fc entropy config") if freePageReporting { - err = p.client.enableFreePageReporting(ctx) + err = p.client.installBalloon(ctx, freePageReporting) if err != nil { fcStopErr := p.Stop(ctx) - return errors.Join(fmt.Errorf("error enabling free page reporting: %w", err), fcStopErr) + return errors.Join(fmt.Errorf("error installing balloon device: %w", err), fcStopErr) } - telemetry.ReportEvent(ctx, "enabled free page reporting") + telemetry.ReportEvent(ctx, "installed balloon device") } err = p.client.startVM(ctx)