diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 5e33214c..d6a0a919 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -150,6 +150,23 @@ jobs: done exit 1 + - name: Test Windows stopped forks + run: | + TEST_PATH="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:$PATH" + for attempt in 1 2 3; do + if sudo env \ + "PATH=$TEST_PATH" \ + "CI=true" \ + "HYPEMAN_RUN_WINDOWS_LIFECYCLE_INTEGRATION=1" \ + "HYPEMAN_WINDOWS_OVMF_CODE=$HYPEMAN_WINDOWS_OVMF_CODE" \ + "HYPEMAN_WINDOWS_OVMF_VARS=$HYPEMAN_WINDOWS_OVMF_VARS" \ + go test -count=1 -run '^TestWindowsStoppedForkIntegration$' -timeout 2m ./lib/instances; then + exit 0 + fi + test "$attempt" = 3 || sleep 5 + done + exit 1 + # Slash-command runs are maintainer-approved and need authenticated pulls # for images that are not covered by the prewarm cache. - name: Login to Docker Hub diff --git a/docs/windows-images.md b/docs/windows-images.md index 0b73a147..607ce255 100644 --- a/docs/windows-images.md +++ b/docs/windows-images.md @@ -13,6 +13,7 @@ A machine image uses these OCI config labels: | `io.hypeman.machine-image.base` | omitted | digest-pinned base reference | | `io.hypeman.machine-image.tpm` | `2.0` | `2.0` | | `io.hypeman.machine-image.secure-boot` | `required` | `required` | +| `io.hypeman.machine-image.bitlocker` | omitted | `disabled` for forkable images; `reseal-required` otherwise | The base must be pulled before its dependent Windows images. A base cannot be deleted while any cached image references its digest. Instance references are not tracked by the image cache, matching existing Linux behavior: do not delete a base while a dependent Windows instance exists. diff --git a/docs/windows-snapshots.md b/docs/windows-snapshots.md new file mode 100644 index 00000000..5acd6dc7 --- /dev/null +++ b/docs/windows-snapshots.md @@ -0,0 +1,23 @@ +# Windows snapshots and forks + +Windows 11 QEMU instances support standby, restore, stopped snapshots, and forks. Snapshot payloads treat the writable qcow2 disk, Secure Boot NVRAM, software TPM state, saved QEMU configuration, and memory image as one machine. + +## Same-instance standby and restore + +Standby pauses QEMU, captures memory and device state, stops QEMU and swtpm, and retains the instance disk, NVRAM, and TPM directory. Restore starts swtpm from that same state before loading QEMU memory. The Windows machine identity and TPM remain unchanged. + +## Fork identity + +A fork receives independent disk and NVRAM files. A stopped fork removes the copied TPM state before cold boot, so swtpm initializes a new endorsement key and TPM identity. A memory fork retains the parent's TPM identity because QEMU includes the TPM's permanent and volatile state in its migration stream. Workloads that depend on unique TPM attestation must use stopped forks. + +The Windows guest agent writes a new `MachineGuid` and records the child instance ID before the child is returned. Memory forks retain the source SID and hostname, and services that cached `MachineGuid` before standby may observe the previous value until the next cold boot. + +Fork admission requires the image OCI label: + +```text +io.hypeman.machine-image.bitlocker=disabled +``` + +Images marked `reseal-required`, unlabeled images, and unknown policies can still use same-instance snapshots, but cannot be forked. Hypeman does not expose a child whose encrypted disk was cloned without resealing it to the child's TPM. + +Stopped forks cold-boot with a unique vsock CID and can run concurrently. A standby snapshot contains the Windows VioSock driver's current CID in guest memory, so a memory-restored child initially retains that CID. The source and child must not be restored concurrently until the child has been stopped and cold-started; Hypeman reports a state error instead of allowing QEMU to fail with a CID collision. Creating a running fork directly from a running Windows source therefore requires `target_state=Stopped`. diff --git a/lib/guest/client.go b/lib/guest/client.go index cfdee83b..f01c3032 100644 --- a/lib/guest/client.go +++ b/lib/guest/client.go @@ -942,6 +942,32 @@ func CopyFromInstance(ctx context.Context, dialer hypervisor.VsockDialer, opts C return nil } +func RebindInstanceIdentity(ctx context.Context, dialer hypervisor.VsockDialer, instanceID string, waitForAgent time.Duration) (string, error) { + deadline := time.Now().Add(waitForAgent) + for { + conn, err := GetOrCreateConn(ctx, dialer) + if err == nil { + attemptCtx, cancel := context.WithTimeout(ctx, 2*time.Second) + resp, rpcErr := NewGuestServiceClient(conn).RebindIdentity(attemptCtx, &RebindIdentityRequest{InstanceId: instanceID}) + cancel() + if rpcErr == nil { + return resp.MachineId, nil + } + err = fmt.Errorf("rebind guest identity: %w", rpcErr) + } + retryable := isRetryableConnectionError(err) || status.Code(err) == codes.DeadlineExceeded + if !retryable || waitForAgent == 0 || time.Now().After(deadline) { + return "", err + } + CloseConn(dialer.Key()) + select { + case <-ctx.Done(): + return "", ctx.Err() + case <-time.After(guestExecSlowRetryInterval): + } + } +} + // ShutdownInstance sends a shutdown signal to the guest VM's init process (PID 1). // The guest-agent forwards the signal to init, which forwards it to the entrypoint. // sig is the signal number to send (0 = SIGTERM default). diff --git a/lib/guest/guest.pb.go b/lib/guest/guest.pb.go index d520602e..7a8fc154 100644 --- a/lib/guest/guest.pb.go +++ b/lib/guest/guest.pb.go @@ -1442,6 +1442,94 @@ func (*ReconfigureNetworkResponse) Descriptor() ([]byte, []int) { return file_lib_guest_guest_proto_rawDescGZIP(), []int{18} } +type RebindIdentityRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + InstanceId string `protobuf:"bytes,1,opt,name=instance_id,json=instanceId,proto3" json:"instance_id,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *RebindIdentityRequest) Reset() { + *x = RebindIdentityRequest{} + mi := &file_lib_guest_guest_proto_msgTypes[19] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *RebindIdentityRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*RebindIdentityRequest) ProtoMessage() {} + +func (x *RebindIdentityRequest) ProtoReflect() protoreflect.Message { + mi := &file_lib_guest_guest_proto_msgTypes[19] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use RebindIdentityRequest.ProtoReflect.Descriptor instead. +func (*RebindIdentityRequest) Descriptor() ([]byte, []int) { + return file_lib_guest_guest_proto_rawDescGZIP(), []int{19} +} + +func (x *RebindIdentityRequest) GetInstanceId() string { + if x != nil { + return x.InstanceId + } + return "" +} + +type RebindIdentityResponse struct { + state protoimpl.MessageState `protogen:"open.v1"` + MachineId string `protobuf:"bytes,1,opt,name=machine_id,json=machineId,proto3" json:"machine_id,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *RebindIdentityResponse) Reset() { + *x = RebindIdentityResponse{} + mi := &file_lib_guest_guest_proto_msgTypes[20] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *RebindIdentityResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*RebindIdentityResponse) ProtoMessage() {} + +func (x *RebindIdentityResponse) ProtoReflect() protoreflect.Message { + mi := &file_lib_guest_guest_proto_msgTypes[20] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use RebindIdentityResponse.ProtoReflect.Descriptor instead. +func (*RebindIdentityResponse) Descriptor() ([]byte, []int) { + return file_lib_guest_guest_proto_rawDescGZIP(), []int{20} +} + +func (x *RebindIdentityResponse) GetMachineId() string { + if x != nil { + return x.MachineId + } + return "" +} + var File_lib_guest_guest_proto protoreflect.FileDescriptor const file_lib_guest_guest_proto_rawDesc = "" + @@ -1544,17 +1632,24 @@ const file_lib_guest_guest_proto_rawDesc = "" + "\agateway\x18\x05 \x01(\tR\agateway\x12\x1f\n" + "\vdns_servers\x18\x06 \x03(\tR\n" + "dnsServers\"\x1c\n" + - "\x1aReconfigureNetworkResponse*@\n" + + "\x1aReconfigureNetworkResponse\"8\n" + + "\x15RebindIdentityRequest\x12\x1f\n" + + "\vinstance_id\x18\x01 \x01(\tR\n" + + "instanceId\"7\n" + + "\x16RebindIdentityResponse\x12\x1d\n" + + "\n" + + "machine_id\x18\x01 \x01(\tR\tmachineId*@\n" + "\vExecSession\x12\x17\n" + "\x13EXEC_SESSION_SYSTEM\x10\x00\x12\x18\n" + - "\x14EXEC_SESSION_DESKTOP\x10\x012\xae\x03\n" + + "\x14EXEC_SESSION_DESKTOP\x10\x012\xfd\x03\n" + "\fGuestService\x123\n" + "\x04Exec\x12\x12.guest.ExecRequest\x1a\x13.guest.ExecResponse(\x010\x01\x12F\n" + "\vCopyToGuest\x12\x19.guest.CopyToGuestRequest\x1a\x1a.guest.CopyToGuestResponse(\x01\x12L\n" + "\rCopyFromGuest\x12\x1b.guest.CopyFromGuestRequest\x1a\x1c.guest.CopyFromGuestResponse0\x01\x12;\n" + "\bStatPath\x12\x16.guest.StatPathRequest\x1a\x17.guest.StatPathResponse\x12;\n" + "\bShutdown\x12\x16.guest.ShutdownRequest\x1a\x17.guest.ShutdownResponse\x12Y\n" + - "\x12ReconfigureNetwork\x12 .guest.ReconfigureNetworkRequest\x1a!.guest.ReconfigureNetworkResponseB'Z%github.com/onkernel/hypeman/lib/guestb\x06proto3" + "\x12ReconfigureNetwork\x12 .guest.ReconfigureNetworkRequest\x1a!.guest.ReconfigureNetworkResponse\x12M\n" + + "\x0eRebindIdentity\x12\x1c.guest.RebindIdentityRequest\x1a\x1d.guest.RebindIdentityResponseB'Z%github.com/onkernel/hypeman/lib/guestb\x06proto3" var ( file_lib_guest_guest_proto_rawDescOnce sync.Once @@ -1569,7 +1664,7 @@ func file_lib_guest_guest_proto_rawDescGZIP() []byte { } var file_lib_guest_guest_proto_enumTypes = make([]protoimpl.EnumInfo, 1) -var file_lib_guest_guest_proto_msgTypes = make([]protoimpl.MessageInfo, 20) +var file_lib_guest_guest_proto_msgTypes = make([]protoimpl.MessageInfo, 22) var file_lib_guest_guest_proto_goTypes = []any{ (ExecSession)(0), // 0: guest.ExecSession (*ExecRequest)(nil), // 1: guest.ExecRequest @@ -1591,12 +1686,14 @@ var file_lib_guest_guest_proto_goTypes = []any{ (*ShutdownResponse)(nil), // 17: guest.ShutdownResponse (*ReconfigureNetworkRequest)(nil), // 18: guest.ReconfigureNetworkRequest (*ReconfigureNetworkResponse)(nil), // 19: guest.ReconfigureNetworkResponse - nil, // 20: guest.ExecStart.EnvEntry + (*RebindIdentityRequest)(nil), // 20: guest.RebindIdentityRequest + (*RebindIdentityResponse)(nil), // 21: guest.RebindIdentityResponse + nil, // 22: guest.ExecStart.EnvEntry } var file_lib_guest_guest_proto_depIdxs = []int32{ 2, // 0: guest.ExecRequest.start:type_name -> guest.ExecStart 3, // 1: guest.ExecRequest.resize:type_name -> guest.WindowSize - 20, // 2: guest.ExecStart.env:type_name -> guest.ExecStart.EnvEntry + 22, // 2: guest.ExecStart.env:type_name -> guest.ExecStart.EnvEntry 0, // 3: guest.ExecStart.session:type_name -> guest.ExecSession 6, // 4: guest.CopyToGuestRequest.start:type_name -> guest.CopyToGuestStart 7, // 5: guest.CopyToGuestRequest.end:type_name -> guest.CopyToGuestEnd @@ -1609,14 +1706,16 @@ var file_lib_guest_guest_proto_depIdxs = []int32{ 14, // 12: guest.GuestService.StatPath:input_type -> guest.StatPathRequest 16, // 13: guest.GuestService.Shutdown:input_type -> guest.ShutdownRequest 18, // 14: guest.GuestService.ReconfigureNetwork:input_type -> guest.ReconfigureNetworkRequest - 4, // 15: guest.GuestService.Exec:output_type -> guest.ExecResponse - 8, // 16: guest.GuestService.CopyToGuest:output_type -> guest.CopyToGuestResponse - 10, // 17: guest.GuestService.CopyFromGuest:output_type -> guest.CopyFromGuestResponse - 15, // 18: guest.GuestService.StatPath:output_type -> guest.StatPathResponse - 17, // 19: guest.GuestService.Shutdown:output_type -> guest.ShutdownResponse - 19, // 20: guest.GuestService.ReconfigureNetwork:output_type -> guest.ReconfigureNetworkResponse - 15, // [15:21] is the sub-list for method output_type - 9, // [9:15] is the sub-list for method input_type + 20, // 15: guest.GuestService.RebindIdentity:input_type -> guest.RebindIdentityRequest + 4, // 16: guest.GuestService.Exec:output_type -> guest.ExecResponse + 8, // 17: guest.GuestService.CopyToGuest:output_type -> guest.CopyToGuestResponse + 10, // 18: guest.GuestService.CopyFromGuest:output_type -> guest.CopyFromGuestResponse + 15, // 19: guest.GuestService.StatPath:output_type -> guest.StatPathResponse + 17, // 20: guest.GuestService.Shutdown:output_type -> guest.ShutdownResponse + 19, // 21: guest.GuestService.ReconfigureNetwork:output_type -> guest.ReconfigureNetworkResponse + 21, // 22: guest.GuestService.RebindIdentity:output_type -> guest.RebindIdentityResponse + 16, // [16:23] is the sub-list for method output_type + 9, // [9:16] is the sub-list for method input_type 9, // [9:9] is the sub-list for extension type_name 9, // [9:9] is the sub-list for extension extendee 0, // [0:9] is the sub-list for field type_name @@ -1654,7 +1753,7 @@ func file_lib_guest_guest_proto_init() { GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: unsafe.Slice(unsafe.StringData(file_lib_guest_guest_proto_rawDesc), len(file_lib_guest_guest_proto_rawDesc)), NumEnums: 1, - NumMessages: 20, + NumMessages: 22, NumExtensions: 0, NumServices: 1, }, diff --git a/lib/guest/guest.proto b/lib/guest/guest.proto index 205b4c47..62525993 100644 --- a/lib/guest/guest.proto +++ b/lib/guest/guest.proto @@ -23,6 +23,9 @@ service GuestService { // ReconfigureNetwork updates the guest network identity without spawning shell commands rpc ReconfigureNetwork(ReconfigureNetworkRequest) returns (ReconfigureNetworkResponse); + + // RebindIdentity assigns a forked guest a new machine identity. + rpc RebindIdentity(RebindIdentityRequest) returns (RebindIdentityResponse); } // ExecRequest represents messages from client to server @@ -176,3 +179,11 @@ message ReconfigureNetworkRequest { // ReconfigureNetworkResponse acknowledges the network reconfiguration request message ReconfigureNetworkResponse {} + +message RebindIdentityRequest { + string instance_id = 1; +} + +message RebindIdentityResponse { + string machine_id = 1; +} diff --git a/lib/guest/guest_grpc.pb.go b/lib/guest/guest_grpc.pb.go index acad4a3d..892b4334 100644 --- a/lib/guest/guest_grpc.pb.go +++ b/lib/guest/guest_grpc.pb.go @@ -25,6 +25,7 @@ const ( GuestService_StatPath_FullMethodName = "/guest.GuestService/StatPath" GuestService_Shutdown_FullMethodName = "/guest.GuestService/Shutdown" GuestService_ReconfigureNetwork_FullMethodName = "/guest.GuestService/ReconfigureNetwork" + GuestService_RebindIdentity_FullMethodName = "/guest.GuestService/RebindIdentity" ) // GuestServiceClient is the client API for GuestService service. @@ -45,6 +46,8 @@ type GuestServiceClient interface { Shutdown(ctx context.Context, in *ShutdownRequest, opts ...grpc.CallOption) (*ShutdownResponse, error) // ReconfigureNetwork updates the guest network identity without spawning shell commands ReconfigureNetwork(ctx context.Context, in *ReconfigureNetworkRequest, opts ...grpc.CallOption) (*ReconfigureNetworkResponse, error) + // RebindIdentity assigns a forked guest a new machine identity. + RebindIdentity(ctx context.Context, in *RebindIdentityRequest, opts ...grpc.CallOption) (*RebindIdentityResponse, error) } type guestServiceClient struct { @@ -130,6 +133,16 @@ func (c *guestServiceClient) ReconfigureNetwork(ctx context.Context, in *Reconfi return out, nil } +func (c *guestServiceClient) RebindIdentity(ctx context.Context, in *RebindIdentityRequest, opts ...grpc.CallOption) (*RebindIdentityResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(RebindIdentityResponse) + err := c.cc.Invoke(ctx, GuestService_RebindIdentity_FullMethodName, in, out, cOpts...) + if err != nil { + return nil, err + } + return out, nil +} + // GuestServiceServer is the server API for GuestService service. // All implementations must embed UnimplementedGuestServiceServer // for forward compatibility. @@ -148,6 +161,8 @@ type GuestServiceServer interface { Shutdown(context.Context, *ShutdownRequest) (*ShutdownResponse, error) // ReconfigureNetwork updates the guest network identity without spawning shell commands ReconfigureNetwork(context.Context, *ReconfigureNetworkRequest) (*ReconfigureNetworkResponse, error) + // RebindIdentity assigns a forked guest a new machine identity. + RebindIdentity(context.Context, *RebindIdentityRequest) (*RebindIdentityResponse, error) mustEmbedUnimplementedGuestServiceServer() } @@ -176,6 +191,9 @@ func (UnimplementedGuestServiceServer) Shutdown(context.Context, *ShutdownReques func (UnimplementedGuestServiceServer) ReconfigureNetwork(context.Context, *ReconfigureNetworkRequest) (*ReconfigureNetworkResponse, error) { return nil, status.Error(codes.Unimplemented, "method ReconfigureNetwork not implemented") } +func (UnimplementedGuestServiceServer) RebindIdentity(context.Context, *RebindIdentityRequest) (*RebindIdentityResponse, error) { + return nil, status.Error(codes.Unimplemented, "method RebindIdentity not implemented") +} func (UnimplementedGuestServiceServer) mustEmbedUnimplementedGuestServiceServer() {} func (UnimplementedGuestServiceServer) testEmbeddedByValue() {} @@ -276,6 +294,24 @@ func _GuestService_ReconfigureNetwork_Handler(srv interface{}, ctx context.Conte return interceptor(ctx, in, info, handler) } +func _GuestService_RebindIdentity_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(RebindIdentityRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(GuestServiceServer).RebindIdentity(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: GuestService_RebindIdentity_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(GuestServiceServer).RebindIdentity(ctx, req.(*RebindIdentityRequest)) + } + return interceptor(ctx, in, info, handler) +} + // GuestService_ServiceDesc is the grpc.ServiceDesc for GuestService service. // It's only intended for direct use with grpc.RegisterService, // and not to be introspected or modified (even as a copy) @@ -295,6 +331,10 @@ var GuestService_ServiceDesc = grpc.ServiceDesc{ MethodName: "ReconfigureNetwork", Handler: _GuestService_ReconfigureNetwork_Handler, }, + { + MethodName: "RebindIdentity", + Handler: _GuestService_RebindIdentity_Handler, + }, }, Streams: []grpc.StreamDesc{ { diff --git a/lib/images/machine.go b/lib/images/machine.go index 66fccd36..8fcab1e8 100644 --- a/lib/images/machine.go +++ b/lib/images/machine.go @@ -20,6 +20,7 @@ const ( MachineImageBaseLabel = "io.hypeman.machine-image.base" MachineImageTPMLabel = "io.hypeman.machine-image.tpm" MachineImageSecureBootLabel = "io.hypeman.machine-image.secure-boot" + MachineImageBitLockerLabel = "io.hypeman.machine-image.bitlocker" MachineImageVersion = "1" ) @@ -40,6 +41,7 @@ type MachineImage struct { Base string `json:"base,omitempty"` TPM string `json:"tpm"` SecureBoot string `json:"secure_boot"` + BitLocker string `json:"bitlocker,omitempty"` VirtualSize int64 `json:"virtual_size"` } @@ -65,6 +67,7 @@ func parseMachineImage(meta *containerMetadata) (*MachineImage, error) { Base: strings.TrimSpace(meta.Labels[MachineImageBaseLabel]), TPM: strings.TrimSpace(meta.Labels[MachineImageTPMLabel]), SecureBoot: strings.TrimSpace(meta.Labels[MachineImageSecureBootLabel]), + BitLocker: strings.TrimSpace(meta.Labels[MachineImageBitLockerLabel]), } if machine.DiskPath == "" || filepath.IsAbs(machine.DiskPath) || !filepath.IsLocal(machine.DiskPath) { return nil, fmt.Errorf("machine image disk path must be a local relative path") @@ -87,6 +90,9 @@ func parseMachineImage(meta *containerMetadata) (*MachineImage, error) { return nil, fmt.Errorf("Windows base image cannot reference another base") } case MachineImageWindowsImage: + if machine.BitLocker != "" && machine.BitLocker != "disabled" && machine.BitLocker != "reseal-required" { + return nil, fmt.Errorf("unsupported Windows image BitLocker policy %q", machine.BitLocker) + } if machine.DiskFormat != "qcow2" { return nil, fmt.Errorf("Windows image disk format must be qcow2") } diff --git a/lib/images/machine_test.go b/lib/images/machine_test.go index c01495be..811826a2 100644 --- a/lib/images/machine_test.go +++ b/lib/images/machine_test.go @@ -28,19 +28,19 @@ func windowsMachineMetadata(kind MachineImageKind, diskPath, base string) *conta if kind == MachineImageWindowsImage { format = "qcow2" } - return &containerMetadata{ - OS: "windows", - Architecture: "amd64", - Labels: map[string]string{ - MachineImageVersionLabel: MachineImageVersion, - MachineImageKindLabel: string(kind), - MachineImageDiskPathLabel: diskPath, - MachineImageDiskFormatLabel: format, - MachineImageBaseLabel: base, - MachineImageTPMLabel: "2.0", - MachineImageSecureBootLabel: "required", - }, + labels := map[string]string{ + MachineImageVersionLabel: MachineImageVersion, + MachineImageKindLabel: string(kind), + MachineImageDiskPathLabel: diskPath, + MachineImageDiskFormatLabel: format, + MachineImageBaseLabel: base, + MachineImageTPMLabel: "2.0", + MachineImageSecureBootLabel: "required", } + if kind == MachineImageWindowsImage { + labels[MachineImageBitLockerLabel] = "disabled" + } + return &containerMetadata{OS: "windows", Architecture: "amd64", Labels: labels} } func TestParseMachineImage(t *testing.T) { @@ -55,6 +55,16 @@ func TestParseMachineImage(t *testing.T) { )) require.NoError(t, err) assert.Equal(t, MachineImageWindowsImage, image.Kind) + assert.Equal(t, "disabled", image.BitLocker) + + invalidBitLocker := windowsMachineMetadata( + MachineImageWindowsImage, + "hypeman/disk.qcow2", + "registry.example/base@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + ) + invalidBitLocker.Labels[MachineImageBitLockerLabel] = "unknown" + _, err = parseMachineImage(invalidBitLocker) + assert.ErrorContains(t, err, "unsupported Windows image BitLocker policy") _, err = parseMachineImage(&containerMetadata{OS: "windows", Architecture: "amd64", Labels: map[string]string{}}) assert.ErrorContains(t, err, "ordinary Windows container images are not bootable") diff --git a/lib/instances/README.md b/lib/instances/README.md index 36278a00..851bd41b 100644 --- a/lib/instances/README.md +++ b/lib/instances/README.md @@ -36,6 +36,14 @@ Windows uses the same host-side TAP allocation as Linux. Once the guest agent is Create treats network configuration as part of readiness and tears down a VM if it fails. Start reapplies the current allocation because a stopped instance may receive a different address or MAC before its next boot. +### Windows snapshots and forks + +A Windows machine consists of its writable qcow2 disk, Secure Boot NVRAM, software TPM state, saved QEMU configuration, and—while in standby—memory and device state. Same-instance restore keeps these components together so Windows identity and TPM state remain stable. + +Forking separates identity according to the boot path. A stopped fork receives independent disk and NVRAM files, clears copied TPM state, cold-boots with a new VioSock CID, and asks the guest agent to assign a new `MachineGuid`. A memory fork must retain the captured TPM and VioSock state from QEMU's migration stream; it receives independent files and a new `MachineGuid`, but inherits the TPM endorsement key until its next cold boot. The source and memory-restored child cannot run concurrently with the same captured CID. + +Fork admission requires the image to declare `io.hypeman.machine-image.bitlocker=disabled`. Other policies remain valid for same-instance snapshots but are rejected for forks because Hypeman does not reseal BitLocker keys to a child TPM. + ### Why Config Disk? (configdisk.go) **What:** Read-only erofs disk with instance configuration diff --git a/lib/instances/create.go b/lib/instances/create.go index 109ea461..070186ac 100644 --- a/lib/instances/create.go +++ b/lib/instances/create.go @@ -392,6 +392,9 @@ func (m *manager) createInstance( HealthCheck: cloneHealthCheckPolicy(req.HealthCheck), RestartPolicy: cloneRestartPolicy(req.RestartPolicy), } + if windows { + stored.WindowsBitLockerPolicy = imageInfo.Machine.BitLocker + } // 12. Ensure directories log.DebugContext(ctx, "creating instance directories", "instance_id", id) diff --git a/lib/instances/fork.go b/lib/instances/fork.go index 260b123e..bdbb0c29 100644 --- a/lib/instances/fork.go +++ b/lib/instances/fork.go @@ -42,14 +42,14 @@ func (m *manager) forkInstance(ctx context.Context, id string, req ForkInstanceR if err != nil { return nil, "", false, err } - if err := rejectWindowsSnapshotLifecycle(meta.Platform, "fork"); err != nil { - return nil, "", false, err - } source := m.toInstance(ctx, meta) targetState, err := resolveForkTargetState(req.TargetState, source.State) if err != nil { return nil, "", false, err } + if isWindowsPlatform(source.Platform) && source.State == StateRunning && targetState != StateStopped { + return nil, "", false, fmt.Errorf("%w: Windows forks from a running source require target_state=%s", ErrNotSupported, StateStopped) + } switch source.State { case StateRunning: @@ -141,9 +141,13 @@ func ensureGuestAgentReadyForForkPhase(ctx context.Context, inst *StoredMetadata return fmt.Errorf("create vsock dialer for %s readiness check: %w", phase, err) } + command := []string{"true"} + if isWindowsPlatform(inst.Platform) { + command = []string{"cmd.exe", "/d", "/c", "exit", "0"} + } var stdout, stderr bytes.Buffer exit, err := guest.ExecIntoInstance(ctx, dialer, guest.ExecOptions{ - Command: []string{"true"}, + Command: command, Stdout: &stdout, Stderr: &stderr, WaitForAgent: 120 * time.Second, @@ -215,6 +219,9 @@ func (m *manager) forkInstanceFromStoppedOrStandby(ctx context.Context, id strin source := m.toInstance(ctx, meta) stored := &meta.StoredMetadata + if err := validateWindowsForkPolicy(stored); err != nil { + return nil, false, err + } switch source.State { case StateStopped, StateStandby: @@ -308,16 +315,21 @@ func (m *manager) forkInstanceFromStoppedOrStandby(ctx context.Context, id strin forkMeta.Phases.Record(phasetracking.PhaseStopped, now) } - // Keep the original CID for snapshot-based forks. Rewriting CID in restored - // memory snapshots is not reliable across hypervisors. Concurrent standby - // fork prepare is currently enabled only for Firecracker, whose host vsock - // dialer routes through the per-VM UDS path rather than this metadata CID. + // Keep the original CID for snapshot-based forks. Windows' restored VioSock + // driver retains the CID captured in guest memory until the next cold boot. if source.State == StateStandby { forkMeta.VsockCID = stored.VsockCID } else { forkMeta.VsockCID = generateVsockCID(forkID) } + // QEMU memory snapshots contain the TPM's permanent and volatile state. + // Resetting the copied directory is therefore only meaningful for cold forks. + resetWindowsTPM := source.State != StateStandby + if err := m.prepareWindowsForkIdentity(&forkMeta, resetWindowsTPM); err != nil { + return nil, false, err + } + if forkMeta.NetworkEnabled { // Clear inherited network identity. For stopped instances this is regenerated on start, // and for standby instances restore allocates if identity is empty. diff --git a/lib/instances/restore.go b/lib/instances/restore.go index 1d6c9169..14c25e20 100644 --- a/lib/instances/restore.go +++ b/lib/instances/restore.go @@ -43,9 +43,6 @@ func (m *manager) restoreInstance( return nil, err } - if err := rejectWindowsSnapshotLifecycle(meta.Platform, "restore from standby"); err != nil { - return nil, err - } inst := m.toInstance(ctx, meta) stored := &meta.StoredMetadata ctx = enrichInstancesTrace(ctx, attribute.String("hypervisor", string(stored.HypervisorType))) @@ -61,6 +58,9 @@ func (m *manager) restoreInstance( log.ErrorContext(ctx, "no snapshot available", "instance_id", id) return nil, fmt.Errorf("no snapshot available for instance %s", id) } + if err := m.ensureWindowsVsockCIDAvailable(ctx, stored); err != nil { + return nil, err + } // 2a. Validate the instance's image (rootfs) still exists before reserving // resources or invoking the hypervisor shim. A deleted image otherwise fails @@ -361,6 +361,14 @@ func (m *manager) restoreInstance( } reconfigureSpanEnd(nil) } + if stored.WindowsIdentityPending { + if err := rebindWindowsIdentity(ctx, stored); err != nil { + _ = hv.Shutdown(ctx) + m.rollbackAdmissionAllocationActive(stored) + releaseNetwork() + return nil, fmt.Errorf("rebind Windows fork identity: %w", err) + } + } releaseRestoreSlotOnce() // 8. Delete snapshot after successful restore unless the hypervisor is keeping it diff --git a/lib/instances/snapshot.go b/lib/instances/snapshot.go index 41709f7b..e00c04a5 100644 --- a/lib/instances/snapshot.go +++ b/lib/instances/snapshot.go @@ -63,9 +63,6 @@ func (m *manager) createSnapshot(ctx context.Context, id string, req CreateSnaps if err != nil { return nil, err } - if err := rejectWindowsSnapshotLifecycle(meta.Platform, "snapshot creation"); err != nil { - return nil, err - } inst := m.toInstance(ctx, meta) stored := &meta.StoredMetadata @@ -254,9 +251,6 @@ func (m *manager) restoreSnapshot(ctx context.Context, id string, snapshotID str if err != nil { return nil, err } - if err := rejectWindowsSnapshotLifecycle(rec.StoredMetadata.Platform, "snapshot restore"); err != nil { - return nil, err - } if rec.Snapshot.SourceInstanceID != id { return nil, fmt.Errorf("%w: snapshot %s belongs to instance %s", ErrInvalidRequest, snapshotID, rec.Snapshot.SourceInstanceID) } @@ -377,12 +371,12 @@ func (m *manager) forkSnapshot(ctx context.Context, snapshotID string, req ForkS if err != nil { return nil, err } - if err := rejectWindowsSnapshotLifecycle(rec.StoredMetadata.Platform, "snapshot fork"); err != nil { - return nil, err - } if err := validateForkVolumeSafety(rec.StoredMetadata.Volumes); err != nil { return nil, fmt.Errorf("%w: snapshot requires readonly volume attachments: %v", ErrNotSupported, err) } + if err := validateWindowsForkPolicy(&rec.StoredMetadata); err != nil { + return nil, err + } if err := m.ensureInstanceNameAvailableForSnapshotFork(ctx, req.Name, rec.StoredMetadata.NetworkEnabled); err != nil { return nil, err @@ -392,6 +386,17 @@ func (m *manager) forkSnapshot(ctx context.Context, snapshotID string, req ForkS if err != nil { return nil, err } + if isWindowsPlatform(rec.StoredMetadata.Platform) && rec.Snapshot.Kind == SnapshotKindStandby && targetState == StateRunning { + sourceMeta, sourceErr := m.loadMetadata(rec.Snapshot.SourceInstanceID) + if sourceErr == nil { + sourceState := m.toInstance(ctx, sourceMeta).State + if sourceState == StateRunning || sourceState == StateInitializing { + return nil, fmt.Errorf("%w: stop or standby the Windows snapshot source before restoring a running fork", ErrNotSupported) + } + } else if !errors.Is(sourceErr, ErrNotFound) { + return nil, sourceErr + } + } targetHypervisor, err := m.resolveSnapshotTargetHypervisor(rec, req.TargetHypervisor) if err != nil { return nil, err @@ -453,6 +458,12 @@ func (m *manager) forkSnapshot(ctx context.Context, snapshotID string, req ForkS } else { forkMeta.VsockCID = generateVsockCID(forkID) } + // QEMU memory snapshots contain the TPM's permanent and volatile state. + // Resetting the copied directory is therefore only meaningful for cold forks. + resetWindowsTPM := rec.Snapshot.Kind != SnapshotKindStandby + if err := m.prepareWindowsForkIdentity(&forkMeta, resetWindowsTPM); err != nil { + return nil, err + } if forkMeta.NetworkEnabled { forkMeta.IP = "" forkMeta.MAC = "" diff --git a/lib/instances/standby.go b/lib/instances/standby.go index ae2a728b..6913a989 100644 --- a/lib/instances/standby.go +++ b/lib/instances/standby.go @@ -44,9 +44,6 @@ func (m *manager) standbyInstance( return nil, err } - if err := rejectWindowsSnapshotLifecycle(meta.Platform, "standby"); err != nil { - return nil, err - } inst := m.toInstance(ctx, meta) stored := &meta.StoredMetadata ctx = enrichInstancesTrace(ctx, attribute.String("hypervisor", string(stored.HypervisorType))) diff --git a/lib/instances/start.go b/lib/instances/start.go index 4c07a804..fd4a7def 100644 --- a/lib/instances/start.go +++ b/lib/instances/start.go @@ -54,6 +54,9 @@ func (m *manager) startInstance( stored.ExitMessage = "" stored.ProgramStartedAt = nil stored.GuestAgentReadyAt = nil + if isWindowsPlatform(stored.Platform) { + stored.VsockCID = generateVsockCID(fmt.Sprintf("%s:%d", stored.Id, time.Now().UnixNano())) + } if len(req.Entrypoint) > 0 { stored.Entrypoint = req.Entrypoint } @@ -236,6 +239,17 @@ func (m *manager) startInstance( } networkSpanEnd(nil) } + if stored.WindowsIdentityPending { + if err := rebindWindowsIdentity(ctx, stored); err != nil { + _, _ = m.stopInstance(ctx, id) + return nil, fmt.Errorf("rebind Windows fork identity: %w", err) + } + meta = &metadata{StoredMetadata: *stored} + if err := m.saveMetadata(meta); err != nil { + _, _ = m.stopInstance(ctx, id) + return nil, fmt.Errorf("save rebound Windows identity: %w", err) + } + } // Return instance state from current metadata without forcing a log scan. finalInst := m.toInstanceWithoutHydration(ctx, meta) diff --git a/lib/instances/types.go b/lib/instances/types.go index df3ab7ff..6004c1a9 100644 --- a/lib/instances/types.go +++ b/lib/instances/types.go @@ -92,6 +92,11 @@ type StoredMetadata struct { // Read-only; echoed on the instance API. Platform string + // Windows machine identity and fork policy. These fields are internal and + // remain empty for Linux instances. + WindowsBitLockerPolicy string + WindowsIdentityPending bool + // Resources (matching Cloud Hypervisor terminology) Size int64 // Base memory in bytes HotplugSize int64 // Hotplug memory in bytes diff --git a/lib/instances/windows.go b/lib/instances/windows.go index 3a767a69..1f6c54a8 100644 --- a/lib/instances/windows.go +++ b/lib/instances/windows.go @@ -1,11 +1,14 @@ package instances import ( + "context" "fmt" "os" "strings" + "time" "github.com/kernel/hypeman/lib/forkvm" + "github.com/kernel/hypeman/lib/guest" "github.com/kernel/hypeman/lib/hypervisor" "github.com/kernel/hypeman/lib/images" "github.com/kernel/hypeman/lib/network" @@ -49,16 +52,75 @@ func validateWindowsCreate(req CreateInstanceRequest, image *images.Image, caps if req.NetworkEgress != nil || len(req.Credentials) != 0 { return fmt.Errorf("%w: Windows instances do not yet support managed egress or credentials", ErrInvalidRequest) } - if req.SnapshotPolicy != nil || req.AutoStandby != nil { - return fmt.Errorf("%w: Windows snapshot policies are added in the snapshots phase", ErrInvalidRequest) + return nil +} + +func validateWindowsForkPolicy(stored *StoredMetadata) error { + if stored != nil && isWindowsPlatform(stored.Platform) && stored.WindowsBitLockerPolicy != "disabled" { + return fmt.Errorf("%w: Windows forks require an image declared with %s=disabled", ErrNotSupported, images.MachineImageBitLockerLabel) } return nil } -func rejectWindowsSnapshotLifecycle(platform, operation string) error { - if isWindowsPlatform(platform) { - return fmt.Errorf("%w: %s is not supported for Windows until the snapshots phase", ErrNotSupported, operation) +func (m *manager) ensureWindowsVsockCIDAvailable(ctx context.Context, stored *StoredMetadata) error { + if stored == nil || !isWindowsPlatform(stored.Platform) { + return nil + } + instances, err := m.listInstances(ctx) + if err != nil { + return err + } + for _, instance := range instances { + if instance.Id == stored.Id || instance.VsockCID != stored.VsockCID { + continue + } + if instance.State == StateRunning || instance.State == StateInitializing { + return fmt.Errorf("%w: Windows snapshot restore requires instance %s with the same captured vsock CID to be stopped", ErrInvalidState, instance.Id) + } + } + return nil +} + +func (m *manager) prepareWindowsForkIdentity(stored *StoredMetadata, resetTPM bool) error { + if stored == nil || !isWindowsPlatform(stored.Platform) { + return nil + } + if err := validateWindowsForkPolicy(stored); err != nil { + return err + } + if resetTPM { + if err := os.RemoveAll(m.paths.InstanceTPMDir(stored.Id)); err != nil { + return fmt.Errorf("clear forked Windows TPM state: %w", err) + } + if err := os.MkdirAll(m.paths.InstanceTPMDir(stored.Id), 0700); err != nil { + return fmt.Errorf("create forked Windows TPM state: %w", err) + } + if err := os.Remove(m.paths.InstanceTPMSocket(stored.Id)); err != nil && !os.IsNotExist(err) { + return fmt.Errorf("remove forked Windows TPM socket: %w", err) + } + } + stored.WindowsIdentityPending = true + return nil +} + +func rebindWindowsIdentity(ctx context.Context, stored *StoredMetadata) error { + if stored == nil || !isWindowsPlatform(stored.Platform) || !stored.WindowsIdentityPending { + return nil + } + dialer, err := hypervisor.NewVsockDialer(stored.HypervisorType, stored.VsockSocket, stored.VsockCID) + if err != nil { + return fmt.Errorf("create Windows identity dialer: %w", err) + } + rebindCtx, cancel := context.WithTimeout(ctx, 120*time.Second) + defer cancel() + machineID, err := guest.RebindInstanceIdentity(rebindCtx, dialer, stored.Id, 120*time.Second) + if err != nil { + return err + } + if machineID == "" { + return fmt.Errorf("Windows guest agent returned an empty machine identity") } + stored.WindowsIdentityPending = false return nil } diff --git a/lib/instances/windows_networking_integration_linux_test.go b/lib/instances/windows_lifecycle_integration_linux_test.go similarity index 50% rename from lib/instances/windows_networking_integration_linux_test.go rename to lib/instances/windows_lifecycle_integration_linux_test.go index 0bf2c6f6..d1762016 100644 --- a/lib/instances/windows_networking_integration_linux_test.go +++ b/lib/instances/windows_lifecycle_integration_linux_test.go @@ -24,8 +24,79 @@ import ( ) func TestWindowsLifecycleIntegration(t *testing.T) { + manager, p, image := setupWindowsLifecycleIntegration(t) + ctx := context.Background() + source := createWindowsLifecycleInstance(t, ctx, manager, image, "windows-lifecycle-source", true) + + assertWindowsGuestControl(t, ctx, manager, source.Id) + assertWindowsNetworkReady(t, ctx, manager, source.Id, source.IP) + sourceMachineID := windowsMachineID(t, ctx, manager, source.Id) + sourceTPMEK := windowsTPMEK(t, ctx, manager, source.Id) + windowsPowerShell(t, ctx, manager, source.Id, `New-Item -ItemType Directory -Force C:\ProgramData\Hypeman | Out-Null; Set-Content C:\ProgramData\Hypeman\standby.txt inherited`) + + standby, err := manager.StandbyInstance(ctx, source.Id, StandbyInstanceRequest{}) + require.NoError(t, err) + require.Equal(t, StateStandby, standby.State) + forked, err := manager.ForkInstance(ctx, source.Id, ForkInstanceRequest{ + Name: "windows-standby-child", + TargetState: StateRunning, + }) + require.NoError(t, err) + t.Cleanup(func() { _ = deleteTestInstanceNow(context.Background(), manager, forked.Id) }) + assert.Equal(t, source.VsockCID, forked.VsockCID, "memory-restored Windows forks retain the captured VioSock CID until cold boot") + assert.NotEqual(t, sourceMachineID, windowsMachineID(t, ctx, manager, forked.Id)) + assert.Equal(t, sourceTPMEK, windowsTPMEK(t, ctx, manager, forked.Id), "memory forks inherit TPM state from the QEMU migration stream") + assert.Equal(t, "inherited", windowsPowerShell(t, ctx, manager, forked.Id, `Get-Content C:\ProgramData\Hypeman\standby.txt`)) + assertIndependentFile(t, p.InstanceOVMFVars(source.Id), p.InstanceOVMFVars(forked.Id)) + assertIndependentFile(t, p.InstanceWindowsDisk(source.Id), p.InstanceWindowsDisk(forked.Id)) + + stoppedFork, err := manager.StopInstance(ctx, forked.Id) + require.NoError(t, err) + require.Equal(t, StateStopped, stoppedFork.State) + restored, err := manager.RestoreInstance(ctx, source.Id) + require.NoError(t, err) + waitForWindowsRunning(t, ctx, manager, restored.Id) + assert.Equal(t, sourceMachineID, windowsMachineID(t, ctx, manager, restored.Id), "same-instance restore must preserve Windows identity") + assertWindowsNetworkReady(t, ctx, manager, restored.Id, source.IP) +} + +func TestWindowsStoppedForkIntegration(t *testing.T) { + manager, p, image := setupWindowsLifecycleIntegration(t) + ctx := context.Background() + source := createWindowsLifecycleInstance(t, ctx, manager, image, "windows-stopped-fork-source", false) + sourceMachineID := windowsMachineID(t, ctx, manager, source.Id) + sourceTPMEK := windowsTPMEK(t, ctx, manager, source.Id) + windowsPowerShell(t, ctx, manager, source.Id, `New-Item -ItemType Directory -Force C:\ProgramData\Hypeman | Out-Null; Set-Content C:\ProgramData\Hypeman\identity.txt source`) + + stopped, err := manager.StopInstance(ctx, source.Id) + require.NoError(t, err) + require.Equal(t, StateStopped, stopped.State) + forked, err := manager.ForkInstance(ctx, source.Id, ForkInstanceRequest{ + Name: "windows-stopped-child", + TargetState: StateRunning, + }) + require.NoError(t, err) + t.Cleanup(func() { _ = deleteTestInstanceNow(context.Background(), manager, forked.Id) }) + assert.NotEqual(t, sourceMachineID, windowsMachineID(t, ctx, manager, forked.Id), "fork must receive a new Windows machine identity") + assert.NotEqual(t, sourceTPMEK, windowsTPMEK(t, ctx, manager, forked.Id), "cold forks must initialize a fresh TPM") + assert.NotEqual(t, source.VsockCID, forked.VsockCID, "forks need unique vsock CIDs") + assertIndependentFile(t, p.InstanceOVMFVars(source.Id), p.InstanceOVMFVars(forked.Id)) + assertIndependentFile(t, p.InstanceWindowsDisk(source.Id), p.InstanceWindowsDisk(forked.Id)) + require.DirExists(t, p.InstanceTPMDir(forked.Id)) + + sourceDiskBefore, err := os.Stat(p.InstanceWindowsDisk(source.Id)) + require.NoError(t, err) + windowsPowerShell(t, ctx, manager, forked.Id, `Set-Content C:\ProgramData\Hypeman\identity.txt child`) + assert.Equal(t, "child", windowsPowerShell(t, ctx, manager, forked.Id, `Get-Content C:\ProgramData\Hypeman\identity.txt`)) + sourceDiskAfter, err := os.Stat(p.InstanceWindowsDisk(source.Id)) + require.NoError(t, err) + assert.Equal(t, sourceDiskBefore.ModTime(), sourceDiskAfter.ModTime(), "child writes must not modify the stopped source disk") +} + +func setupWindowsLifecycleIntegration(t *testing.T) (*manager, *paths.Paths, *images.Image) { + t.Helper() if os.Getenv("HYPEMAN_RUN_WINDOWS_LIFECYCLE_INTEGRATION") != "1" { - t.Skip("run by the dedicated Windows networking CI gate") + t.Skip("run by the dedicated Windows lifecycle CI gates") } fixture := os.Getenv("HYPEMAN_WINDOWS_TEST_AGENT_IMAGE") if fixture == "" { @@ -33,17 +104,17 @@ func TestWindowsLifecycleIntegration(t *testing.T) { } if _, err := os.Stat(fixture); err != nil { if os.Getenv("CI") == "true" { - t.Fatalf("required Windows networking fixture is missing: %s", fixture) + t.Fatalf("required Windows lifecycle fixture is missing: %s", fixture) } - t.Skipf("Windows networking fixture is unavailable: %s", fixture) + t.Skipf("Windows lifecycle fixture is unavailable: %s", fixture) } acquireHeavyIO(t) manager, dataDir := setupTestManagerForQEMU(t) p := paths.New(dataDir) - const digestHex = "abababababababababababababababababababababababababababababababab" + const digestHex = "acacacacacacacacacacacacacacacacacacacacacacacacacacacacacacacac" image := &images.Image{ - Name: "registry.example/windows/image:networking-integration", + Name: "registry.example/windows/image:lifecycle-integration", Digest: "sha256:" + digestHex, Platform: "windows/amd64", Status: images.StatusReady, @@ -52,6 +123,7 @@ func TestWindowsLifecycleIntegration(t *testing.T) { Base: "registry.example/windows/base@sha256:cdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcd", TPM: "2.0", SecureBoot: "required", + BitLocker: "disabled", VirtualSize: 80 << 30, }, } @@ -60,28 +132,32 @@ func TestWindowsLifecycleIntegration(t *testing.T) { require.NoError(t, err) require.NoError(t, forkvm.CopyRegularFile(fixture, imagePath)) require.NoError(t, os.Chmod(imagePath, 0444)) + return manager, p, image +} - ctx := context.Background() +func createWindowsLifecycleInstance(t *testing.T, ctx context.Context, manager *manager, image *images.Image, name string, networkEnabled bool) *Instance { + t.Helper() instance, err := manager.CreateInstance(ctx, CreateInstanceRequest{ - Name: "windows-networking-integration", + Name: name, Image: image.Name, Platform: "windows/amd64", - Size: 8 << 30, + Size: 4 << 30, Vcpus: 4, - NetworkEnabled: true, + NetworkEnabled: networkEnabled, Hypervisor: hypervisor.TypeQEMU, }) require.NoError(t, err) t.Cleanup(func() { _ = deleteTestInstanceNow(context.Background(), manager, instance.Id) }) - require.NotEmpty(t, instance.IP) - require.NotEmpty(t, instance.MAC) + waitForWindowsRunning(t, ctx, manager, instance.Id) + return instance +} + +func waitForWindowsRunning(t *testing.T, ctx context.Context, manager *manager, instanceID string) { + t.Helper() require.Eventually(t, func() bool { - current, err := manager.GetInstance(ctx, instance.Id) + current, err := manager.GetInstance(ctx, instanceID) return err == nil && current.State == StateRunning - }, 4*time.Minute, time.Second) - - assertWindowsGuestControl(t, ctx, manager, instance.Id) - assertWindowsNetworkReady(t, ctx, manager, instance.Id, instance.IP) + }, 75*time.Second, 500*time.Millisecond) } func assertWindowsGuestControl(t *testing.T, ctx context.Context, manager *manager, instanceID string) { @@ -100,7 +176,7 @@ func assertWindowsGuestControl(t *testing.T, ctx context.Context, manager *manag require.NoError(t, err, stderr.String()) assert.Less(t, time.Since(jobStart), 10*time.Second) - time.Sleep(5 * time.Second) + time.Sleep(time.Second) stdout.Reset() stderr.Reset() exit, err = guest.ExecIntoInstance(ctx, dialer, guest.ExecOptions{ @@ -201,3 +277,41 @@ func assertWindowsNetworkReady(t *testing.T, ctx context.Context, manager *manag ping := exec.Command("ping", "-c", "3", "-W", "2", expectedIP) require.NoError(t, ping.Run(), "allocated Windows IP did not answer ICMP") } + +func windowsMachineID(t *testing.T, ctx context.Context, manager *manager, instanceID string) string { + t.Helper() + return windowsPowerShell(t, ctx, manager, instanceID, `(Get-ItemProperty 'HKLM:\SOFTWARE\Microsoft\Cryptography').MachineGuid`) +} + +func windowsTPMEK(t *testing.T, ctx context.Context, manager *manager, instanceID string) string { + t.Helper() + hash := windowsPowerShell(t, ctx, manager, instanceID, `(Get-TpmEndorsementKeyInfo -HashAlgorithm Sha256).PublicKeyHash`) + require.NotEmpty(t, hash, "TPM endorsement key hash") + return hash +} + +func windowsPowerShell(t *testing.T, ctx context.Context, manager *manager, instanceID, command string) string { + t.Helper() + dialer, err := manager.GetVsockDialer(ctx, instanceID) + require.NoError(t, err) + var stdout, stderr bytes.Buffer + exit, err := guest.ExecIntoInstance(ctx, dialer, guest.ExecOptions{ + Command: []string{"powershell.exe", "-NoProfile", "-NonInteractive", "-Command", command}, + Stdout: &stdout, + Stderr: &stderr, + WaitForAgent: 30 * time.Second, + Timeout: 30, + }) + require.NoError(t, err, stderr.String()) + require.Equal(t, 0, exit.Code, stderr.String()) + return string(bytes.TrimSpace(stdout.Bytes())) +} + +func assertIndependentFile(t *testing.T, source, fork string) { + t.Helper() + sourceInfo, err := os.Stat(source) + require.NoError(t, err) + forkInfo, err := os.Stat(fork) + require.NoError(t, err) + assert.False(t, os.SameFile(sourceInfo, forkInfo), "%s and %s must not share an inode", source, fork) +} diff --git a/lib/instances/windows_test.go b/lib/instances/windows_test.go index 87f71e7c..bcc5ef0f 100644 --- a/lib/instances/windows_test.go +++ b/lib/instances/windows_test.go @@ -2,6 +2,7 @@ package instances import ( "os" + "path/filepath" "testing" "github.com/kernel/hypeman/lib/autostandby" @@ -23,6 +24,7 @@ func windowsImageFixture() *images.Image { Base: "registry.example/windows/base@sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", TPM: "2.0", SecureBoot: "required", + BitLocker: "disabled", VirtualSize: 80 << 30, }, } @@ -33,6 +35,8 @@ func TestValidateWindowsCreate(t *testing.T) { windowsCaps := hypervisor.Capabilities{SupportsUEFIBoot: true, SupportsTPM: true} require.NoError(t, validateWindowsCreate(CreateInstanceRequest{}, image, windowsCaps)) require.NoError(t, validateWindowsCreate(CreateInstanceRequest{NetworkEnabled: true}, image, windowsCaps)) + require.NoError(t, validateWindowsCreate(CreateInstanceRequest{SnapshotPolicy: &SnapshotPolicy{}}, image, windowsCaps)) + require.NoError(t, validateWindowsCreate(CreateInstanceRequest{AutoStandby: &autostandby.Policy{}}, image, windowsCaps)) tests := []struct { name string @@ -43,8 +47,6 @@ func TestValidateWindowsCreate(t *testing.T) { {name: "small memory", caps: windowsCaps, req: CreateInstanceRequest{Size: 2 << 30}}, {name: "one CPU", caps: windowsCaps, req: CreateInstanceRequest{Vcpus: 1}}, {name: "command", caps: windowsCaps, req: CreateInstanceRequest{Cmd: []string{"cmd.exe"}}}, - {name: "snapshot policy", caps: windowsCaps, req: CreateInstanceRequest{SnapshotPolicy: &SnapshotPolicy{}}}, - {name: "auto standby", caps: windowsCaps, req: CreateInstanceRequest{AutoStandby: &autostandby.Policy{}}}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { @@ -53,9 +55,27 @@ func TestValidateWindowsCreate(t *testing.T) { } } -func TestRejectWindowsSnapshotLifecycle(t *testing.T) { - assert.ErrorIs(t, rejectWindowsSnapshotLifecycle("windows/amd64", "fork"), ErrNotSupported) - assert.NoError(t, rejectWindowsSnapshotLifecycle("linux/amd64", "fork")) +func TestPrepareWindowsForkIdentity(t *testing.T) { + p := paths.New(t.TempDir()) + m := &manager{paths: p} + stored := &StoredMetadata{Id: "fork", Platform: "windows/amd64", WindowsBitLockerPolicy: "disabled"} + require.NoError(t, os.MkdirAll(p.InstanceTPMDir(stored.Id), 0700)) + require.NoError(t, os.WriteFile(filepath.Join(p.InstanceTPMDir(stored.Id), "state"), []byte("source identity"), 0600)) + + require.NoError(t, m.prepareWindowsForkIdentity(stored, true)) + assert.True(t, stored.WindowsIdentityPending) + entries, err := os.ReadDir(p.InstanceTPMDir(stored.Id)) + require.NoError(t, err) + assert.Empty(t, entries) + + require.NoError(t, os.WriteFile(filepath.Join(p.InstanceTPMDir(stored.Id), "state"), []byte("memory identity"), 0600)) + require.NoError(t, m.prepareWindowsForkIdentity(stored, false)) + state, err := os.ReadFile(filepath.Join(p.InstanceTPMDir(stored.Id), "state")) + require.NoError(t, err) + assert.Equal(t, "memory identity", string(state)) + + stored.WindowsBitLockerPolicy = "" + assert.ErrorIs(t, m.prepareWindowsForkIdentity(stored, true), ErrNotSupported) } func TestBuildWindowsHypervisorConfig(t *testing.T) { diff --git a/lib/system/guest_agent/identity_windows.go b/lib/system/guest_agent/identity_windows.go new file mode 100644 index 00000000..131a33c8 --- /dev/null +++ b/lib/system/guest_agent/identity_windows.go @@ -0,0 +1,52 @@ +//go:build windows + +package main + +import ( + "context" + "crypto/rand" + "fmt" + + pb "github.com/kernel/hypeman/lib/guest" + "golang.org/x/sys/windows/registry" +) + +func (s *guestServer) RebindIdentity(_ context.Context, req *pb.RebindIdentityRequest) (*pb.RebindIdentityResponse, error) { + if req.InstanceId == "" { + return nil, fmt.Errorf("instance id is required") + } + + machineID, err := newWindowsMachineID() + if err != nil { + return nil, fmt.Errorf("generate Windows machine id: %w", err) + } + cryptography, err := registry.OpenKey(registry.LOCAL_MACHINE, `SOFTWARE\Microsoft\Cryptography`, registry.SET_VALUE) + if err != nil { + return nil, fmt.Errorf("open Windows machine identity: %w", err) + } + defer cryptography.Close() + if err := cryptography.SetStringValue("MachineGuid", machineID); err != nil { + return nil, fmt.Errorf("set Windows machine identity: %w", err) + } + + marker, _, err := registry.CreateKey(registry.LOCAL_MACHINE, `SOFTWARE\Kernel\Hypeman`, registry.SET_VALUE) + if err != nil { + return nil, fmt.Errorf("open Hypeman identity marker: %w", err) + } + defer marker.Close() + if err := marker.SetStringValue("InstanceID", req.InstanceId); err != nil { + return nil, fmt.Errorf("set Hypeman instance identity: %w", err) + } + return &pb.RebindIdentityResponse{MachineId: machineID}, nil +} + +func newWindowsMachineID() (string, error) { + var value [16]byte + if _, err := rand.Read(value[:]); err != nil { + return "", err + } + value[6] = (value[6] & 0x0f) | 0x40 + value[8] = (value[8] & 0x3f) | 0x80 + return fmt.Sprintf("%x-%x-%x-%x-%x", + value[0:4], value[4:6], value[6:8], value[8:10], value[10:16]), nil +}