Add kernel-paged differential snapshots with hotplug

.github/workflows/release.yaml

@@ -1,5 +1,11 @@
 name: Cloud Hypervisor Release
-on: [create, merge_group]
+on:
+  push:
+    tags:
+      - "v*-kernel.*"
+  merge_group:
+permissions:
+  contents: write
 concurrency:
   group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}-${{ github.event_name }}
   cancel-in-progress: true
@@ -8,7 +14,7 @@ env:
 
 jobs:
   release:
-    if: (github.event_name == 'create' && github.event.ref_type == 'tag') || github.event_name == 'merge_group'
+    if: github.event_name == 'push' || github.event_name == 'merge_group'
     name: Release ${{ matrix.platform.target }}
     strategy:
       fail-fast: false
@@ -35,9 +41,9 @@ jobs:
         run: sudo apt install -y musl-tools
       - name: Create release directory
         if: |
-          github.event_name == 'create' && github.event.ref_type == 'tag' &&
+          github.event_name == 'push' &&
           matrix.platform.target == 'x86_64-unknown-linux-gnu'
-        run: rsync -rv --exclude=.git . ../cloud-hypervisor-${{ github.event.ref }}
+        run: rsync -rv --exclude=.git . ../cloud-hypervisor-${{ github.ref_name }}
       - name: Build ${{ matrix.platform.target }}
         uses: houseabsolute/actions-rust-cross@v1
         with:
@@ -47,13 +53,13 @@ jobs:
           strip: true
           toolchain: "1.89.0"
       - name: Copy Release Binaries
-        if: github.event_name == 'create' && github.event.ref_type == 'tag'
+        if: github.event_name == 'push'
         shell: bash
         run: |
           cp target/${{ matrix.platform.target }}/release/cloud-hypervisor ./${{ matrix.platform.name_ch }}
           cp target/${{ matrix.platform.target }}/release/ch-remote ./${{ matrix.platform.name_ch_remote }}
       - name: Upload Release Artifacts
-        if: github.event_name == 'create' && github.event.ref_type == 'tag'
+        if: github.event_name == 'push'
         uses: actions/upload-artifact@v7
         with:
           name: Artifacts for ${{ matrix.platform.target }}
@@ -62,34 +68,34 @@ jobs:
             ./${{ matrix.platform.name_ch_remote }}
       - name: Vendor
         if: |
-          github.event_name == 'create' && github.event.ref_type == 'tag' &&
+          github.event_name == 'push' &&
           matrix.platform.target == 'x86_64-unknown-linux-gnu'
-        working-directory: ../cloud-hypervisor-${{ github.event.ref }}
+        working-directory: ../cloud-hypervisor-${{ github.ref_name }}
         run: |
           mkdir ../vendor-cargo-home
           export CARGO_HOME=$(realpath ../vendor-cargo-home)
           mkdir .cargo
           cargo vendor > .cargo/config.toml
       - name: Create vendored source archive
         if: |
-          github.event_name == 'create' && github.event.ref_type == 'tag' &&
+          github.event_name == 'push' &&
           matrix.platform.target == 'x86_64-unknown-linux-gnu'
-        run: tar cJf cloud-hypervisor-${{ github.event.ref }}.tar.xz ../cloud-hypervisor-${{ github.event.ref }}
+        run: tar cJf cloud-hypervisor-${{ github.ref_name }}.tar.xz ../cloud-hypervisor-${{ github.ref_name }}
       - name: Upload cloud-hypervisor vendored source archive
         if: |
-          github.event_name == 'create' && github.event.ref_type == 'tag' &&
+          github.event_name == 'push' &&
           matrix.platform.target == 'x86_64-unknown-linux-gnu'
         id: upload-release-cloud-hypervisor-vendored-sources
         uses: actions/upload-artifact@v7
         with:
-          path: cloud-hypervisor-${{ github.event.ref }}.tar.xz
-          name: cloud-hypervisor-${{ github.event.ref }}.tar.xz
+          path: cloud-hypervisor-${{ github.ref_name }}.tar.xz
+          name: cloud-hypervisor-${{ github.ref_name }}.tar.xz
       - name: Create GitHub Release
-        if: github.event_name == 'create' && github.event.ref_type == 'tag'
+        if: github.event_name == 'push'
         uses: softprops/action-gh-release@v3
         with:
           draft: true
           files: |
             ./${{ matrix.platform.name_ch }}
             ./${{ matrix.platform.name_ch_remote }}
-            ./cloud-hypervisor-${{ github.event.ref }}.tar.xz
+            ./cloud-hypervisor-${{ github.ref_name }}.tar.xz

cloud-hypervisor/src/bin/ch-remote.rs

@@ -925,6 +925,7 @@ fn add_vsock_config(config: &str) -> Result<String, Error> {
 fn snapshot_config(url: &str) -> String {
     let snapshot_config = api::VmSnapshotConfig {
         destination_url: String::from(url),
+        snapshot_type: Default::default(),
     };
 
     serde_json::to_string(&snapshot_config).unwrap()

fuzz/fuzz_targets/http_api.rs

@@ -15,7 +15,7 @@ use vm_migration::MigratableError;
 use vmm::api::http::*;
 use vmm::api::{
     ApiRequest, RequestHandler, VmInfoResponse, VmReceiveMigrationData, VmSendMigrationData,
-    VmmPingResponse,
+    VmSnapshotConfig, VmmPingResponse,
 };
 use vmm::config::RestoreConfig;
 use vmm::vm::{Error as VmError, VmState};
@@ -100,7 +100,7 @@ impl RequestHandler for StubApiRequestHandler {
         Ok(())
     }
 
-    fn vm_snapshot(&mut self, _: &str) -> Result<(), VmError> {
+    fn vm_snapshot(&mut self, _: &VmSnapshotConfig) -> Result<(), VmError> {
         Ok(())
     }
 

virtio-devices/src/mem.rs

@@ -810,9 +810,13 @@ impl Mem {
             (avail_features, 0, config, false)
         };
 
-        let host_fd = region
-            .file_offset()
-            .map(|f_offset| f_offset.file().as_raw_fd());
+        let host_fd = if region.flags() & libc::MAP_SHARED == libc::MAP_SHARED {
+            region
+                .file_offset()
+                .map(|f_offset| f_offset.file().as_raw_fd())
+        } else {
+            None
+        };
 
         Ok(Mem {
             common: VirtioCommon {

vmm/src/api/mod.rs

@@ -259,10 +259,24 @@ pub struct VmRemoveDeviceData {
     pub id: String,
 }
 
+/// Type of a VM snapshot: full memory dump or dirty-pages-only delta.
+#[derive(Copy, Clone, Default, Deserialize, Serialize, Debug, PartialEq, Eq)]
+#[serde(rename_all = "lowercase")]
+pub enum VmSnapshotType {
+    /// Complete guest memory dump.
+    #[default]
+    Full,
+    /// Only pages dirtied since the tracked baseline.
+    Diff,
+}
+
 #[derive(Clone, Deserialize, Serialize, Default, Debug)]
 pub struct VmSnapshotConfig {
     /// The snapshot destination URL
     pub destination_url: String,
+    /// Full dump or dirty-pages delta.
+    #[serde(default)]
+    pub snapshot_type: VmSnapshotType,
 }
 
 #[derive(Clone, Deserialize, Serialize, Default, Debug)]
@@ -689,7 +703,7 @@ pub trait RequestHandler {
 
     fn vm_resume(&mut self) -> Result<(), VmError>;
 
-    fn vm_snapshot(&mut self, destination_url: &str) -> Result<(), VmError>;
+    fn vm_snapshot(&mut self, config: &VmSnapshotConfig) -> Result<(), VmError>;
 
     fn vm_restore(&mut self, restore_cfg: RestoreConfig) -> Result<(), VmError>;
 
@@ -1810,7 +1824,7 @@ impl ApiAction for VmSnapshot {
             info!("API request event: VmSnapshot {config:?}");
 
             let response = vmm
-                .vm_snapshot(&config.destination_url)
+                .vm_snapshot(&config)
                 .map_err(ApiError::VmSnapshot)
                 .map(|_| ApiResponsePayload::Empty);
 

vmm/src/api/openapi/cloud-hypervisor.yaml

@@ -1475,6 +1475,10 @@ components:
       properties:
         destination_url:
           type: string
+        snapshot_type:
+          type: string
+          enum: [full, diff]
+          default: full
 
     VmCoredumpData:
       type: object
@@ -1498,6 +1502,9 @@ components:
           type: boolean
         memory_restore_mode:
           $ref: "#/components/schemas/MemoryRestoreMode"
+        track_dirty_pages:
+          type: boolean
+          default: false
         resume:
           type: boolean
 

vmm/src/config.rs

@@ -2809,6 +2809,10 @@ pub struct RestoreConfig {
     pub prefault: bool,
     #[serde(default)]
     pub memory_restore_mode: MemoryRestoreMode,
+    /// Enable dirty-page tracking after restore so a later diff snapshot can
+    /// use this snapshot as its baseline.
+    #[serde(default)]
+    pub track_dirty_pages: bool,
     #[serde(default)]
     pub net_fds: Option<Vec<RestoredNetConfig>>,
     #[serde(default)]
@@ -2818,10 +2822,11 @@ pub struct RestoreConfig {
 impl RestoreConfig {
     pub const SYNTAX: &'static str = "Restore from a VM snapshot. \
         \nRestore parameters \"source_url=<source_url>,prefault=on|off,memory_restore_mode=copy|ondemand,\
-        net_fds=<list_of_net_ids_with_their_associated_fds>,resume=true|false\" \
+        track_dirty_pages=on|off,net_fds=<list_of_net_ids_with_their_associated_fds>,resume=true|false\" \
         \n`source_url` should be a valid URL (e.g file:///foo/bar or tcp://192.168.1.10/foo) \
         \n`prefault` controls eager prefaulting for the copy-based restore path (disabled by default) \
         \n`memory_restore_mode=copy` preserves the existing eager read-copy restore behavior, while `memory_restore_mode=ondemand` enables lazy demand paging and fails restore if userfaultfd support is unavailable \
+        \n`track_dirty_pages` enables differential snapshots after restore (disabled by default) \
         \n`net_fds` is a list of net ids with new file descriptors. \
         Only net devices backed by FDs directly are needed as input.\
         \n `resume` controls whether the VM will be directly resumed after restore ";
@@ -2832,6 +2837,7 @@ impl RestoreConfig {
             .add("source_url")
             .add("prefault")
             .add("memory_restore_mode")
+            .add("track_dirty_pages")
             .add("net_fds")
             .add("resume");
         parser.parse(restore).map_err(Error::ParseRestore)?;
@@ -2849,6 +2855,11 @@ impl RestoreConfig {
             .convert::<MemoryRestoreMode>("memory_restore_mode")
             .map_err(Error::ParseRestore)?
             .unwrap_or_default();
+        let track_dirty_pages = parser
+            .convert::<Toggle>("track_dirty_pages")
+            .map_err(Error::ParseRestore)?
+            .unwrap_or(Toggle(false))
+            .0;
         let net_fds = parser
             .convert::<Tuple<String, Vec<u64>>>("net_fds")
             .map_err(Error::ParseRestore)?
@@ -2871,6 +2882,7 @@ impl RestoreConfig {
             source_url,
             prefault,
             memory_restore_mode,
+            track_dirty_pages,
             net_fds,
             resume,
         })
@@ -5089,18 +5101,20 @@ id=\"{id}\",pci_segment={pci_segment},queue_sizes={queue_sizes}"
                 source_url: PathBuf::from("/path/to/snapshot"),
                 prefault: false,
                 memory_restore_mode: MemoryRestoreMode::Copy,
+                track_dirty_pages: false,
                 net_fds: None,
                 resume: false,
             }
         );
         assert_eq!(
             RestoreConfig::parse(
-                "source_url=/path/to/snapshot,prefault=off,net_fds=[net0@[3,4],net1@[5,6,7,8]]"
+                "source_url=/path/to/snapshot,prefault=off,track_dirty_pages=on,net_fds=[net0@[3,4],net1@[5,6,7,8]]"
             )?,
             RestoreConfig {
                 source_url: PathBuf::from("/path/to/snapshot"),
                 prefault: false,
                 memory_restore_mode: MemoryRestoreMode::Copy,
+                track_dirty_pages: true,
                 net_fds: Some(vec![
                     RestoredNetConfig {
                         id: "net0".to_string(),
@@ -5122,6 +5136,7 @@ id=\"{id}\",pci_segment={pci_segment},queue_sizes={queue_sizes}"
                 source_url: PathBuf::from("/path/to/snapshot"),
                 prefault: false,
                 memory_restore_mode: MemoryRestoreMode::OnDemand,
+                track_dirty_pages: false,
                 net_fds: None,
                 resume: false,
             }
@@ -5132,6 +5147,7 @@ id=\"{id}\",pci_segment={pci_segment},queue_sizes={queue_sizes}"
                 source_url: PathBuf::from("/path/to/snapshot"),
                 prefault: false,
                 memory_restore_mode: MemoryRestoreMode::Copy,
+                track_dirty_pages: false,
                 net_fds: None,
                 resume: true,
             }
@@ -5233,6 +5249,7 @@ id=\"{id}\",pci_segment={pci_segment},queue_sizes={queue_sizes}"
             source_url: PathBuf::from("/path/to/snapshot"),
             prefault: false,
             memory_restore_mode: MemoryRestoreMode::Copy,
+            track_dirty_pages: false,
             net_fds: Some(vec![
                 RestoredNetConfig {
                     id: "net0".to_string(),
@@ -5309,6 +5326,7 @@ id=\"{id}\",pci_segment={pci_segment},queue_sizes={queue_sizes}"
             source_url: PathBuf::from("/path/to/snapshot"),
             prefault: false,
             memory_restore_mode: MemoryRestoreMode::Copy,
+            track_dirty_pages: false,
             net_fds: None,
             resume: false,
         };
@@ -5326,6 +5344,7 @@ id=\"{id}\",pci_segment={pci_segment},queue_sizes={queue_sizes}"
             source_url: PathBuf::from("/path/to/snapshot"),
             prefault: true,
             memory_restore_mode: MemoryRestoreMode::OnDemand,
+            track_dirty_pages: false,
             net_fds: None,
             resume: false,
         };