From 119dd7434dd36d8aaf39a8d57766551077e9c89d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kamil=20Chodo=C5=82a?= Date: Tue, 14 Jul 2026 20:33:18 +0200 Subject: [PATCH] feat(checkpoint-restore): add restore_in_place option MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Restore the checkpointed container in place between tests instead of importing the checkpoint archive into a fresh container. The checkpoint is taken locally (Keep, no Export) so the checkpoint data stays in the container's storage directory, and the same container — same name, mounts, and IP — is restored repeatedly, with a zero-timeout stop between tests. This skips the per-test archive extraction and container creation of the export/import path. On a ~7.9G Nethermind checkpoint the per-test podman restore drops from ~15s (of which CRIU page restore was only ~2.9s — the rest was extracting the archive into container storage and creating the container) to roughly the CRIU restore time. Repeated in-place restores are safe: podman accepts a restore of a stopped container whenever its kept checkpoint data is present, restores resume from the identical memory state each time, and the container IP is stable (verified on podman 5.x + CRIU 4.x: checkpoint, restore --keep, kill, restore --keep, ... loops indefinitely). tmpfs_threshold / tmpfs_max_size only apply to the export archive and are ignored in this mode. Default remains the export/import path (restore_in_place: false). --- config.example.yaml | 7 + docs/configuration.md | 1 + pkg/config/config.go | 17 +++ pkg/config/config_test.go | 52 +++++++ pkg/podman/checkpoint.go | 141 ++++++++++++++++++ pkg/runner/strategy_checkpoint.go | 131 ++++++++++++---- ui/src/api/types.ts | 1 + .../run-detail/RunConfiguration.tsx | 6 + 8 files changed, 323 insertions(+), 33 deletions(-) diff --git a/config.example.yaml b/config.example.yaml index 5bb0a766..da650aa0 100644 --- a/config.example.yaml +++ b/config.example.yaml @@ -442,6 +442,13 @@ runner: # # Whether to restart the container before taking a CRIU checkpoint. # # Restarting ensures a clean process state (cold caches, clean DB shutdown). # restart_container: false + # # Restore the checkpointed container in place between tests instead of + # # importing the checkpoint archive into a fresh container. Skips the + # # per-test archive extraction and container creation, cutting the + # # restore latency roughly to CRIU's page-restore time. The checkpoint + # # data is kept in the container's storage directory; tmpfs_threshold / + # # tmpfs_max_size do not apply in this mode. + # restore_in_place: false # Optional: Wait duration after RPC becomes ready before running tests. # Useful for clients like Erigon that need time to complete internal sync pipelines # after their RPC endpoint becomes available. diff --git a/docs/configuration.md b/docs/configuration.md index 2140ec48..3b4ef91d 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -852,6 +852,7 @@ Options for the `container-checkpoint-restore` rollback strategy, nested under ` | `tmpfs_max_size` | string | 2× `tmpfs_threshold` | Maximum size of the tmpfs mount for checkpoint storage. Same format as `tmpfs_threshold` (e.g., `"16g"`, `"1024m"`). When not set, defaults to twice the `tmpfs_threshold` value. | | `wait_after_tcp_drop_connections` | string | `10s` | How long to wait after dropping TCP connections before checkpointing, giving the process time to close file descriptors (Go duration string). | | `restart_container` | bool | `false` | Whether to restart the container before taking a CRIU checkpoint. Restarting ensures a clean process state (cold caches, clean DB shutdown). | +| `restore_in_place` | bool | `false` | Restore the checkpointed container in place between tests instead of importing the checkpoint archive into a fresh container. The checkpoint data is kept in the container's storage directory and the same container (same name, mounts, and IP) is restored repeatedly. This skips the per-test archive extraction and container creation of the export/import path, cutting the restore latency roughly to CRIU's page-restore time (e.g. ~15s → ~3s for an ~8g checkpoint). `tmpfs_threshold` / `tmpfs_max_size` do not apply in this mode. | ```yaml runner: diff --git a/pkg/config/config.go b/pkg/config/config.go index b1991885..5171b1cf 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -1088,6 +1088,7 @@ type CheckpointRestoreStrategyOptions struct { TmpfsMaxSize string `yaml:"tmpfs_max_size,omitempty" mapstructure:"tmpfs_max_size" json:"tmpfs_max_size,omitempty"` WaitAfterTCPDropConns string `yaml:"wait_after_tcp_drop_connections,omitempty" mapstructure:"wait_after_tcp_drop_connections" json:"wait_after_tcp_drop_connections,omitempty"` RestartContainer bool `yaml:"restart_container,omitempty" mapstructure:"restart_container" json:"restart_container,omitempty"` + RestoreInPlace bool `yaml:"restore_in_place,omitempty" mapstructure:"restore_in_place" json:"restore_in_place,omitempty"` } // BootstrapFCUConfig configures the bootstrap FCU call used to confirm the @@ -2939,6 +2940,22 @@ func (c *Config) GetCheckpointRestartContainer(instance *ClientInstance) bool { return opts.RestartContainer } +// GetCheckpointRestoreInPlace returns whether per-test rollbacks restore the +// checkpointed container in place from the checkpoint data kept in its +// storage directory, instead of importing the checkpoint archive into a +// fresh container. In-place restores skip the per-test archive extraction +// and container creation, cutting the per-test restore latency roughly to +// CRIU's page-restore time. Instance-level setting takes precedence over +// global default. +func (c *Config) GetCheckpointRestoreInPlace(instance *ClientInstance) bool { + opts := c.GetCheckpointRestoreStrategyOptions(instance) + if opts == nil { + return false + } + + return opts.RestoreInPlace +} + // GetMetadataLabels returns the merged metadata labels for an instance. // Client-level metadata labels serve as defaults; instance-level labels // override specific keys. diff --git a/pkg/config/config_test.go b/pkg/config/config_test.go index 2475d45f..f37842b4 100644 --- a/pkg/config/config_test.go +++ b/pkg/config/config_test.go @@ -2156,6 +2156,58 @@ func TestGetCheckpointTmpfsThreshold(t *testing.T) { } } +func TestGetCheckpointRestoreInPlace(t *testing.T) { + tests := []struct { + name string + global *CheckpointRestoreStrategyOptions + instance *CheckpointRestoreStrategyOptions + expected bool + }{ + { + name: "no options defaults to false", + global: nil, + instance: nil, + expected: false, + }, + { + name: "global enabled, no instance options inherits global", + global: &CheckpointRestoreStrategyOptions{RestoreInPlace: true}, + instance: nil, + expected: true, + }, + { + name: "instance options replace global entirely", + global: &CheckpointRestoreStrategyOptions{RestoreInPlace: true}, + instance: &CheckpointRestoreStrategyOptions{TmpfsThreshold: "4g"}, + expected: false, + }, + { + name: "instance enabled", + global: nil, + instance: &CheckpointRestoreStrategyOptions{RestoreInPlace: true}, + expected: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + cfg := &Config{ + Runner: RunnerConfig{ + Client: ClientConfig{ + Config: ClientDefaults{ + CheckpointRestoreStrategyOptions: tt.global, + }, + }, + }, + } + instance := &ClientInstance{ + CheckpointRestoreStrategyOptions: tt.instance, + } + assert.Equal(t, tt.expected, cfg.GetCheckpointRestoreInPlace(instance)) + }) + } +} + // createTestTarball creates a minimal .tar.gz file at the given path for testing. func createTestTarball(t *testing.T, path string) { t.Helper() diff --git a/pkg/podman/checkpoint.go b/pkg/podman/checkpoint.go index da47157d..46818411 100644 --- a/pkg/podman/checkpoint.go +++ b/pkg/podman/checkpoint.go @@ -39,6 +39,22 @@ type CheckpointManager interface { // export file. Returns the new container's ID. RestoreContainer(ctx context.Context, exportPath string, opts *RestoreOptions) (string, error) + // CheckpointContainerLocal checkpoints a running container, keeping the + // checkpoint data inside the container's storage directory instead of + // exporting an archive. The container stops after checkpointing. The + // kept checkpoint data allows repeated in-place restores of the same + // container via RestoreContainerInPlace. + CheckpointContainerLocal( + ctx context.Context, containerID string, + waitAfterTCPDrop time.Duration, + ) error + + // RestoreContainerInPlace restores a previously checkpointed container + // from the checkpoint data kept in its storage directory (no archive + // import, no new container). The checkpoint data is kept after the + // restore so the container can be stopped and restored again. + RestoreContainerInPlace(ctx context.Context, containerID string) error + // ReadFileFromImage extracts a file from an OCI image by running a // throwaway container. Used to read config files that need patching // before the real container starts. @@ -143,6 +159,131 @@ func (m *manager) CheckpointContainer( return nil } +// CheckpointContainerLocal checkpoints a running container, keeping the +// checkpoint data in the container's storage directory (no export archive). +// The container stops as part of the checkpoint. Because the checkpoint +// data stays on disk (Keep), the same container can be restored in place +// repeatedly — podman accepts a restore of a stopped container whenever +// its kept checkpoint data is present. +func (m *manager) CheckpointContainerLocal( + ctx context.Context, + containerID string, + waitAfterTCPDrop time.Duration, +) error { + m.log.WithField("container", containerID[:12]).Info( + "Checkpointing container (local, in-place restores)", + ) + + // Same connection-drop dance as the export path: CRIU refuses to + // checkpoint established connections without --tcp-established, and + // even though in-place restores keep the container IP, dropping the + // sockets keeps both checkpoint flavours behaviourally identical. + if err := m.dropConnections(ctx, containerID, waitAfterTCPDrop); err != nil { + m.log.WithError(err).Warn("Failed to drop connections before checkpoint") + } + + checkpointStart := time.Now() + + fileLocks := true + keep := true + tcpEstablished := true + printStats := true + + conn, connCancel := m.connWithCtx(ctx) + defer connCancel() + + report, err := containers.Checkpoint(conn, containerID, &containers.CheckpointOptions{ + FileLocks: &fileLocks, + Keep: &keep, + TCPEstablished: &tcpEstablished, + PrintStats: &printStats, + }) + if err != nil { + m.logCRIUDumpLog(containerID) + + return fmt.Errorf("checkpointing container %s locally: %w", containerID[:12], err) + } + + fields := logrus.Fields{ + "duration": time.Since(checkpointStart).Round(time.Millisecond), + "runtime_duration": time.Duration(report.RuntimeDuration) * time.Microsecond, + } + + if s := report.CRIUStatistics; s != nil { + fields["freezing_time"] = time.Duration(s.FreezingTime) * time.Microsecond + fields["frozen_time"] = time.Duration(s.FrozenTime) * time.Microsecond + fields["memdump_time"] = time.Duration(s.MemdumpTime) * time.Microsecond + fields["memwrite_time"] = time.Duration(s.MemwriteTime) * time.Microsecond + fields["pages_scanned"] = s.PagesScanned + fields["pages_written"] = s.PagesWritten + } + + m.log.WithFields(fields).Info("Container checkpointed successfully (local)") + + return nil +} + +// RestoreContainerInPlace restores a stopped, previously checkpointed +// container from the checkpoint data kept in its storage directory. The +// process resumes mid-execution with its original name, mounts, and IP. +// Keep leaves the checkpoint data in place for the next restore. +func (m *manager) RestoreContainerInPlace( + ctx context.Context, + containerID string, +) error { + m.log.WithField("container", containerID[:12]).Info( + "Restoring container in place from kept checkpoint", + ) + + restoreStart := time.Now() + + fileLocks := true + keep := true + tcpEstablished := true + tcpClose := true + printStats := true + + // Note: no Name option — renaming is only valid for archive imports. + // The container to restore is addressed via the positional nameOrID. + restoreOpts := &containers.RestoreOptions{ + FileLocks: &fileLocks, + Keep: &keep, + TCPEstablished: &tcpEstablished, + TCPClose: &tcpClose, + PrintStats: &printStats, + } + + conn, connCancel := m.connWithCtx(ctx) + defer connCancel() + + report, err := containers.Restore(conn, containerID, restoreOpts) + if err != nil { + m.logCRIURestoreLog(containerID) + + return fmt.Errorf( + "restoring container %s in place: %w", containerID[:12], err, + ) + } + + fields := logrus.Fields{ + "id": report.Id[:12], + "duration": time.Since(restoreStart).Round(time.Millisecond), + "runtime_duration": time.Duration(report.RuntimeDuration) * time.Microsecond, + } + + if s := report.CRIUStatistics; s != nil { + fields["forking_time"] = time.Duration(s.ForkingTime) * time.Microsecond + fields["restore_time"] = time.Duration(s.RestoreTime) * time.Microsecond + fields["pages_compared"] = s.PagesCompared + fields["pages_skipped_cow"] = s.PagesSkippedCow + fields["pages_restored"] = s.PagesRestored + } + + m.log.WithFields(fields).Info("Container restored in place successfully") + + return nil +} + // RestoreContainer restores a container from a checkpoint export file. It // creates a new container with the given name and mounts, then starts it from // the checkpointed state (the process resumes mid-execution). diff --git a/pkg/runner/strategy_checkpoint.go b/pkg/runner/strategy_checkpoint.go index 84a25564..8c6eefa1 100644 --- a/pkg/runner/strategy_checkpoint.go +++ b/pkg/runner/strategy_checkpoint.go @@ -192,6 +192,13 @@ func (r *runner) runTestsWithCheckpointRestore( log.WithField("steps", n).Info("Pre-run steps completed before checkpoint") } + // In-place mode keeps the checkpoint data in the container's storage + // directory and repeatedly restores the same container, skipping the + // per-test archive extraction and container creation of the + // export/import path. tmpfs_threshold/tmpfs_max_size only apply to the + // export archive, so they are ignored in this mode. + restoreInPlace := r.cfg.FullConfig.GetCheckpointRestoreInPlace(params.Instance) + // 3. Decide checkpoint export path: tmpfs (RAM) or disk. // // When checkpoint_tmpfs_threshold is configured and the container's @@ -201,6 +208,10 @@ func (r *runner) runTestsWithCheckpointRestore( tmpfsDir := "" thresholdStr := r.cfg.FullConfig.GetCheckpointTmpfsThreshold(params.Instance) + if restoreInPlace { + thresholdStr = "" + } + if thresholdStr != "" { threshold, parseErr := config.ParseByteSize(thresholdStr) if parseErr != nil { @@ -288,8 +299,14 @@ func (r *runner) runTestsWithCheckpointRestore( cpStart := time.Now() - if err := cpMgr.CheckpointContainer(ctx, containerID, exportPath, waitAfterTCPDrop); err != nil { - return nil, fmt.Errorf("checkpointing container: %w", err) + if restoreInPlace { + if err := cpMgr.CheckpointContainerLocal(ctx, containerID, waitAfterTCPDrop); err != nil { + return nil, fmt.Errorf("checkpointing container locally: %w", err) + } + } else { + if err := cpMgr.CheckpointContainer(ctx, containerID, exportPath, waitAfterTCPDrop); err != nil { + return nil, fmt.Errorf("checkpointing container: %w", err) + } } log.WithField("duration", time.Since(cpStart)).Info( @@ -297,6 +314,10 @@ func (r *runner) runTestsWithCheckpointRestore( ) defer func() { + if restoreInPlace { + return + } + _ = os.Remove(exportPath) if tmpfsDir != "" { @@ -430,35 +451,63 @@ func (r *runner) runTestsWithCheckpointRestore( } // Restore container from checkpoint. - restoreName := fmt.Sprintf("%s-restore-%d", params.ContainerSpec.Name, i) - testLog.Info("Restoring container from checkpoint") + var ( + restoreName string + restoredID string + err error + ) restoreStart := time.Now() - restoredID, err := cpMgr.RestoreContainer(ctx, exportPath, &podman.RestoreOptions{ - Name: restoreName, - NetworkName: r.cfg.ContainerNetwork, - }) - if err != nil { - combined.TotalDuration = time.Since(startTime) + if restoreInPlace { + // Restore the original container from its kept checkpoint + // data — same ID, name, mounts, and IP every iteration. + restoreName = params.ContainerSpec.Name + restoredID = containerID + + testLog.Info("Restoring container in place from checkpoint") + + if err = cpMgr.RestoreContainerInPlace(ctx, containerID); err != nil { + combined.TotalDuration = time.Since(startTime) + + return combined, fmt.Errorf( + "restoring container in place for test %d: %w", i, err, + ) + } + } else { + restoreName = fmt.Sprintf("%s-restore-%d", params.ContainerSpec.Name, i) - return combined, fmt.Errorf("restoring container for test %d: %w", i, err) + testLog.Info("Restoring container from checkpoint") + + restoredID, err = cpMgr.RestoreContainer(ctx, exportPath, &podman.RestoreOptions{ + Name: restoreName, + NetworkName: r.cfg.ContainerNetwork, + }) + if err != nil { + combined.TotalDuration = time.Since(startTime) + + return combined, fmt.Errorf("restoring container for test %d: %w", i, err) + } } testLog.WithField("duration", time.Since(restoreStart)).Info( "Container restored from checkpoint", ) - // Register cleanup for this iteration. - iterID := restoredID - - *cleanupFuncs = append(*cleanupFuncs, func() { - if rmErr := r.containerMgr.RemoveContainer( - context.Background(), iterID, - ); rmErr != nil && !isContainerNotFound(rmErr) { - testLog.WithError(rmErr).Warn("Failed to remove restored container") - } - }) + // Register cleanup for this iteration. In-place mode reuses the + // original container, whose removal is already handled by the + // container lifecycle cleanup. + if !restoreInPlace { + iterID := restoredID + + *cleanupFuncs = append(*cleanupFuncs, func() { + if rmErr := r.containerMgr.RemoveContainer( + context.Background(), iterID, + ); rmErr != nil && !isContainerNotFound(rmErr) { + testLog.WithError(rmErr).Warn("Failed to remove restored container") + } + }) + } // Get container IP. restoredIP, err := r.containerMgr.GetContainerIP( @@ -523,28 +572,44 @@ func (r *runner) runTestsWithCheckpointRestore( testLog.WithError(execErr).Error("Test execution failed") } - // Force-remove the container (no graceful stop needed — ZFS - // rollback discards the datadir anyway). Use a fresh context - // so this succeeds even if the parent was cancelled (CTRL+C). - testLog.Info("Force-removing restored container") - + // Tear the restored container down for the next iteration. Use a + // fresh context so this succeeds even if the parent was cancelled + // (CTRL+C). In-place mode only stops the container (SIGKILL via + // zero timeout — the datadir rollback discards its writes anyway), + // keeping it and its checkpoint data for the next restore; the + // export/import mode removes the throwaway container entirely. rmStart := time.Now() rmCtx, rmCancel := context.WithTimeout( context.Background(), 30*time.Second, ) - if rmErr := r.containerMgr.RemoveContainer( - rmCtx, restoredID, - ); rmErr != nil && !isContainerNotFound(rmErr) { - testLog.WithError(rmErr).Warn( - "Failed to remove restored container", - ) + if restoreInPlace { + testLog.Info("Stopping restored container") + + zeroTimeout := 0 + if stopErr := r.containerMgr.StopContainer( + rmCtx, restoredID, &zeroTimeout, + ); stopErr != nil && !isContainerNotFound(stopErr) { + testLog.WithError(stopErr).Warn( + "Failed to stop restored container", + ) + } + } else { + testLog.Info("Force-removing restored container") + + if rmErr := r.containerMgr.RemoveContainer( + rmCtx, restoredID, + ); rmErr != nil && !isContainerNotFound(rmErr) { + testLog.WithError(rmErr).Warn( + "Failed to remove restored container", + ) + } } rmCancel() testLog.WithField("duration", time.Since(rmStart)).Info( - "Restored container removed", + "Restored container torn down", ) waitForLogDrain(logDone, logCancel, logDrainTimeout) diff --git a/ui/src/api/types.ts b/ui/src/api/types.ts index 8daabf9e..5c24a8b8 100644 --- a/ui/src/api/types.ts +++ b/ui/src/api/types.ts @@ -312,6 +312,7 @@ export interface CheckpointRestoreStrategyOptions { tmpfs_max_size?: string wait_after_tcp_drop_connections?: string restart_container?: boolean + restore_in_place?: boolean } export interface OpcodeExtractionConfig { diff --git a/ui/src/components/run-detail/RunConfiguration.tsx b/ui/src/components/run-detail/RunConfiguration.tsx index 8c2dadda..370f5936 100644 --- a/ui/src/components/run-detail/RunConfiguration.tsx +++ b/ui/src/components/run-detail/RunConfiguration.tsx @@ -271,6 +271,12 @@ export function RunConfiguration({ instance, system, startBlock, metadata, bench {instance.checkpoint_restore_strategy_options.restart_container ? 'true' : 'false'} )} + {instance.checkpoint_restore_strategy_options.restore_in_place !== undefined && ( +
+ restore_in_place: + {instance.checkpoint_restore_strategy_options.restore_in_place ? 'true' : 'false'} +
+ )}

Options for the container-checkpoint-restore rollback strategy.