Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 7 additions & 0 deletions config.example.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -442,6 +442,13 @@ runner:
# # Whether to restart the container before taking a CRIU checkpoint.
# # Restarting ensures a clean process state (cold caches, clean DB shutdown).
# restart_container: false
# # Restore the checkpointed container in place between tests instead of
# # importing the checkpoint archive into a fresh container. Skips the
# # per-test archive extraction and container creation, cutting the
# # restore latency roughly to CRIU's page-restore time. The checkpoint
# # data is kept in the container's storage directory; tmpfs_threshold /
# # tmpfs_max_size do not apply in this mode.
# restore_in_place: false
# Optional: Wait duration after RPC becomes ready before running tests.
# Useful for clients like Erigon that need time to complete internal sync pipelines
# after their RPC endpoint becomes available.
Expand Down
1 change: 1 addition & 0 deletions docs/configuration.md
Original file line number Diff line number Diff line change
Expand Up @@ -852,6 +852,7 @@ Options for the `container-checkpoint-restore` rollback strategy, nested under `
| `tmpfs_max_size` | string | 2× `tmpfs_threshold` | Maximum size of the tmpfs mount for checkpoint storage. Same format as `tmpfs_threshold` (e.g., `"16g"`, `"1024m"`). When not set, defaults to twice the `tmpfs_threshold` value. |
| `wait_after_tcp_drop_connections` | string | `10s` | How long to wait after dropping TCP connections before checkpointing, giving the process time to close file descriptors (Go duration string). |
| `restart_container` | bool | `false` | Whether to restart the container before taking a CRIU checkpoint. Restarting ensures a clean process state (cold caches, clean DB shutdown). |
| `restore_in_place` | bool | `false` | Restore the checkpointed container in place between tests instead of importing the checkpoint archive into a fresh container. The checkpoint data is kept in the container's storage directory and the same container (same name, mounts, and IP) is restored repeatedly. This skips the per-test archive extraction and container creation of the export/import path, cutting the restore latency roughly to CRIU's page-restore time (e.g. ~15s → ~3s for an ~8g checkpoint). `tmpfs_threshold` / `tmpfs_max_size` do not apply in this mode. |

```yaml
runner:
Expand Down
17 changes: 17 additions & 0 deletions pkg/config/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -1088,6 +1088,7 @@ type CheckpointRestoreStrategyOptions struct {
TmpfsMaxSize string `yaml:"tmpfs_max_size,omitempty" mapstructure:"tmpfs_max_size" json:"tmpfs_max_size,omitempty"`
WaitAfterTCPDropConns string `yaml:"wait_after_tcp_drop_connections,omitempty" mapstructure:"wait_after_tcp_drop_connections" json:"wait_after_tcp_drop_connections,omitempty"`
RestartContainer bool `yaml:"restart_container,omitempty" mapstructure:"restart_container" json:"restart_container,omitempty"`
RestoreInPlace bool `yaml:"restore_in_place,omitempty" mapstructure:"restore_in_place" json:"restore_in_place,omitempty"`
}

// BootstrapFCUConfig configures the bootstrap FCU call used to confirm the
Expand Down Expand Up @@ -2939,6 +2940,22 @@ func (c *Config) GetCheckpointRestartContainer(instance *ClientInstance) bool {
return opts.RestartContainer
}

// GetCheckpointRestoreInPlace returns whether per-test rollbacks restore the
// checkpointed container in place from the checkpoint data kept in its
// storage directory, instead of importing the checkpoint archive into a
// fresh container. In-place restores skip the per-test archive extraction
// and container creation, cutting the per-test restore latency roughly to
// CRIU's page-restore time. Instance-level setting takes precedence over
// global default.
func (c *Config) GetCheckpointRestoreInPlace(instance *ClientInstance) bool {
opts := c.GetCheckpointRestoreStrategyOptions(instance)
if opts == nil {
return false
}

return opts.RestoreInPlace
}

// GetMetadataLabels returns the merged metadata labels for an instance.
// Client-level metadata labels serve as defaults; instance-level labels
// override specific keys.
Expand Down
52 changes: 52 additions & 0 deletions pkg/config/config_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -2156,6 +2156,58 @@ func TestGetCheckpointTmpfsThreshold(t *testing.T) {
}
}

func TestGetCheckpointRestoreInPlace(t *testing.T) {
tests := []struct {
name string
global *CheckpointRestoreStrategyOptions
instance *CheckpointRestoreStrategyOptions
expected bool
}{
{
name: "no options defaults to false",
global: nil,
instance: nil,
expected: false,
},
{
name: "global enabled, no instance options inherits global",
global: &CheckpointRestoreStrategyOptions{RestoreInPlace: true},
instance: nil,
expected: true,
},
{
name: "instance options replace global entirely",
global: &CheckpointRestoreStrategyOptions{RestoreInPlace: true},
instance: &CheckpointRestoreStrategyOptions{TmpfsThreshold: "4g"},
expected: false,
},
{
name: "instance enabled",
global: nil,
instance: &CheckpointRestoreStrategyOptions{RestoreInPlace: true},
expected: true,
},
}

for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
cfg := &Config{
Runner: RunnerConfig{
Client: ClientConfig{
Config: ClientDefaults{
CheckpointRestoreStrategyOptions: tt.global,
},
},
},
}
instance := &ClientInstance{
CheckpointRestoreStrategyOptions: tt.instance,
}
assert.Equal(t, tt.expected, cfg.GetCheckpointRestoreInPlace(instance))
})
}
}

// createTestTarball creates a minimal .tar.gz file at the given path for testing.
func createTestTarball(t *testing.T, path string) {
t.Helper()
Expand Down
141 changes: 141 additions & 0 deletions pkg/podman/checkpoint.go
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,22 @@ type CheckpointManager interface {
// export file. Returns the new container's ID.
RestoreContainer(ctx context.Context, exportPath string, opts *RestoreOptions) (string, error)

// CheckpointContainerLocal checkpoints a running container, keeping the
// checkpoint data inside the container's storage directory instead of
// exporting an archive. The container stops after checkpointing. The
// kept checkpoint data allows repeated in-place restores of the same
// container via RestoreContainerInPlace.
CheckpointContainerLocal(
ctx context.Context, containerID string,
waitAfterTCPDrop time.Duration,
) error

// RestoreContainerInPlace restores a previously checkpointed container
// from the checkpoint data kept in its storage directory (no archive
// import, no new container). The checkpoint data is kept after the
// restore so the container can be stopped and restored again.
RestoreContainerInPlace(ctx context.Context, containerID string) error

// ReadFileFromImage extracts a file from an OCI image by running a
// throwaway container. Used to read config files that need patching
// before the real container starts.
Expand Down Expand Up @@ -143,6 +159,131 @@ func (m *manager) CheckpointContainer(
return nil
}

// CheckpointContainerLocal checkpoints a running container, keeping the
// checkpoint data in the container's storage directory (no export archive).
// The container stops as part of the checkpoint. Because the checkpoint
// data stays on disk (Keep), the same container can be restored in place
// repeatedly — podman accepts a restore of a stopped container whenever
// its kept checkpoint data is present.
func (m *manager) CheckpointContainerLocal(
ctx context.Context,
containerID string,
waitAfterTCPDrop time.Duration,
) error {
m.log.WithField("container", containerID[:12]).Info(
"Checkpointing container (local, in-place restores)",
)

// Same connection-drop dance as the export path: CRIU refuses to
// checkpoint established connections without --tcp-established, and
// even though in-place restores keep the container IP, dropping the
// sockets keeps both checkpoint flavours behaviourally identical.
if err := m.dropConnections(ctx, containerID, waitAfterTCPDrop); err != nil {
m.log.WithError(err).Warn("Failed to drop connections before checkpoint")
}

checkpointStart := time.Now()

fileLocks := true
keep := true
tcpEstablished := true
printStats := true

conn, connCancel := m.connWithCtx(ctx)
defer connCancel()

report, err := containers.Checkpoint(conn, containerID, &containers.CheckpointOptions{
FileLocks: &fileLocks,
Keep: &keep,
TCPEstablished: &tcpEstablished,
PrintStats: &printStats,
})
if err != nil {
m.logCRIUDumpLog(containerID)

return fmt.Errorf("checkpointing container %s locally: %w", containerID[:12], err)
}

fields := logrus.Fields{
"duration": time.Since(checkpointStart).Round(time.Millisecond),
"runtime_duration": time.Duration(report.RuntimeDuration) * time.Microsecond,
}

if s := report.CRIUStatistics; s != nil {
fields["freezing_time"] = time.Duration(s.FreezingTime) * time.Microsecond
fields["frozen_time"] = time.Duration(s.FrozenTime) * time.Microsecond
fields["memdump_time"] = time.Duration(s.MemdumpTime) * time.Microsecond
fields["memwrite_time"] = time.Duration(s.MemwriteTime) * time.Microsecond
fields["pages_scanned"] = s.PagesScanned
fields["pages_written"] = s.PagesWritten
}

m.log.WithFields(fields).Info("Container checkpointed successfully (local)")

return nil
}

// RestoreContainerInPlace restores a stopped, previously checkpointed
// container from the checkpoint data kept in its storage directory. The
// process resumes mid-execution with its original name, mounts, and IP.
// Keep leaves the checkpoint data in place for the next restore.
func (m *manager) RestoreContainerInPlace(
ctx context.Context,
containerID string,
) error {
m.log.WithField("container", containerID[:12]).Info(
"Restoring container in place from kept checkpoint",
)

restoreStart := time.Now()

fileLocks := true
keep := true
tcpEstablished := true
tcpClose := true
printStats := true

// Note: no Name option — renaming is only valid for archive imports.
// The container to restore is addressed via the positional nameOrID.
restoreOpts := &containers.RestoreOptions{
FileLocks: &fileLocks,
Keep: &keep,
TCPEstablished: &tcpEstablished,
TCPClose: &tcpClose,
PrintStats: &printStats,
}

conn, connCancel := m.connWithCtx(ctx)
defer connCancel()

report, err := containers.Restore(conn, containerID, restoreOpts)
if err != nil {
m.logCRIURestoreLog(containerID)

return fmt.Errorf(
"restoring container %s in place: %w", containerID[:12], err,
)
}

fields := logrus.Fields{
"id": report.Id[:12],
"duration": time.Since(restoreStart).Round(time.Millisecond),
"runtime_duration": time.Duration(report.RuntimeDuration) * time.Microsecond,
}

if s := report.CRIUStatistics; s != nil {
fields["forking_time"] = time.Duration(s.ForkingTime) * time.Microsecond
fields["restore_time"] = time.Duration(s.RestoreTime) * time.Microsecond
fields["pages_compared"] = s.PagesCompared
fields["pages_skipped_cow"] = s.PagesSkippedCow
fields["pages_restored"] = s.PagesRestored
}

m.log.WithFields(fields).Info("Container restored in place successfully")

return nil
}

// RestoreContainer restores a container from a checkpoint export file. It
// creates a new container with the given name and mounts, then starts it from
// the checkpointed state (the process resumes mid-execution).
Expand Down
Loading
Loading