Skip to content

fix: harden cleanup apply resumability #619

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
chubes4 merged 2 commits into from
Jun 10, 2026
Merged

fix: harden cleanup apply resumability #619

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
95fe26a
fix: harden cleanup apply resumability
chubes4 Jun 10, 2026
0808078
fix: satisfy cleanup lint
chubes4 Jun 10, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
47 changes: 31 additions & 16 deletions inc/Cleanup/CleanupRemainingWorkSummary.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,10 +25,6 @@ public static function from_items( array $items ): array {
$summary = self::empty_summary();

foreach ( $items as $item ) {
if ( ! is_array($item) ) {
continue;
}

$type = (string) ( $item['item_type'] ?? 'unknown' );
$status = (string) ( $item['status'] ?? 'unknown' );
$evidence = (array) ( $item['evidence'] ?? array() );
Expand All @@ -50,7 +46,7 @@ public static function from_items( array $items ): array {
if ( 'artifact_cleanup' === $type && 'applied' !== $status ) {
$summary['remaining_reclaimable_artifact_bytes'] += self::row_bytes($row, array( 'artifact_size_bytes', 'size_bytes' ));
}
if ( 'worktree_removal' === $type && in_array($status, array( 'pending', 'failed' ), true) ) {
if ( 'worktree_removal' === $type && in_array($status, array( 'pending', 'failed', 'applying' ), true) ) {
++$summary['remaining_safely_removable_worktrees'];
}
}
Expand Down Expand Up @@ -181,18 +177,22 @@ private static function recommended_commands( array $summary ): array {
$commands = array();
if ( (int) $summary['remaining_reclaimable_artifact_bytes'] > 0 ) {
$commands[] = array(
'bucket' => 'remaining_artifacts',
'command' => 'studio wp datamachine-code workspace cleanup run --mode=artifacts --dry-run --format=json',
'apply' => 'studio wp datamachine-code workspace cleanup run --mode=artifacts',
'destructive' => false,
'bucket' => 'remaining_artifacts',
'command' => 'studio wp datamachine-code workspace cleanup run --mode=artifacts --dry-run --format=json',
'apply' => 'studio wp datamachine-code workspace cleanup run --mode=artifacts',
'destructive' => false,
'apply_destructive' => true,
'why' => 'Preview remaining artifact cleanup first; the apply command removes revalidated artifacts.',
);
}
if ( (int) $summary['remaining_safely_removable_worktrees'] > 0 ) {
$commands[] = array(
'bucket' => 'remaining_worktrees',
'command' => 'studio wp datamachine-code workspace worktree bounded-cleanup-eligible-apply --dry-run --limit=25',
'apply' => 'studio wp datamachine-code workspace cleanup run --mode=retention',
'destructive' => false,
'bucket' => 'remaining_worktrees',
'command' => 'studio wp datamachine-code workspace worktree bounded-cleanup-eligible-apply --dry-run --limit=25',
'apply' => 'studio wp datamachine-code workspace cleanup run --mode=retention',
'destructive' => false,
'apply_destructive' => true,
'why' => 'Preview cleanup-eligible worktrees first; the apply command removes revalidated worktrees.',
);
}
foreach ( array_keys( (array) $summary['skipped_by_reason'] ) as $reason ) {
Expand Down Expand Up @@ -225,9 +225,24 @@ private static function command_for_reason( string $reason, string $bucket ): ar
};

return array(
'bucket' => $bucket . ':' . $reason,
'command' => $command,
'destructive' => false,
'bucket' => $bucket . ':' . $reason,
'command' => $command,
'destructive' => false,
'apply_destructive' => false,
'why' => self::reason_remediation($reason),
);
}

private static function reason_remediation( string $reason ): string {
return match ( $reason ) {
'dirty_worktree' => 'Inspect dirty files before applying cleanup; artifact-only dirt may be removable through artifact cleanup, source dirt needs review.',
'artifact_plan_mismatch', 'plan_mismatch' => 'Regenerate a fresh plan because the saved row no longer matches current filesystem or branch state.',
'artifact_plan_not_current', 'artifact_already_removed' => 'Regenerate artifact cleanup evidence; the saved artifact row is no longer a current candidate.',
'needs_metadata_reconcile' => 'Run metadata reconciliation so DMC can classify the worktree without a full cleanup scan.',
'lifecycle_reconciliation_candidate' => 'Run lifecycle reconciliation to collect PR/merge signals before emitting removal rows.',
'unpushed_commits' => 'Push, merge, or intentionally abandon commits before retrying cleanup.',
'probe_timeout' => 'Retry the review path with a smaller bounded page or investigate the git probe timeout.',
default => 'Run the review command to refresh evidence before applying cleanup.',
};
}
}
38 changes: 35 additions & 3 deletions inc/Cli/Commands/WorkspaceCommand.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -767,6 +767,9 @@ private function render_cleanup_control_result( array $result, array $assoc_args
WP_CLI::log(sprintf('%s: %s', ucfirst(str_replace('_', ' ', $key)), (string) $result[ $key ]));
}
}
if ( ! empty($result['progress']) && is_array($result['progress']) ) {
$this->render_cleanup_progress_summary( (array) $result['progress']);
}
if ( ! empty($result['remaining_work_summary']) && is_array($result['remaining_work_summary']) ) {
$this->render_cleanup_remaining_work_summary( (array) $result['remaining_work_summary']);
}
Expand Down Expand Up @@ -810,12 +813,41 @@ private function render_cleanup_remaining_work_summary( array $summary ): void {
WP_CLI::log('Recommended next commands:');
$rows = array_map(
fn( $row ) => array(
'bucket' => is_array($row) ? (string) ( $row['bucket'] ?? '' ) : '',
'command' => is_array($row) ? (string) ( $row['command'] ?? '' ) : '',
'bucket' => is_array($row) ? (string) ( $row['bucket'] ?? '' ) : '',
'review_command' => is_array($row) ? (string) ( $row['command'] ?? '' ) : '',
'apply_command' => is_array($row) ? (string) ( $row['apply'] ?? '' ) : '',
'apply_destructive' => is_array($row) && ! empty($row['apply_destructive']) ? 'yes' : 'no',
),
array_slice($commands, 0, 10)
);
$this->format_items($rows, array( 'bucket', 'command' ), array( 'format' => 'table' ), 'bucket');
$this->format_items($rows, array( 'bucket', 'review_command', 'apply_command', 'apply_destructive' ), array( 'format' => 'table' ), 'bucket');
}
}

private function render_cleanup_progress_summary( array $progress ): void {
WP_CLI::log('');
WP_CLI::log('Progress:');
$this->format_items(
array(
array(
'metric' => 'applying_rows',
'value' => (int) ( $progress['applying_rows'] ?? 0 ),
),
array(
'metric' => 'pending_or_failed',
'value' => (int) ( $progress['pending_or_failed'] ?? 0 ),
),
array(
'metric' => 'resumable',
'value' => ! empty($progress['resumable']) ? 'yes' : 'no',
),
),
array( 'metric', 'value' ),
array( 'format' => 'table' ),
'metric'
);
if ( ! empty($progress['note']) ) {
WP_CLI::log( (string) $progress['note']);
}
}

Expand Down
130 changes: 119 additions & 11 deletions inc/Workspace/CleanupRunService.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,8 +14,9 @@

class CleanupRunService {

private const DEFAULT_APPLY_LIMIT = 25;
private const MAX_APPLY_LIMIT = 100;
private const DEFAULT_APPLY_LIMIT = 25;
private const MAX_APPLY_LIMIT = 100;
private const WORKTREE_APPLY_BATCH_LIMIT = 1;



Expand Down Expand Up @@ -101,9 +102,10 @@ public function apply( string $run_id, array $opts = array() ): array|\WP_Error
$results = array();

if ( array() !== $artifact_rows ) {
$artifact_batch = array_slice($artifact_rows, 0, $limit);
$processed_rows += count($artifact_batch);
$batch_type = 'artifact_cleanup';
$artifact_batch = array_slice($artifact_rows, 0, $limit);
$processed_rows += count($artifact_batch);
$batch_type = 'artifact_cleanup';
$this->mark_batch_applying($artifact_batch, $run_id, $batch_type, $limit, $remaining_rows);
$results['artifact_cleanup'] = $this->workspace->worktree_cleanup_artifacts(
array(
'apply_plan' => array( 'candidates' => array_map(fn( $item ) => $item['evidence'], $artifact_batch) ),
Expand All @@ -116,9 +118,10 @@ public function apply( string $run_id, array $opts = array() ): array|\WP_Error

$remaining_capacity = max(0, $limit - $processed_rows);
if ( $remaining_capacity > 0 && array() !== $worktree_rows ) {
$worktree_batch = array_slice($worktree_rows, 0, $remaining_capacity);
$processed_rows += count($worktree_batch);
$batch_type = '' === $batch_type ? 'worktree_removal' : 'mixed';
$worktree_batch = array_slice($worktree_rows, 0, min($remaining_capacity, self::WORKTREE_APPLY_BATCH_LIMIT));
$processed_rows += count($worktree_batch);
$batch_type = '' === $batch_type ? 'worktree_removal' : 'mixed';
$this->mark_batch_applying($worktree_batch, $run_id, $batch_type, $limit, $remaining_rows);
$results['worktree_removal'] = $this->workspace->worktree_cleanup_merged(
array(
'apply_plan' => array( 'candidates' => array_map(fn( $item ) => $item['evidence'], $worktree_batch) ),
Expand Down Expand Up @@ -195,13 +198,15 @@ public function status( string $run_id ): array|\WP_Error {
$summary['items_by_status'][ $status ] = ( $summary['items_by_status'][ $status ] ?? 0 ) + 1;
$summary['items_by_type'][ $type ] = ( $summary['items_by_type'][ $type ] ?? 0 ) + 1;
$summary['bytes_reclaimed'] += max(0, (int) ( $item['bytes_reclaimed'] ?? 0 ));
if ( in_array($status, array( 'pending', 'failed' ), true) ) {
if ( in_array($status, array( 'pending', 'failed', 'applying' ), true) ) {
++$summary['pending_or_failed'];
}
}
ksort($summary['items_by_status']);
ksort($summary['items_by_type']);

$progress = $this->run_progress($run, $items, $summary);

return array(
'success' => true,
'state' => (string) ( $run['status'] ?? 'unknown' ),
Expand All @@ -210,7 +215,8 @@ public function status( string $run_id ): array|\WP_Error {
'mode' => $run['mode'] ?? '',
'run' => $run,
'summary' => $summary,
'remaining_work_summary' => CleanupRemainingWorkSummary::from_items($items),
'progress' => $progress,
'remaining_work_summary' => $this->remaining_work_summary($run_id, $items, $progress),
);
}

Expand Down Expand Up @@ -285,7 +291,109 @@ private function plan_items( array $plan ): array {
}

private function pending_rows_of_type( array $items, string $type ): array {
return array_values(array_filter($items, fn( $item ) => (string) ( $item['item_type'] ?? '' ) === $type && in_array( (string) ( $item['status'] ?? '' ), array( 'pending', 'failed' ), true)));
return array_values(array_filter($items, fn( $item ) => (string) ( $item['item_type'] ?? '' ) === $type && in_array( (string) ( $item['status'] ?? '' ), array( 'pending', 'failed', 'applying' ), true)));
}

/**
* Mark rows as in-progress before invoking destructive cleanup so interrupted
* operator runs leave a visible, resumable checkpoint instead of silent state.
*
* @param array<int,array<string,mixed>> $items Batch rows.
* @param string $run_id Run ID.
* @param string $batch_type Batch type label.
* @param int $limit Requested apply limit.
* @param int $remaining_rows Rows remaining before this batch.
*/
private function mark_batch_applying( array $items, string $run_id, string $batch_type, int $limit, int $remaining_rows ): void {
$started_at = gmdate('Y-m-d H:i:s');
foreach ( $items as $item ) {
$this->repository->update_item(
(int) $item['id'],
array(
'status' => 'applying',
'evidence' => array_merge(
(array) ( $item['evidence'] ?? array() ),
array(
'applying_started_at' => $started_at,
'applying_batch_type' => $batch_type,
)
),
)
);
}

$this->repository->update_run(
$run_id,
array(
'summary' => array(
'applying_batch' => array(
'type' => $batch_type,
'limit' => $limit,
'row_count' => count($items),
'remaining_before' => $remaining_rows,
'started_at' => $started_at,
),
),
)
);
}

/**
* Build operator progress metadata for status/evidence output.
*
* @param array<string,mixed> $run Run row.
* @param array<int,array<string,mixed>> $items Item rows.
* @param array<string,mixed> $summary Aggregate summary.
* @return array<string,mixed>
*/
private function run_progress( array $run, array $items, array $summary ): array {
$applying = array_values(array_filter($items, fn( $item ) => 'applying' === (string) ( $item['status'] ?? '' )));
$examples = array_slice(array_map(fn( $item ) => array(
'handle' => (string) ( $item['handle'] ?? '' ),
'type' => (string) ( $item['item_type'] ?? '' ),
), $applying), 0, 3);

$started_at = (string) ( $run['started_at'] ?? '' );
$age = '' !== $started_at ? max(0, time() - strtotime($started_at)) : 0;
$run_status = (string) ( $run['status'] ?? '' );
$resumable = (int) ( $summary['pending_or_failed'] ?? 0 ) > 0 && in_array($run_status, array( 'applying', 'needs_resume' ), true);

return array(
'applying_rows' => count($applying),
'applying_examples' => $examples,
'pending_or_failed' => (int) ( $summary['pending_or_failed'] ?? 0 ),
'started_at' => $started_at,
'age_seconds' => $age,
'resumable' => $resumable,
'note' => count($applying) > 0 ? 'Rows marked applying are safe to retry with workspace cleanup resume if the previous apply process was interrupted.' : '',
);
}

/**
* Build remaining-work summary and prepend the current run resume command.
*
* @param string $run_id Run ID.
* @param array<int,array<string,mixed>> $items Item rows.
* @param array<string,mixed> $progress Progress metadata.
* @return array<string,mixed>
*/
private function remaining_work_summary( string $run_id, array $items, array $progress ): array {
$summary = CleanupRemainingWorkSummary::from_items($items);
if ( ! empty($progress['resumable']) ) {
array_unshift(
$summary['recommended_commands'],
array(
'bucket' => 'current_run_resume',
'command' => sprintf('studio wp datamachine-code workspace cleanup status %s --format=json', $run_id),
'apply' => sprintf('studio wp datamachine-code workspace cleanup resume %s --limit=%d', $run_id, self::DEFAULT_APPLY_LIMIT),
'destructive' => false,
'apply_destructive' => true,
'why' => 'Resume the reviewed DB-backed cleanup run from persisted pending/failed/applying rows.',
)
);
}

return $summary;
}

private function apply_limit( array $opts ): int {
Expand Down
Loading
Loading