diff --git a/packages/cli/src/__tests__/runtime-host-launch-agent-service.test.ts b/packages/cli/src/__tests__/runtime-host-launch-agent-service.test.ts index 4bb50c7708..7c13e65cf0 100644 --- a/packages/cli/src/__tests__/runtime-host-launch-agent-service.test.ts +++ b/packages/cli/src/__tests__/runtime-host-launch-agent-service.test.ts @@ -22,10 +22,13 @@ import { chmod, mkdir, mkdtemp, readFile, rm, writeFile } from 'node:fs/promises import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { test } from 'node:test'; +import { RUNTIME_HOST_SERVICE_LOG_MAX_BYTES } from '@maka/runtime-host/operator'; import { createLaunchAgentRuntimeHostService, renderLaunchAgentPlist, + renderLaunchAgentUpdatePlist, resolveLaunchAgentPath, + resolveLaunchAgentUpdatePath, } from '../runtime-host-launch-agent-service.js'; import type { RuntimeHostManagedServiceConfig } from '../runtime-host-service-manager.js'; @@ -34,6 +37,8 @@ const UID = 501; const LABEL = `com.maka.runtime-host.${SERVICE_ID}`; const DOMAIN = `gui/${String(UID)}`; const TARGET = `${DOMAIN}/${LABEL}`; +const UPDATE_LABEL = `${LABEL}.update`; +const UPDATE_TARGET = `${DOMAIN}/${UPDATE_LABEL}`; test('renders the canonical Runtime Host command as a private persistent LaunchAgent', () => { const config = fixtureConfig('/tmp/node & tool', '/tmp/maka ', '/tmp/state > root'); @@ -55,6 +60,111 @@ test('renders the canonical Runtime Host command as a private persistent LaunchA assert.match(plist, /workspace=\/tmp\/projects<\/string>/u); }); +test('renders managed update reconciliation as a periodic one-shot LaunchAgent', () => { + const config = { + ...fixtureConfig('/tmp/node', '/tmp/maka', '/tmp/state'), + managedDeploymentRoot: '/tmp/managed deployment', + }; + const plist = renderLaunchAgentUpdatePlist(config, { + label: UPDATE_LABEL, + stdoutPath: '/tmp/update.stdout.log', + stderrPath: '/tmp/update.stderr.log', + }); + + assert.match(plist, /\/tmp\/managed deployment\/operator<\/string>/u); + assert.match(plist, /reconcile-update<\/string>/u); + assert.match(plist, /--framed<\/string>/u); + assert.match(plist, /StartInterval<\/key>\n 86400<\/integer>/u); + assert.doesNotMatch(plist, /KeepAlive<\/key>/u); +}); + +test('installs and removes the update scheduler with a managed LaunchAgent', async () => { + await withFixture(async ({ homeDir, cliPath, launchctl }) => { + const backend = createLaunchAgentRuntimeHostService(SERVICE_ID, { + homeDir, + uid: UID, + runLaunchctl: launchctl.run, + isProcessAlive: () => false, + }); + const config = { + ...fixtureConfig(process.execPath, cliPath, join(homeDir, 'state')), + managedDeploymentRoot: join(homeDir, 'managed'), + }; + + await backend.install(config); + await backend.verifyDeployment(config); + const updatePath = resolveLaunchAgentUpdatePath(SERVICE_ID, homeDir); + assert.match(await readFile(updatePath, 'utf8'), /reconcile-update/u); + const logDirectory = join(homeDir, 'Library', 'Logs', 'Maka', 'runtime-host-services'); + await Promise.all([ + writeFile(join(logDirectory, `${LABEL}.stdout.log`), 'h'.repeat(64 * 1024)), + writeFile(join(logDirectory, `${LABEL}.stderr.log`), 'host stderr'), + writeFile(join(logDirectory, `${UPDATE_LABEL}.stdout.log`), 'update stdout'), + writeFile( + join(logDirectory, `${UPDATE_LABEL}.stderr.log`), + 'scheduler reconciliation failed', + ), + ]); + const logs = await backend.logs(); + assert.match(logs, /scheduler reconciliation failed/u); + assert.ok(Buffer.byteLength(logs) <= RUNTIME_HOST_SERVICE_LOG_MAX_BYTES); + + const updateBootouts = () => + launchctl.calls.filter( + ([command, target]) => command === 'bootout' && target === UPDATE_TARGET, + ).length; + const bootoutsBeforeReplace = updateBootouts(); + await backend.replace(config); + assert.equal(updateBootouts(), bootoutsBeforeReplace); + + launchctl.updateRunning = true; + launchctl.failNextBootstrap = true; + await assert.rejects(backend.replace(config), /Starting the Runtime Host LaunchAgent failed/u); + assert.equal(updateBootouts(), bootoutsBeforeReplace); + assert.equal(launchctl.updateRunning, true); + launchctl.updateRunning = false; + + await writeFile(updatePath, 'stale\n', { mode: 0o600 }); + await assert.rejects( + backend.verifyReplacementPreconditions(config), + (error: unknown) => + error instanceof Error && 'code' in error && error.code === 'target_mismatch', + ); + await assert.rejects( + backend.verifyDeployment(config), + (error: unknown) => + error instanceof Error && 'code' in error && error.code === 'target_mismatch', + ); + await backend.install(config); + await backend.verifyDeployment(config); + + await backend.stop(); + assert.equal(launchctl.updateLoaded, false); + await backend.verifyDeployment(config); + await assert.rejects( + backend.verifyDeployment(config, { requireSchedulerReady: true }), + (error: unknown) => + error instanceof Error && 'code' in error && error.code === 'target_mismatch', + ); + await backend.replace(config); + assert.equal(launchctl.updateLoaded, true); + await backend.verifyDeployment(config, { requireSchedulerReady: true }); + + const { managedDeploymentRoot: _managedDeploymentRoot, ...unmanagedConfig } = config; + await backend.install(unmanagedConfig); + await backend.verifyDeployment(unmanagedConfig); + assert.equal(await fileExists(updatePath), false); + assert.equal(launchctl.updateLoaded, false); + await backend.verifyReplacementPreconditions(config); + await backend.replace(config); + await backend.verifyDeployment(config); + assert.equal(launchctl.updateLoaded, true); + + await backend.uninstall(); + assert.equal(await fileExists(updatePath), false); + }); +}); + test('maps install, stop, start, restart, and uninstall onto one LaunchAgent service', async () => { await withFixture(async ({ homeDir, cliPath, launchctl }) => { let processChecks = 0; @@ -145,6 +255,8 @@ test('restores the previous loaded LaunchAgent when deployment bootstrap fails', interface FakeLaunchctl { loaded: boolean; running: boolean; + updateLoaded: boolean; + updateRunning: boolean; failNextBootstrap: boolean; readonly calls: string[][]; readonly run: (args: readonly string[]) => Promise<{ @@ -159,6 +271,8 @@ function createFakeLaunchctl(): FakeLaunchctl { const fake: FakeLaunchctl = { loaded: false, running: false, + updateLoaded: false, + updateRunning: false, failNextBootstrap: false, calls: [], run: async (args) => { @@ -166,11 +280,14 @@ function createFakeLaunchctl(): FakeLaunchctl { if (args[0] === 'print' && args[1] === DOMAIN) { return { exitCode: 0, stdout: 'domain = gui\n', stderr: '' }; } - if (args[0] === 'print' && args[1] === TARGET) { - return fake.loaded + if (args[0] === 'print' && (args[1] === TARGET || args[1] === UPDATE_TARGET)) { + const update = args[1] === UPDATE_TARGET; + const loaded = update ? fake.updateLoaded : fake.loaded; + const running = update ? fake.updateRunning : fake.running; + return loaded ? { exitCode: 0, - stdout: fake.running + stdout: running ? `state = running\npid = ${String(pid)}\nlast exit code = 0\n` : 'state = not running\nlast exit code = 0\n', stderr: '', @@ -182,14 +299,25 @@ function createFakeLaunchctl(): FakeLaunchctl { fake.failNextBootstrap = false; return { exitCode: 5, stdout: '', stderr: 'Input/output error' }; } - fake.loaded = true; - fake.running = true; + const update = args[2]?.endsWith('.update.plist') ?? false; + if (update) { + fake.updateLoaded = true; + fake.updateRunning = false; + } else { + fake.loaded = true; + fake.running = true; + } pid += 1; return { exitCode: 0, stdout: '', stderr: '' }; } if (args[0] === 'bootout') { - fake.running = false; - fake.loaded = false; + if (args[1] === UPDATE_TARGET) { + fake.updateRunning = false; + fake.updateLoaded = false; + } else { + fake.running = false; + fake.loaded = false; + } return { exitCode: 0, stdout: '', stderr: '' }; } if (args[0] === 'kickstart') { diff --git a/packages/cli/src/__tests__/runtime-host-service-manager.test.ts b/packages/cli/src/__tests__/runtime-host-service-manager.test.ts index a301ba6f54..3d92256d5c 100644 --- a/packages/cli/src/__tests__/runtime-host-service-manager.test.ts +++ b/packages/cli/src/__tests__/runtime-host-service-manager.test.ts @@ -37,6 +37,7 @@ import { RUNTIME_HOST_OPERATOR_ACCESS_MANAGEMENT_CAPABILITY, RUNTIME_HOST_OPERATOR_CAPABILITY_REQUEST_ENV, RUNTIME_HOST_OPERATOR_PROCESS_LIFETIME_LOCK_CAPABILITY, + RUNTIME_HOST_SERVICE_LOG_MAX_BYTES, type RuntimeHostOperatorCapability, type RuntimeHostServiceManagementFrame, } from '@maka/runtime-host/operator'; @@ -69,7 +70,11 @@ import { import { createSystemdUserRuntimeHostService, renderSystemdUnit, + renderSystemdUpdateService, + renderSystemdUpdateTimer, resolveSystemdUserRuntimeHostServicePath, + resolveSystemdUserRuntimeHostUpdateServicePath, + resolveSystemdUserRuntimeHostUpdateTimerPath, } from '../runtime-host-systemd-service.js'; describe('managed Runtime Host service', () => { @@ -329,6 +334,14 @@ describe('managed Runtime Host service', () => { assert.equal(installed.service.enabled, true); assert.equal(installed.service.config?.websocket.port, 47_777); assert.match(await readFile(unitPath, 'utf8'), /ExecStart=.*runtime-host.*serve/u); + const updateServicePath = resolveSystemdUserRuntimeHostUpdateServicePath( + serviceId, + env, + homeDir, + ); + const updateTimerPath = resolveSystemdUserRuntimeHostUpdateTimerPath(serviceId, env, homeDir); + assert.match(await readFile(updateServicePath, 'utf8'), /operator.*reconcile-update/u); + assert.match(await readFile(updateTimerPath, 'utf8'), /^OnUnitInactiveSec=86400s$/mu); const resetFailed = systemd.calls.findIndex(([command]) => command === 'reset-failed'); const restart = systemd.calls.findIndex(([command]) => command === 'restart'); assert.ok(resetFailed >= 0 && resetFailed < restart); @@ -343,6 +356,81 @@ describe('managed Runtime Host service', () => { { label: 'Projects', path: await realpath(projectPath) }, ]); assert.equal(reinstalled.service.lastExitCode, 0); + assert.equal( + systemd.calls.filter( + ([command, target]) => command === 'restart' && target === basename(updateTimerPath), + ).length, + 1, + ); + const managedConfig = reinstalled.service.config; + assert.ok(managedConfig); + const repairBackend = backend(); + const updateTimerName = basename(updateTimerPath); + systemd.setUnitDropInPaths(updateTimerName, ['/tmp/update-timer-override.conf']); + await assert.rejects(repairBackend.install(managedConfig), /systemd drop-in overrides/u); + systemd.setUnitDropInPaths(updateTimerName, []); + await writeFile(updateTimerPath, '[Timer]\n# stale\n', 'utf8'); + await assert.rejects( + repairBackend.verifyReplacementPreconditions(managedConfig), + (error: unknown) => + error instanceof RuntimeHostServiceManagerError && error.code === 'target_mismatch', + ); + await assert.rejects( + repairBackend.verifyDeployment(managedConfig), + (error: unknown) => + error instanceof RuntimeHostServiceManagerError && error.code === 'target_mismatch', + ); + await repairBackend.install(managedConfig); + await repairBackend.verifyDeployment(managedConfig); + + const updateServiceName = basename(updateServicePath); + systemd.activateUnitWhenStopping(updateTimerName, updateServiceName); + await repairBackend.stop(); + assert.ok( + systemd.calls.some( + ([command, ...targets]) => + command === 'stop' && + targets.includes(updateTimerName) && + targets.includes(updateServiceName), + ), + ); + await repairBackend.verifyDeployment(managedConfig); + await assert.rejects( + repairBackend.verifyDeployment(managedConfig, { requireSchedulerReady: true }), + (error: unknown) => + error instanceof RuntimeHostServiceManagerError && error.code === 'target_mismatch', + ); + await repairBackend.replace(managedConfig); + assert.ok( + systemd.calls.some(([command, target]) => command === 'start' && target === updateTimerName), + ); + await repairBackend.verifyDeployment(managedConfig, { requireSchedulerReady: true }); + + const diagnosticBackend = createSystemdUserRuntimeHostService(serviceId, { + env, + homeDir, + uid: 1000, + runSystemctl: systemd.run, + runLoginctl: async () => success('yes\n'), + runJournalctl: async (args) => + success( + args.includes(updateServiceName) + ? 'scheduler reconciliation failed' + : 'h'.repeat(RUNTIME_HOST_SERVICE_LOG_MAX_BYTES), + ), + }); + const logs = await diagnosticBackend.logs(); + assert.match(logs, /scheduler reconciliation failed/u); + assert.ok(Buffer.byteLength(logs) <= RUNTIME_HOST_SERVICE_LOG_MAX_BYTES); + + const { managedDeploymentRoot: _managedDeploymentRoot, ...unmanagedConfig } = managedConfig; + await repairBackend.install(unmanagedConfig); + await repairBackend.verifyDeployment(unmanagedConfig); + await assert.rejects(readFile(updateServicePath, 'utf8'), { code: 'ENOENT' }); + await assert.rejects(readFile(updateTimerPath, 'utf8'), { code: 'ENOENT' }); + await repairBackend.verifyReplacementPreconditions(managedConfig); + await repairBackend.replace(managedConfig); + await repairBackend.verifyDeployment(managedConfig); const root = await resolveStorageRoot({ path: rootPath, kind: 'interactive' }); await writeFile(configPath, '{not-json', 'utf8'); @@ -503,6 +591,8 @@ describe('managed Runtime Host service', () => { await access(movedRootPath); await assert.rejects(access(configPath)); await assert.rejects(access(unitPath)); + await assert.rejects(access(updateServicePath)); + await assert.rejects(access(updateTimerPath)); await assert.rejects(access(deploymentRoot)); const repeated = await manageRuntimeHostService({ ...common, action: 'uninstall' }, backend()); @@ -654,6 +744,15 @@ describe('managed Runtime Host service', () => { assert.match(unit, /^Restart=on-failure$/mu); assert.match(unit, /^StartLimitIntervalSec=60s$/mu); assert.match(unit, /^StartLimitBurst=5$/mu); + + const managed = { ...config, managedDeploymentRoot: '/opt/Maka/$managed root' }; + const updateService = renderSystemdUpdateService(managed); + const updateTimer = renderSystemdUpdateTimer('a'.repeat(64)); + assert.match(updateService, /"\/opt\/Maka\/\$\$managed root\/operator"/u); + assert.match(updateService, /"reconcile-update" "--framed"/u); + assert.match(updateTimer, /^OnActiveSec=900s$/mu); + assert.match(updateTimer, /^OnUnitInactiveSec=86400s$/mu); + assert.match(updateTimer, /^RandomizedDelaySec=3600s$/mu); }); it('emits one stable machine error for an unmet service prerequisite', async () => { @@ -989,6 +1088,53 @@ describe('managed Runtime Host service', () => { assert.equal(repaired.service.config?.rootPath, root.canonicalPath); }); + it('stops partial deployment state when start or restart fails', async (t) => { + const base = await mkdtemp(join(tmpdir(), 'maka-runtime-host-service-start-failure-')); + t.after(() => rm(base, { recursive: true, force: true })); + const cliPath = join(base, 'cli.js'); + await writeFile(cliPath, '#!/usr/bin/env node\n', 'utf8'); + let stops = 0; + let stopFails = false; + const failedAction = async () => { + throw new Error('scheduler start failed'); + }; + const backend: RuntimeHostServiceBackend = { + ...createReadyBackend(), + start: failedAction, + restart: failedAction, + stop: async () => { + stops += 1; + if (stopFails) throw new Error('partial deployment stop failed'); + }, + }; + const common = { + clientDataRoot: join(base, 'config'), + defaultRootPath: join(base, 'state'), + nodePath: process.execPath, + cliPath, + } as const; + await manageRuntimeHostService({ ...common, action: 'install' }, backend, { + waitForReady: async () => undefined, + }); + + await assert.rejects( + manageRuntimeHostService({ ...common, action: 'start' }, backend), + /scheduler start failed/u, + ); + assert.equal(stops, 1); + + stopFails = true; + await assert.rejects( + manageRuntimeHostService({ ...common, action: 'restart' }, backend), + (error: unknown) => + error instanceof RuntimeHostServiceManagerError && + error.code === 'service_manager_operation_failed' && + error.cause instanceof AggregateError && + error.cause.errors.length === 2, + ); + assert.equal(stops, 2); + }); + it('retires the exact managed Host only after active work is authorized', async (t) => { const base = await mkdtemp(join(tmpdir(), 'maka-runtime-host-retirement-')); t.after(() => rm(base, { recursive: true, force: true })); @@ -1012,7 +1158,7 @@ describe('managed Runtime Host service', () => { pid: serviceState === 'running' ? 42 : serviceState === 'starting' ? startingPid : null, lastExitCode: 0, }), - stop: async () => { + retire: async () => { stops += 1; if (serviceState === 'starting' && startingPid === null) { const contender = await tryAcquireInteractiveRootOwner(root); @@ -1179,7 +1325,7 @@ describe('managed Runtime Host service', () => { pid: serviceState === 'running' ? servicePid : null, lastExitCode: 0, }), - stop: async () => { + retire: async () => { stops += 1; serviceState = 'stopped'; servicePid = null; @@ -1334,7 +1480,7 @@ describe('managed Runtime Host service', () => { if (replaceFails) throw new Error('replacement was not committed'); state = 'running'; }, - stop: async () => { + retire: async () => { if (stopFails) throw new Error('replacement could not be stopped'); state = 'stopped'; }, @@ -1443,6 +1589,7 @@ describe('managed Runtime Host service', () => { let legacyLeaseCalls = 0; let operatorStatusFailure = false; let operatorFailure: Extract | undefined; + let replacementPreconditionFailure = false; let replaceFailure = false; let cleanupFailure = false; let expectAllowInterruptActiveTasks = true; @@ -1477,7 +1624,21 @@ describe('managed Runtime Host service', () => { }, }); const overrides = { - createBackend: createUnusedBackend, + createBackend: () => ({ + ...createUnusedBackend(), + verifyReplacementPreconditions: async () => { + if (replacementPreconditionFailure) { + throw new RuntimeHostServiceManagerError( + 'target_mismatch', + 'The update scheduler is not ready for replacement', + ); + } + }, + retire: async () => { + assert.equal(insideLifecycle, true); + order.push('force-retire'); + }, + }), withLifecycleLock: async (_root: string, operation: () => Promise) => { assert.equal(insideLifecycle, false); insideLifecycle = true; @@ -1623,6 +1784,19 @@ describe('managed Runtime Host service', () => { 'updated', ); + replacementPreconditionFailure = true; + order.length = 0; + statusReads = 0; + output = ''; + assert.equal(await runManagedRuntimeHostUpdateCli(options, overrides), 1); + assert.deepEqual(order, []); + const preconditionFailure = decodeRuntimeHostServiceManagementFrame(output.trim()); + assert.equal( + preconditionFailure?.kind === 'error' ? preconditionFailure.error.code : undefined, + 'target_mismatch', + ); + replacementPreconditionFailure = false; + order.length = 0; statusReads = 0; observedVersion = '2.0.0'; @@ -1740,7 +1914,7 @@ describe('managed Runtime Host service', () => { output = ''; operatorStatusFailure = true; assert.equal(await runManagedRuntimeHostUpdateCli(options, overrides), 0); - assert.deepEqual(order, ['stop', 'activate', 'replace', 'cleanup']); + assert.deepEqual(order, ['force-retire', 'activate', 'replace', 'cleanup']); operatorStatusFailure = false; order.length = 0; @@ -1771,7 +1945,7 @@ describe('managed Runtime Host service', () => { output = ''; expectAllowInterruptActiveTasks = true; assert.equal(await runManagedRuntimeHostUpdateCli(options, overrides), 0); - assert.deepEqual(order, ['retire', 'stop', 'activate', 'replace', 'cleanup']); + assert.deepEqual(order, ['retire', 'force-retire', 'activate', 'replace', 'cleanup']); assert.equal(legacyLeaseCalls, 1); statusReads = 0; @@ -1973,7 +2147,9 @@ describe('managed Runtime Host service', () => { function createFakeSystemd(unitPath: string): { readonly failNext: (command: string) => void; + readonly activateUnitWhenStopping: (triggerUnit: string, unitToActivate: string) => void; readonly setDropInPaths: (paths: readonly string[]) => void; + readonly setUnitDropInPaths: (unitName: string, paths: readonly string[]) => void; readonly calls: readonly (readonly string[])[]; readonly run: (args: readonly string[]) => Promise<{ exitCode: number; @@ -1981,71 +2157,91 @@ function createFakeSystemd(unitPath: string): { stderr: string; }>; } { - let loaded = false; - let enabled = false; - let active = false; + const states = new Map(); let failureCommand: string | undefined; - let dropInPaths: readonly string[] = []; + const dropInPaths = new Map(); + let stopActivation: { triggerUnit: string; unitToActivate: string } | undefined; const calls: string[][] = []; return { calls, failNext: (command) => { failureCommand = command; }, + activateUnitWhenStopping: (triggerUnit, unitToActivate) => { + stopActivation = { triggerUnit, unitToActivate }; + }, setDropInPaths: (paths) => { - dropInPaths = paths; + dropInPaths.set(basename(unitPath), paths); + }, + setUnitDropInPaths: (unitName, paths) => { + dropInPaths.set(unitName, paths); }, run: async (args) => { calls.push([...args]); - if ( - ['show', 'enable', 'disable', 'start', 'restart', 'stop', 'reset-failed'].includes( - args[0] ?? '', - ) - ) { - assert.equal(args[1], basename(unitPath)); - } + const unitName = args[1]; + const unitState = unitName + ? (states.get(unitName) ?? { enabled: false, active: false }) + : undefined; + if (unitName && unitState) states.set(unitName, unitState); if (args[0] === failureCommand) { failureCommand = undefined; return { exitCode: 1, stdout: '', stderr: `${args[0]} failed` }; } if (args[0] === 'show-environment') return success('PATH=/usr/bin\n'); - if (args[0] === 'daemon-reload') { - loaded = await access(unitPath).then( - () => true, - () => false, - ); - return success(); - } + if (args[0] === 'daemon-reload') return success(); if (args[0] === 'enable') { - enabled = true; + assert.ok(unitState); + unitState.enabled = true; return success(); } if (args[0] === 'disable') { - enabled = false; + assert.ok(unitState); + unitState.enabled = false; return success(); } if (args[0] === 'start' || args[0] === 'restart') { - active = true; - loaded = true; + assert.ok(unitState); + unitState.active = true; return success(); } if (args[0] === 'stop') { - active = false; + const targets = args.slice(1); + if (stopActivation && targets.includes(stopActivation.triggerUnit)) { + const activated = states.get(stopActivation.unitToActivate) ?? { + enabled: false, + active: false, + }; + activated.active = true; + states.set(stopActivation.unitToActivate, activated); + stopActivation = undefined; + } + for (const target of targets) { + const targetState = states.get(target) ?? { enabled: false, active: false }; + targetState.active = false; + states.set(target, targetState); + } return success(); } if (args[0] === 'reset-failed') return success(); if (args[0] === 'show') { + assert.ok(unitName && unitState); + const path = join(dirname(unitPath), unitName); + const loaded = await access(path).then( + () => true, + () => false, + ); + const isMainService = unitName === basename(unitPath); return { exitCode: loaded ? 0 : 4, stdout: [ `LoadState=${loaded ? 'loaded' : 'not-found'}`, - `ActiveState=${active ? 'active' : 'inactive'}`, - `SubState=${active ? 'running' : 'dead'}`, - `UnitFileState=${enabled ? 'enabled' : 'disabled'}`, - `FragmentPath=${loaded ? unitPath : ''}`, + `ActiveState=${unitState.active ? 'active' : 'inactive'}`, + `SubState=${unitState.active ? 'running' : 'dead'}`, + `UnitFileState=${unitState.enabled ? 'enabled' : 'disabled'}`, + `FragmentPath=${loaded ? path : ''}`, 'NeedDaemonReload=no', - `DropInPaths=${dropInPaths.join(' ')}`, - `MainPID=${active ? '4242' : '0'}`, + `DropInPaths=${dropInPaths.get(unitName)?.join(' ') ?? ''}`, + `MainPID=${unitState.active && isMainService ? '4242' : '0'}`, 'ExecMainStatus=0', '', ].join('\n'), @@ -2065,11 +2261,13 @@ function createUnusedBackend(): RuntimeHostServiceBackend { preflightInstall: unexpected, install: unexpected, replace: unexpected, + verifyReplacementPreconditions: unexpected, verifyDeployment: unexpected, status: unexpected, start: unexpected, stop: unexpected, restart: unexpected, + retire: unexpected, logs: unexpected, uninstall: unexpected, }; @@ -2096,11 +2294,13 @@ function createReadyBackend(): RuntimeHostServiceBackend { preflightInstall: async () => undefined, install: async () => ({ rollback: async () => undefined }), replace: async () => undefined, + verifyReplacementPreconditions: async () => undefined, verifyDeployment: async () => undefined, status, start: async () => undefined, stop: async () => undefined, restart: async () => undefined, + retire: async () => undefined, logs: async () => '', uninstall: async () => undefined, }; diff --git a/packages/cli/src/__tests__/runtime-host-setup.test.ts b/packages/cli/src/__tests__/runtime-host-setup.test.ts index f68ec7f934..02efbbb5ed 100644 --- a/packages/cli/src/__tests__/runtime-host-setup.test.ts +++ b/packages/cli/src/__tests__/runtime-host-setup.test.ts @@ -599,11 +599,13 @@ function unusedBackend(): RuntimeHostServiceBackend { preflightInstall: async () => undefined, install: async () => assert.fail('Backend is not expected'), replace: async () => assert.fail('Backend is not expected'), + verifyReplacementPreconditions: async () => assert.fail('Backend is not expected'), verifyDeployment: async () => assert.fail('Backend is not expected'), status: async () => assert.fail('Backend is not expected'), start: async () => assert.fail('Backend is not expected'), stop: async () => assert.fail('Backend is not expected'), restart: async () => assert.fail('Backend is not expected'), + retire: async () => assert.fail('Backend is not expected'), logs: async () => assert.fail('Backend is not expected'), uninstall: async () => assert.fail('Backend is not expected'), }; diff --git a/packages/cli/src/__tests__/runtime-host-update-reconciliation.test.ts b/packages/cli/src/__tests__/runtime-host-update-reconciliation.test.ts index 5f668a0a3b..c7120006a0 100644 --- a/packages/cli/src/__tests__/runtime-host-update-reconciliation.test.ts +++ b/packages/cli/src/__tests__/runtime-host-update-reconciliation.test.ts @@ -403,6 +403,7 @@ function unusedBackend() { preflightInstall: async () => undefined, install: async () => ({ rollback: async () => undefined }), replace: async () => undefined, + verifyReplacementPreconditions: async () => undefined, verifyDeployment: async () => undefined, status: async () => ({ manager: 'systemd_user' as const, @@ -416,6 +417,7 @@ function unusedBackend() { start: async () => undefined, stop: async () => undefined, restart: async () => undefined, + retire: async () => undefined, logs: async () => '', uninstall: async () => undefined, }; diff --git a/packages/cli/src/runtime-host-launch-agent-service.ts b/packages/cli/src/runtime-host-launch-agent-service.ts index f1414efd99..9916e2fadb 100644 --- a/packages/cli/src/runtime-host-launch-agent-service.ts +++ b/packages/cli/src/runtime-host-launch-agent-service.ts @@ -22,6 +22,7 @@ import { homedir } from 'node:os'; import { dirname, join } from 'node:path'; import { RUNTIME_HOST_SERVICE_LOG_MAX_BYTES } from '@maka/runtime-host/operator'; import { + formatRuntimeHostServiceLogs, removeRuntimeHostServiceFile, RuntimeHostServiceManagerError, type RuntimeHostManagedServiceConfig, @@ -31,7 +32,9 @@ import { writeRuntimeHostServiceFile, } from './runtime-host-service-manager.js'; import { + RUNTIME_HOST_UPDATE_INTERVAL_SECONDS, runtimeHostServiceLaunchArguments, + runtimeHostUpdateReconcileLaunchArguments, validateRuntimeHostServiceLaunch, } from './runtime-host-service-launch.js'; import { @@ -97,6 +100,13 @@ export function createLaunchAgentRuntimeHostService( runLaunchctl, isProcessAlive, }; + const scheduler = resolveLaunchAgentUpdateSchedulerContext( + serviceId, + homeDir, + uid, + runLaunchctl, + isProcessAlive, + ); const readDetailedStatus = () => readLaunchAgentStatus(context); const readStatus = async (): Promise => { @@ -107,31 +117,65 @@ export function createLaunchAgentRuntimeHostService( preflightInstall: () => assertLaunchAgentDomain(context), install: async (config) => { await validateRuntimeHostServiceLaunch(config); - const previous = await captureLaunchAgentDeployment(context); + const [previous, previousScheduler] = await Promise.all([ + captureLaunchAgentDeployment(context), + captureLaunchAgentDeployment(scheduler), + ]); + let schedulerMutationStarted = false; try { await applyLaunchAgentDeployment(context, config); + await applyLaunchAgentUpdateSchedulerDesiredState(scheduler, config, () => { + schedulerMutationStarted = true; + }); } catch (error) { - await restoreFailedLaunchAgentDeployment(previous, context, error); + await restoreFailedLaunchAgentDeployment( + previous, + schedulerMutationStarted ? previousScheduler : undefined, + context, + scheduler, + error, + ); } let rolledBack = false; return { rollback: async () => { if (rolledBack) return; rolledBack = true; - await restoreLaunchAgentDeployment(previous, context); + await restoreLaunchAgentManagedDeployment( + previous, + schedulerMutationStarted ? previousScheduler : undefined, + context, + scheduler, + ); }, } satisfies RuntimeHostServiceDeployment; }, replace: async (config) => { await validateRuntimeHostServiceLaunch(config); - const previous = await captureLaunchAgentDeployment(context); + const [previous, previousScheduler] = await Promise.all([ + captureLaunchAgentDeployment(context), + captureLaunchAgentDeployment(scheduler), + ]); + let schedulerMutationStarted = false; try { await applyLaunchAgentDeployment(context, config); + await convergeLaunchAgentUpdateSchedulerForReplacement(scheduler, config, () => { + schedulerMutationStarted = true; + }); } catch (error) { - await restoreFailedLaunchAgentDeployment(previous, context, error, 'update_incomplete'); + await restoreFailedLaunchAgentDeployment( + previous, + schedulerMutationStarted ? previousScheduler : undefined, + context, + scheduler, + error, + 'update_incomplete', + ); } }, - verifyDeployment: async (config) => { + verifyReplacementPreconditions: (config) => + verifyLaunchAgentUpdateSchedulerReplacementState(scheduler, config), + verifyDeployment: async (config, options) => { await validateRuntimeHostServiceLaunch(config); const [status, plist] = await Promise.all([ readDetailedStatus(), @@ -146,57 +190,39 @@ export function createLaunchAgentRuntimeHostService( 'The installed Runtime Host LaunchAgent does not match its managed deployment', ); } + await verifyLaunchAgentUpdateSchedulerDesiredState( + scheduler, + config, + options?.requireSchedulerReady ?? false, + ); }, status: readStatus, start: async () => { - const status = await readDetailedStatus(); - if (!status.installed) { - throw new RuntimeHostServiceManagerError( - 'not_installed', - 'Runtime Host LaunchAgent is not installed', - ); - } - if (status.loaded) { - if (!status.active) { - await requireLaunchctl( - context, - ['kickstart', context.serviceTarget], - 'Starting the Runtime Host LaunchAgent failed', - ); - } - return; - } - await bootstrapLaunchAgent(context); + await startLaunchAgent(context); + await ensureLaunchAgentLoadedIfInstalled(scheduler); }, - stop: () => bootoutLaunchAgent(context), + stop: () => stopLaunchAgentManagedDeployment(context, scheduler), restart: async () => { - const status = await readDetailedStatus(); - if (!status.installed) { - throw new RuntimeHostServiceManagerError( - 'not_installed', - 'Runtime Host LaunchAgent is not installed', - ); - } - if (!status.loaded) { - await bootstrapLaunchAgent(context); - return; - } - await requireLaunchctl( - context, - ['kickstart', '-k', context.serviceTarget], - 'Restarting the Runtime Host LaunchAgent failed', - ); + await restartLaunchAgent(context); + await ensureLaunchAgentLoadedIfInstalled(scheduler); }, + retire: () => bootoutLaunchAgent(context), logs: async () => { - const [stdout, stderr] = await Promise.all([ + const [stdout, stderr, updateStdout, updateStderr] = await Promise.all([ readLogTail(context.stdoutPath), readLogTail(context.stderrPath), + readLogTail(scheduler.stdoutPath), + readLogTail(scheduler.stderrPath), + ]); + return formatRuntimeHostServiceLogs([ + { label: 'stdout', logs: stdout }, + { label: 'stderr', logs: stderr }, + { label: 'update stdout', logs: updateStdout }, + { label: 'update stderr', logs: updateStderr }, ]); - return [stdout && `stdout:\n${stdout}`, stderr && `stderr:\n${stderr}`] - .filter(Boolean) - .join('\n'); }, uninstall: async () => { + await removeLaunchAgentUpdateScheduler(scheduler); await bootoutLaunchAgent(context); await Promise.all([ removeRuntimeHostServiceFile(context.plistPath, 'LaunchAgent plist'), @@ -218,6 +244,15 @@ export function resolveLaunchAgentPath(serviceId: string, homeDir = homedir()): return join(homeDir, 'Library', 'LaunchAgents', `${resolveLaunchAgentLabel(serviceId)}.plist`); } +export function resolveLaunchAgentUpdatePath(serviceId: string, homeDir = homedir()): string { + return join( + homeDir, + 'Library', + 'LaunchAgents', + `${resolveLaunchAgentUpdateLabel(serviceId)}.plist`, + ); +} + export function renderLaunchAgentPlist( config: RuntimeHostManagedServiceConfig, paths: Pick, @@ -256,11 +291,50 @@ export function renderLaunchAgentPlist( ].join('\n'); } +export function renderLaunchAgentUpdatePlist( + config: RuntimeHostManagedServiceConfig, + paths: Pick, +): string { + const args = runtimeHostUpdateReconcileLaunchArguments(config); + if (!args) throw new TypeError('Managed deployment root is required for update scheduling'); + const stringEntry = (value: string) => ` ${escapeXml(value)}`; + return [ + '', + '', + '', + '', + ' Label', + ` ${escapeXml(paths.label)}`, + ' ProgramArguments', + ' ', + args.map(stringEntry).join('\n'), + ' ', + ' StartInterval', + ` ${String(RUNTIME_HOST_UPDATE_INTERVAL_SECONDS)}`, + ' ProcessType', + ' Background', + ' Umask', + ' 63', + ' StandardOutPath', + ` ${escapeXml(paths.stdoutPath)}`, + ' StandardErrorPath', + ` ${escapeXml(paths.stderrPath)}`, + '', + '', + '', + ].join('\n'); +} + function resolveLaunchAgentLabel(serviceId: string): string { if (!/^[0-9a-f]{64}$/u.test(serviceId)) throw new TypeError('Invalid Runtime Host service ID'); return `com.maka.runtime-host.${serviceId}`; } +function resolveLaunchAgentUpdateLabel(serviceId: string): string { + if (!/^[0-9a-f]{64}$/u.test(serviceId)) throw new TypeError('Invalid Runtime Host service ID'); + return `${resolveLaunchAgentLabel(serviceId)}.update`; +} + function resolveLaunchAgentLogPath( serviceId: string, stream: 'stdout' | 'stderr', @@ -276,6 +350,41 @@ function resolveLaunchAgentLogPath( ); } +function resolveLaunchAgentUpdateSchedulerContext( + serviceId: string, + homeDir: string, + uid: number, + runLaunchctl: LaunchctlRunner, + isProcessAlive: (pid: number) => boolean, +): LaunchAgentContext { + const label = resolveLaunchAgentUpdateLabel(serviceId); + return { + domain: `gui/${String(uid)}`, + label, + serviceTarget: `gui/${String(uid)}/${label}`, + plistPath: resolveLaunchAgentUpdatePath(serviceId, homeDir), + stdoutPath: resolveLaunchAgentUpdateLogPath(serviceId, 'stdout', homeDir), + stderrPath: resolveLaunchAgentUpdateLogPath(serviceId, 'stderr', homeDir), + runLaunchctl, + isProcessAlive, + }; +} + +function resolveLaunchAgentUpdateLogPath( + serviceId: string, + stream: 'stdout' | 'stderr', + homeDir: string, +): string { + return join( + homeDir, + 'Library', + 'Logs', + 'Maka', + 'runtime-host-services', + `${resolveLaunchAgentUpdateLabel(serviceId)}.${stream}.log`, + ); +} + async function captureLaunchAgentDeployment( context: LaunchAgentContext, ): Promise { @@ -289,6 +398,141 @@ async function captureLaunchAgentDeployment( return { plist, loaded: status.loaded }; } +async function applyLaunchAgentUpdateSchedulerDesiredState( + context: LaunchAgentContext, + config: RuntimeHostManagedServiceConfig, + onMutation: () => void, +): Promise { + if (!runtimeHostUpdateReconcileLaunchArguments(config)) { + try { + await verifyLaunchAgentUpdateSchedulerAbsent(context); + return; + } catch (error) { + if (!isTargetMismatch(error)) throw error; + } + onMutation(); + await removeLaunchAgentUpdateScheduler(context); + await verifyLaunchAgentUpdateSchedulerAbsent(context); + return; + } + try { + await verifyLaunchAgentUpdateScheduler(context, config, true); + return; + } catch (error) { + if (!isTargetMismatch(error)) throw error; + } + onMutation(); + await bootoutLaunchAgent(context); + await prepareLaunchAgentLogs(context); + await writeRuntimeHostServiceFile( + context.plistPath, + renderLaunchAgentUpdatePlist(config, context), + 0o600, + ); + await bootstrapLaunchAgent(context); + await verifyLaunchAgentUpdateScheduler(context, config, true); +} + +async function verifyLaunchAgentUpdateSchedulerDesiredState( + context: LaunchAgentContext, + config: RuntimeHostManagedServiceConfig, + requireLoaded: boolean, +): Promise { + if (runtimeHostUpdateReconcileLaunchArguments(config)) { + await verifyLaunchAgentUpdateScheduler(context, config, requireLoaded); + return; + } + await verifyLaunchAgentUpdateSchedulerAbsent(context); +} + +async function verifyLaunchAgentUpdateSchedulerReplacementState( + context: LaunchAgentContext, + config: RuntimeHostManagedServiceConfig, +): Promise { + if (!runtimeHostUpdateReconcileLaunchArguments(config)) { + await verifyLaunchAgentUpdateSchedulerAbsent(context); + return; + } + try { + await verifyLaunchAgentUpdateScheduler(context, config, false); + } catch (error) { + if (!isTargetMismatch(error)) throw error; + await verifyLaunchAgentUpdateSchedulerAbsent(context); + } +} + +async function convergeLaunchAgentUpdateSchedulerForReplacement( + context: LaunchAgentContext, + config: RuntimeHostManagedServiceConfig, + onMutation: () => void, +): Promise { + try { + await verifyLaunchAgentUpdateScheduler(context, config, false); + const status = await readLaunchAgentStatus(context); + // The loaded scheduler may be running this replacement. + if (status.loaded) return; + onMutation(); + await ensureLaunchAgentLoadedIfInstalled(context); + } catch (error) { + if (!isTargetMismatch(error)) throw error; + await verifyLaunchAgentUpdateSchedulerAbsent(context); + onMutation(); + await applyLaunchAgentUpdateSchedulerDesiredState(context, config, () => undefined); + } + await verifyLaunchAgentUpdateScheduler(context, config, true); +} + +async function verifyLaunchAgentUpdateScheduler( + context: LaunchAgentContext, + config: RuntimeHostManagedServiceConfig, + requireLoaded: boolean, +): Promise { + const [status, plist] = await Promise.all([ + readLaunchAgentStatus(context), + readFile(context.plistPath, 'utf8').catch((error: unknown) => { + if (isNodeError(error, 'ENOENT')) return null; + throw error; + }), + ]); + if ( + (requireLoaded && !status.loaded) || + plist !== renderLaunchAgentUpdatePlist(config, context) + ) { + throw launchAgentSchedulerMismatch(); + } +} + +async function verifyLaunchAgentUpdateSchedulerAbsent(context: LaunchAgentContext): Promise { + const [status, plist] = await Promise.all([ + readLaunchAgentStatus(context), + readFile(context.plistPath, 'utf8').catch((error: unknown) => { + if (isNodeError(error, 'ENOENT')) return null; + throw error; + }), + ]); + if (status.loaded || plist !== null) throw launchAgentSchedulerMismatch(); +} + +async function removeLaunchAgentUpdateScheduler(context: LaunchAgentContext): Promise { + await bootoutLaunchAgent(context); + await Promise.all([ + removeRuntimeHostServiceFile(context.plistPath, 'LaunchAgent update plist'), + removeRuntimeHostServiceFile(context.stdoutPath, 'LaunchAgent update stdout log'), + removeRuntimeHostServiceFile(context.stderrPath, 'LaunchAgent update stderr log'), + ]); +} + +function launchAgentSchedulerMismatch(): RuntimeHostServiceManagerError { + return new RuntimeHostServiceManagerError( + 'target_mismatch', + 'The Runtime Host update scheduler does not match its managed deployment', + ); +} + +function isTargetMismatch(error: unknown): boolean { + return error instanceof RuntimeHostServiceManagerError && error.code === 'target_mismatch'; +} + async function applyLaunchAgentDeployment( context: LaunchAgentContext, config: RuntimeHostManagedServiceConfig, @@ -303,16 +547,89 @@ async function applyLaunchAgentDeployment( await bootstrapLaunchAgent(context); } +async function startLaunchAgent(context: LaunchAgentContext): Promise { + const status = await readLaunchAgentStatus(context); + if (!status.installed) { + throw new RuntimeHostServiceManagerError( + 'not_installed', + 'Runtime Host LaunchAgent is not installed', + ); + } + if (!status.loaded) { + await bootstrapLaunchAgent(context); + return; + } + if (!status.active) { + await requireLaunchctl( + context, + ['kickstart', context.serviceTarget], + 'Starting the Runtime Host LaunchAgent failed', + ); + } +} + +async function restartLaunchAgent(context: LaunchAgentContext): Promise { + const status = await readLaunchAgentStatus(context); + if (!status.installed) { + throw new RuntimeHostServiceManagerError( + 'not_installed', + 'Runtime Host LaunchAgent is not installed', + ); + } + if (!status.loaded) { + await bootstrapLaunchAgent(context); + return; + } + await requireLaunchctl( + context, + ['kickstart', '-k', context.serviceTarget], + 'Restarting the Runtime Host LaunchAgent failed', + ); +} + +async function ensureLaunchAgentLoadedIfInstalled(context: LaunchAgentContext): Promise { + const status = await readLaunchAgentStatus(context); + if (status.installed && !status.loaded) await bootstrapLaunchAgent(context); +} + +async function stopLaunchAgentManagedDeployment( + context: LaunchAgentContext, + scheduler: LaunchAgentContext, +): Promise { + const errors: unknown[] = []; + for (const target of [scheduler, context]) { + try { + await bootoutLaunchAgent(target); + } catch (error) { + errors.push(error); + } + } + if (errors.length > 0) { + throw new RuntimeHostServiceManagerError( + 'service_manager_operation_failed', + 'Unable to stop the Runtime Host managed deployment', + { cause: new AggregateError(errors) }, + ); + } +} + async function restoreFailedLaunchAgentDeployment( snapshot: LaunchAgentDeploymentSnapshot, + schedulerSnapshot: LaunchAgentDeploymentSnapshot | undefined, context: LaunchAgentContext, + schedulerContext: LaunchAgentContext, originalError: unknown, recoveryFailureCode: | 'service_manager_operation_failed' | 'update_incomplete' = 'service_manager_operation_failed', ): Promise { try { - await restoreLaunchAgentDeployment(snapshot, context); + await restoreLaunchAgentManagedDeployment( + snapshot, + schedulerSnapshot, + context, + schedulerContext, + ); } catch (rollbackError) { throw new RuntimeHostServiceManagerError( recoveryFailureCode, @@ -323,6 +640,30 @@ async function restoreFailedLaunchAgentDeployment( throw originalError; } +async function restoreLaunchAgentManagedDeployment( + snapshot: LaunchAgentDeploymentSnapshot, + schedulerSnapshot: LaunchAgentDeploymentSnapshot | undefined, + context: LaunchAgentContext, + schedulerContext: LaunchAgentContext, +): Promise { + const errors: unknown[] = []; + try { + await restoreLaunchAgentDeployment(snapshot, context); + } catch (error) { + errors.push(error); + } + if (schedulerSnapshot) { + try { + await restoreLaunchAgentDeployment(schedulerSnapshot, schedulerContext); + } catch (error) { + errors.push(error); + } + } + if (errors.length > 0) { + throw new AggregateError(errors, 'Unable to restore the previous LaunchAgent deployment'); + } +} + async function restoreLaunchAgentDeployment( snapshot: LaunchAgentDeploymentSnapshot, context: LaunchAgentContext, diff --git a/packages/cli/src/runtime-host-service-launch.ts b/packages/cli/src/runtime-host-service-launch.ts index e70ae58d45..0324467f47 100644 --- a/packages/cli/src/runtime-host-service-launch.ts +++ b/packages/cli/src/runtime-host-service-launch.ts @@ -19,6 +19,7 @@ import { constants } from 'node:fs'; import { access, realpath, stat } from 'node:fs/promises'; +import { join } from 'node:path'; import { RuntimeHostServiceManagerError, type RuntimeHostManagedServiceConfig, @@ -48,6 +49,18 @@ export function runtimeHostServiceLaunchArguments( ]; } +export const RUNTIME_HOST_UPDATE_INTERVAL_SECONDS = 24 * 60 * 60; +export const RUNTIME_HOST_UPDATE_INITIAL_DELAY_SECONDS = 15 * 60; +export const RUNTIME_HOST_UPDATE_RANDOM_DELAY_SECONDS = 60 * 60; + +export function runtimeHostUpdateReconcileLaunchArguments( + config: RuntimeHostManagedServiceConfig, +): readonly string[] | null { + return config.managedDeploymentRoot + ? [join(config.managedDeploymentRoot, 'operator'), 'reconcile-update', '--framed'] + : null; +} + export async function validateRuntimeHostServiceLaunch( config: RuntimeHostManagedServiceConfig, ): Promise { diff --git a/packages/cli/src/runtime-host-service-manager.ts b/packages/cli/src/runtime-host-service-manager.ts index 48507c07ed..23903f2606 100644 --- a/packages/cli/src/runtime-host-service-manager.ts +++ b/packages/cli/src/runtime-host-service-manager.ts @@ -102,11 +102,19 @@ export interface RuntimeHostServiceBackend { install(config: RuntimeHostManagedServiceConfig): Promise; /** A rejected replacement must restore the previous deployment or report update_incomplete. */ replace(config: RuntimeHostManagedServiceConfig): Promise; - verifyDeployment(config: RuntimeHostManagedServiceConfig): Promise; + /** Reject partial or drifted scheduler state before replacement begins. */ + verifyReplacementPreconditions(config: RuntimeHostManagedServiceConfig): Promise; + /** Verify the deployment definition and, when requested, its scheduler readiness. */ + verifyDeployment( + config: RuntimeHostManagedServiceConfig, + options?: { readonly requireSchedulerReady?: boolean }, + ): Promise; status(): Promise; start(): Promise; stop(): Promise; restart(): Promise; + /** Stop only the Runtime Host process while preserving deployment scheduling. */ + retire(): Promise; logs(): Promise; uninstall(): Promise; } @@ -501,7 +509,7 @@ async function manageRuntimeHostServiceLocked( rootFence = await acquireRuntimeHostRootRetirementFence(root); } try { - await backend.stop(); + await backend.retire(); const stopped = await readServiceStatus(configPath, backend); if (stopped.active || stopped.state !== 'stopped' || stopped.pid !== null) { throw new RuntimeHostServiceManagerError( @@ -535,14 +543,24 @@ async function manageRuntimeHostServiceLocked( ); } await resolveExpectedServiceRoot(config, input); - await backend[input.action](); if (input.action === 'start' || input.action === 'restart') { try { + await backend[input.action](); await deps.waitForReady(config, backend); } catch (error) { - await backend.stop().catch(() => undefined); + try { + await backend.stop(); + } catch (stopError) { + throw new RuntimeHostServiceManagerError( + 'service_manager_operation_failed', + 'Starting the Runtime Host managed deployment failed and its partial state could not be stopped', + { cause: new AggregateError([error, stopError]) }, + ); + } throw error; } + } else { + await backend[input.action](); } return result(input.action, await readServiceStatus(configPath, backend)); } @@ -588,7 +606,7 @@ async function replaceRuntimeHostManagedServiceLocked( await backend.replace(config); } catch (error) { if (error instanceof RuntimeHostServiceManagerError && error.code === 'update_incomplete') { - await backend.stop().catch(() => undefined); + await backend.retire().catch(() => undefined); throw error; } try { @@ -598,7 +616,7 @@ async function replaceRuntimeHostManagedServiceLocked( 0o600, ); } catch (restoreError) { - await backend.stop().catch(() => undefined); + await backend.retire().catch(() => undefined); throw new RuntimeHostServiceManagerError( 'update_incomplete', 'Replacing the Runtime Host service failed and its previous configuration could not be restored', @@ -615,7 +633,7 @@ async function replaceRuntimeHostManagedServiceLocked( await deps.waitForReady(config, backend); } catch (error) { try { - await backend.stop(); + await backend.retire(); } catch (stopError) { throw new RuntimeHostServiceManagerError( 'update_incomplete', @@ -716,6 +734,31 @@ export async function removeRuntimeHostServiceFile(path: string, label: string): } } +export function formatRuntimeHostServiceLogs( + sources: readonly { readonly label: string; readonly logs: string }[], +): string { + const present = sources.filter(({ logs }) => logs.length > 0); + if (present.length === 0) return ''; + const separatorBytes = present.length - 1; + const sourceBudget = Math.floor( + (RUNTIME_HOST_SERVICE_LOG_MAX_BYTES - separatorBytes) / present.length, + ); + return present + .map(({ label, logs }) => { + const heading = `${label}:\n`; + return `${heading}${takeUtf8Tail(logs, sourceBudget - Buffer.byteLength(heading))}`; + }) + .join('\n'); +} + +function takeUtf8Tail(value: string, maximumBytes: number): string { + const encoded = Buffer.from(value); + if (encoded.byteLength <= maximumBytes) return value; + let start = encoded.byteLength - maximumBytes; + while (start < encoded.byteLength && (encoded[start]! & 0xc0) === 0x80) start += 1; + return encoded.subarray(start).toString('utf8'); +} + async function prepareServiceConfig( input: Omit, previous: RuntimeHostManagedServiceConfig | null, @@ -1006,7 +1049,7 @@ export async function verifyRuntimeHostManagedServiceReady( config: RuntimeHostManagedServiceConfig, backend: RuntimeHostServiceBackend, ): Promise { - await backend.verifyDeployment(config); + await backend.verifyDeployment(config, { requireSchedulerReady: true }); const deadline = Date.now() + SERVICE_READY_TIMEOUT_MS; let lastFailure = 'not available'; while (Date.now() < deadline) { diff --git a/packages/cli/src/runtime-host-systemd-service.ts b/packages/cli/src/runtime-host-systemd-service.ts index f3476fc4aa..795ceb505e 100644 --- a/packages/cli/src/runtime-host-systemd-service.ts +++ b/packages/cli/src/runtime-host-systemd-service.ts @@ -22,6 +22,7 @@ import { homedir } from 'node:os'; import { join } from 'node:path'; import { resolveXdgConfigHome } from '@maka/storage/workspace-root'; import { + formatRuntimeHostServiceLogs, removeRuntimeHostServiceFile, RuntimeHostServiceManagerError, type RuntimeHostManagedServiceConfig, @@ -31,7 +32,11 @@ import { writeRuntimeHostServiceFile, } from './runtime-host-service-manager.js'; import { + RUNTIME_HOST_UPDATE_INITIAL_DELAY_SECONDS, + RUNTIME_HOST_UPDATE_INTERVAL_SECONDS, + RUNTIME_HOST_UPDATE_RANDOM_DELAY_SECONDS, runtimeHostServiceLaunchArguments, + runtimeHostUpdateReconcileLaunchArguments, validateRuntimeHostServiceLaunch, } from './runtime-host-service-launch.js'; import { @@ -47,6 +52,12 @@ interface SystemdUnitContext { ) => Promise; } +interface SystemdUpdateSchedulerContext { + readonly serviceId: string; + readonly service: SystemdUnitContext; + readonly timer: SystemdUnitContext; +} + export interface SystemdUserServiceOptions { readonly env?: NodeJS.ProcessEnv; readonly homeDir?: string; @@ -74,6 +85,7 @@ export function createSystemdUserRuntimeHostService( unitPath: resolveSystemdUserRuntimeHostServicePath(serviceId, env, homeDir), runSystemctl, }; + const scheduler = resolveSystemdUpdateSchedulerContext(serviceId, env, homeDir, runSystemctl); const runLoginctl = options.runLoginctl ?? defaultRunLoginctl; const runJournalctl = options.runJournalctl ?? defaultRunJournalctl; const uid = options.uid ?? process.getuid?.(); @@ -98,31 +110,66 @@ export function createSystemdUserRuntimeHostService( }, install: async (config) => { await validateRuntimeHostServiceLaunch(config); - const previous = await captureSystemdDeployment(context.unitPath, readStatus); + const [previous, previousScheduler] = await Promise.all([ + captureSystemdDeployment(context.unitPath, readStatus), + captureSystemdUpdateScheduler(scheduler), + ]); + await assertNoSystemdUpdateSchedulerDropIns(scheduler); + let schedulerMutationStarted = false; try { await applySystemdDeployment(context, config); + await applySystemdUpdateSchedulerDesiredState(scheduler, config, () => { + schedulerMutationStarted = true; + }); } catch (error) { - await restoreFailedSystemdDeployment(previous, context, error); + await restoreFailedSystemdDeployment( + previous, + schedulerMutationStarted ? previousScheduler : undefined, + context, + scheduler, + error, + ); } let rolledBack = false; return { rollback: async () => { if (rolledBack) return; rolledBack = true; - await restoreSystemdDeployment(previous, context); + await restoreSystemdManagedDeployment( + previous, + schedulerMutationStarted ? previousScheduler : undefined, + context, + scheduler, + ); }, } satisfies RuntimeHostServiceDeployment; }, replace: async (config) => { await validateRuntimeHostServiceLaunch(config); - const previous = await captureSystemdDeployment(context.unitPath, readStatus); + const [previous, previousScheduler] = await Promise.all([ + captureSystemdDeployment(context.unitPath, readStatus), + captureSystemdUpdateScheduler(scheduler), + ]); + let schedulerMutationStarted = false; try { await applySystemdDeployment(context, config); + await convergeSystemdUpdateSchedulerForReplacement(scheduler, config, () => { + schedulerMutationStarted = true; + }); } catch (error) { - await restoreFailedSystemdDeployment(previous, context, error, 'update_incomplete'); + await restoreFailedSystemdDeployment( + previous, + schedulerMutationStarted ? previousScheduler : undefined, + context, + scheduler, + error, + 'update_incomplete', + ); } }, - verifyDeployment: async (config) => { + verifyReplacementPreconditions: (config) => + verifySystemdUpdateSchedulerReplacementState(scheduler, config), + verifyDeployment: async (config, options) => { await validateRuntimeHostServiceLaunch(config); const [status, unit] = await Promise.all([ readSystemdStatus(context), @@ -143,31 +190,54 @@ export function createSystemdUserRuntimeHostService( 'The loaded Runtime Host service does not match its managed deployment', ); } + await verifySystemdUpdateSchedulerDesiredState( + scheduler, + config, + options?.requireSchedulerReady ?? false, + ); }, status: readStatus, - start: () => runLifecycleAction(context, 'start'), - stop: () => runLifecycleAction(context, 'stop'), - restart: () => runLifecycleAction(context, 'restart'), + start: async () => { + await runLifecycleAction(context, 'start'); + await ensureSystemdUpdateSchedulerStartedIfInstalled(scheduler); + }, + stop: () => stopSystemdManagedDeployment(context, scheduler), + restart: async () => { + await runLifecycleAction(context, 'restart'); + await ensureSystemdUpdateSchedulerStartedIfInstalled(scheduler); + }, + retire: () => runLifecycleAction(context, 'stop'), logs: async () => { - const result = await runJournalctl([ - '--user-unit', - context.unitName, - '--no-pager', - '--lines=200', - '--output=short-iso', - ]).catch((error) => { - throw new RuntimeHostServiceManagerError( - 'service_manager_unavailable', - 'Unable to read Runtime Host service logs', - { cause: error }, - ); - }); - if (result.exitCode !== 0) { - throw managerError('Reading Runtime Host service logs failed', result); - } - return result.stdout; + const readJournal = async (unitName: string): Promise => { + const result = await runJournalctl([ + '--user-unit', + unitName, + '--no-pager', + '--lines=200', + '--output=short-iso', + ]).catch((error) => { + throw new RuntimeHostServiceManagerError( + 'service_manager_unavailable', + 'Unable to read Runtime Host service logs', + { cause: error }, + ); + }); + if (result.exitCode !== 0) { + throw managerError('Reading Runtime Host service logs failed', result); + } + return result.stdout; + }; + const [hostLogs, updateLogs] = await Promise.all([ + readJournal(context.unitName), + readJournal(scheduler.service.unitName), + ]); + return formatRuntimeHostServiceLogs([ + { label: 'host', logs: hostLogs }, + { label: 'update', logs: updateLogs }, + ]); }, uninstall: async () => { + await removeSystemdUpdateScheduler(scheduler); const before = await readSystemdStatus(context); if (before.loadState !== 'not-found') { await requireSystemctl( @@ -215,11 +285,47 @@ export function resolveSystemdUserRuntimeHostServicePath( ); } +export function resolveSystemdUserRuntimeHostUpdateServicePath( + serviceId: string, + env: NodeJS.ProcessEnv = process.env, + homeDir = homedir(), +): string { + return join( + resolveXdgConfigHome(env, homeDir), + 'systemd', + 'user', + resolveSystemdUserRuntimeHostUpdateServiceName(serviceId), + ); +} + +export function resolveSystemdUserRuntimeHostUpdateTimerPath( + serviceId: string, + env: NodeJS.ProcessEnv = process.env, + homeDir = homedir(), +): string { + return join( + resolveXdgConfigHome(env, homeDir), + 'systemd', + 'user', + resolveSystemdUserRuntimeHostUpdateTimerName(serviceId), + ); +} + function resolveSystemdUserRuntimeHostServiceName(serviceId: string): string { - if (!/^[0-9a-f]{64}$/u.test(serviceId)) throw new TypeError('Invalid Runtime Host service ID'); + assertServiceId(serviceId); return `maka-runtime-host-${serviceId}.service`; } +function resolveSystemdUserRuntimeHostUpdateServiceName(serviceId: string): string { + assertServiceId(serviceId); + return `maka-runtime-host-${serviceId}-update.service`; +} + +function resolveSystemdUserRuntimeHostUpdateTimerName(serviceId: string): string { + assertServiceId(serviceId); + return `maka-runtime-host-${serviceId}-update.timer`; +} + export function renderSystemdUnit(config: RuntimeHostManagedServiceConfig): string { const args = runtimeHostServiceLaunchArguments(config); return [ @@ -244,6 +350,39 @@ export function renderSystemdUnit(config: RuntimeHostManagedServiceConfig): stri ].join('\n'); } +export function renderSystemdUpdateService(config: RuntimeHostManagedServiceConfig): string { + const args = runtimeHostUpdateReconcileLaunchArguments(config); + if (!args) throw new TypeError('Managed deployment root is required for update scheduling'); + return [ + '[Unit]', + 'Description=Maka Runtime Host update reconciliation', + 'After=network-online.target', + '', + '[Service]', + 'Type=oneshot', + `ExecStart=${args.map(quoteSystemdArgument).join(' ')}`, + 'UMask=0077', + '', + ].join('\n'); +} + +export function renderSystemdUpdateTimer(serviceId: string): string { + return [ + '[Unit]', + 'Description=Schedule Maka Runtime Host update reconciliation', + '', + '[Timer]', + `OnActiveSec=${String(RUNTIME_HOST_UPDATE_INITIAL_DELAY_SECONDS)}s`, + `OnUnitInactiveSec=${String(RUNTIME_HOST_UPDATE_INTERVAL_SECONDS)}s`, + `RandomizedDelaySec=${String(RUNTIME_HOST_UPDATE_RANDOM_DELAY_SECONDS)}s`, + `Unit=${resolveSystemdUserRuntimeHostUpdateServiceName(serviceId)}`, + '', + '[Install]', + 'WantedBy=timers.target', + '', + ].join('\n'); +} + interface SystemdStatus { readonly loadState: string; readonly activeState: string; @@ -260,6 +399,354 @@ interface SystemdDeploymentSnapshot { readonly status: RuntimeHostServiceBackendStatus; } +interface SystemdUpdateSchedulerSnapshot { + readonly serviceUnit: string | null; + readonly timerUnit: string | null; + readonly timerStatus: SystemdStatus; +} + +function resolveSystemdUpdateSchedulerContext( + serviceId: string, + env: NodeJS.ProcessEnv, + homeDir: string, + runSystemctl: SystemdUnitContext['runSystemctl'], +): SystemdUpdateSchedulerContext { + return { + serviceId, + service: { + unitName: resolveSystemdUserRuntimeHostUpdateServiceName(serviceId), + unitPath: resolveSystemdUserRuntimeHostUpdateServicePath(serviceId, env, homeDir), + runSystemctl, + }, + timer: { + unitName: resolveSystemdUserRuntimeHostUpdateTimerName(serviceId), + unitPath: resolveSystemdUserRuntimeHostUpdateTimerPath(serviceId, env, homeDir), + runSystemctl, + }, + }; +} + +async function captureSystemdUpdateScheduler( + context: SystemdUpdateSchedulerContext, +): Promise { + const [serviceUnit, timerUnit, timerStatus] = await Promise.all([ + readOptionalFile(context.service.unitPath), + readOptionalFile(context.timer.unitPath), + readSystemdStatus(context.timer), + ]); + return { serviceUnit, timerUnit, timerStatus }; +} + +async function applySystemdUpdateSchedulerDesiredState( + context: SystemdUpdateSchedulerContext, + config: RuntimeHostManagedServiceConfig, + onMutation: () => void, +): Promise { + if (!runtimeHostUpdateReconcileLaunchArguments(config)) { + try { + await verifySystemdUpdateSchedulerAbsent(context); + return; + } catch (error) { + if (!isTargetMismatch(error)) throw error; + } + onMutation(); + await removeSystemdUpdateScheduler(context); + await verifySystemdUpdateSchedulerAbsent(context); + return; + } + try { + await verifySystemdUpdateScheduler(context, config, true); + return; + } catch (error) { + if (!isTargetMismatch(error)) throw error; + } + onMutation(); + await Promise.all([ + writeRuntimeHostServiceFile( + context.service.unitPath, + renderSystemdUpdateService(config), + 0o600, + ), + writeRuntimeHostServiceFile( + context.timer.unitPath, + renderSystemdUpdateTimer(context.serviceId), + 0o600, + ), + ]); + await requireSystemctl(context.timer.runSystemctl, ['daemon-reload'], 'Reloading systemd failed'); + await requireSystemctl( + context.timer.runSystemctl, + ['enable', context.timer.unitName], + 'Enabling Runtime Host update reconciliation failed', + ); + await context.timer.runSystemctl(['reset-failed', context.service.unitName]); + await context.timer.runSystemctl(['reset-failed', context.timer.unitName]); + await requireSystemctl( + context.timer.runSystemctl, + ['restart', context.timer.unitName], + 'Scheduling Runtime Host update reconciliation failed', + ); + await verifySystemdUpdateScheduler(context, config, true); +} + +async function assertNoSystemdUpdateSchedulerDropIns( + context: SystemdUpdateSchedulerContext, +): Promise { + const [serviceStatus, timerStatus] = await Promise.all([ + readSystemdStatus(context.service), + readSystemdStatus(context.timer), + ]); + if (serviceStatus.dropInPaths?.trim() || timerStatus.dropInPaths?.trim()) { + throw new RuntimeHostServiceManagerError( + 'target_mismatch', + 'The Runtime Host update scheduler has systemd drop-in overrides; remove them before repairing the managed deployment', + ); + } +} + +async function verifySystemdUpdateSchedulerDesiredState( + context: SystemdUpdateSchedulerContext, + config: RuntimeHostManagedServiceConfig, + requireActive: boolean, +): Promise { + if (runtimeHostUpdateReconcileLaunchArguments(config)) { + await verifySystemdUpdateScheduler(context, config, requireActive); + return; + } + await verifySystemdUpdateSchedulerAbsent(context); +} + +async function verifySystemdUpdateSchedulerReplacementState( + context: SystemdUpdateSchedulerContext, + config: RuntimeHostManagedServiceConfig, +): Promise { + if (!runtimeHostUpdateReconcileLaunchArguments(config)) { + await verifySystemdUpdateSchedulerAbsent(context); + return; + } + try { + await verifySystemdUpdateScheduler(context, config, false); + } catch (error) { + if (!isTargetMismatch(error)) throw error; + await verifySystemdUpdateSchedulerAbsent(context); + } +} + +async function convergeSystemdUpdateSchedulerForReplacement( + context: SystemdUpdateSchedulerContext, + config: RuntimeHostManagedServiceConfig, + onMutation: () => void, +): Promise { + try { + await verifySystemdUpdateScheduler(context, config, false); + const status = await readSystemdStatus(context.timer); + // The active scheduler may be running this replacement. + if (status.activeState === 'active') return; + onMutation(); + await ensureSystemdUpdateSchedulerStartedIfInstalled(context); + } catch (error) { + if (!isTargetMismatch(error)) throw error; + await verifySystemdUpdateSchedulerAbsent(context); + onMutation(); + await applySystemdUpdateSchedulerDesiredState(context, config, () => undefined); + } + await verifySystemdUpdateScheduler(context, config, true); +} + +async function verifySystemdUpdateScheduler( + context: SystemdUpdateSchedulerContext, + config: RuntimeHostManagedServiceConfig, + requireActive: boolean, +): Promise { + const [serviceUnit, timerUnit, serviceStatus, timerStatus] = await Promise.all([ + readOptionalFile(context.service.unitPath), + readOptionalFile(context.timer.unitPath), + readSystemdStatus(context.service), + readSystemdStatus(context.timer), + ]); + if ( + serviceUnit !== renderSystemdUpdateService(config) || + timerUnit !== renderSystemdUpdateTimer(context.serviceId) || + !isLoadedManagedSystemdUnit(serviceStatus, context.service.unitPath) || + !isLoadedManagedSystemdUnit(timerStatus, context.timer.unitPath) || + (timerStatus.unitFileState !== 'enabled' && timerStatus.unitFileState !== 'enabled-runtime') || + (requireActive && timerStatus.activeState !== 'active') + ) { + throw schedulerMismatch(); + } +} + +async function verifySystemdUpdateSchedulerAbsent( + context: SystemdUpdateSchedulerContext, +): Promise { + const [serviceUnit, timerUnit, serviceStatus, timerStatus] = await Promise.all([ + readOptionalFile(context.service.unitPath), + readOptionalFile(context.timer.unitPath), + readSystemdStatus(context.service), + readSystemdStatus(context.timer), + ]); + if ( + serviceUnit !== null || + timerUnit !== null || + serviceStatus.loadState !== 'not-found' || + timerStatus.loadState !== 'not-found' || + timerStatus.unitFileState === 'enabled' || + timerStatus.unitFileState === 'enabled-runtime' || + serviceStatus.dropInPaths?.trim() || + timerStatus.dropInPaths?.trim() + ) { + throw schedulerMismatch(); + } +} + +async function ensureSystemdUpdateSchedulerStartedIfInstalled( + context: SystemdUpdateSchedulerContext, +): Promise { + const status = await readSystemdStatus(context.timer); + if (status.loadState !== 'not-found' && !isSystemdUnitRunning(status)) { + await requireSystemctl( + context.timer.runSystemctl, + ['start', context.timer.unitName], + 'Starting Runtime Host update scheduling failed', + ); + } +} + +async function stopSystemdManagedDeployment( + service: SystemdUnitContext, + scheduler: SystemdUpdateSchedulerContext, +): Promise { + const errors: unknown[] = []; + try { + await stopSystemdUpdateScheduler(scheduler); + } catch (error) { + errors.push(error); + } + try { + await runLifecycleAction(service, 'stop'); + } catch (error) { + errors.push(error); + } + if (errors.length > 0) { + throw new RuntimeHostServiceManagerError( + 'service_manager_operation_failed', + 'Unable to stop the Runtime Host managed deployment', + { cause: new AggregateError(errors) }, + ); + } +} + +async function removeSystemdUpdateScheduler(context: SystemdUpdateSchedulerContext): Promise { + const timerStatus = await readSystemdStatus(context.timer); + await stopSystemdUpdateScheduler(context); + if ( + timerStatus.loadState !== 'not-found' || + timerStatus.unitFileState === 'enabled' || + timerStatus.unitFileState === 'enabled-runtime' + ) { + await requireSystemctl( + context.timer.runSystemctl, + ['disable', context.timer.unitName], + 'Disabling Runtime Host update scheduling failed', + ); + } + await Promise.all([ + removeRuntimeHostServiceFile(context.service.unitPath, 'systemd update service'), + removeRuntimeHostServiceFile(context.timer.unitPath, 'systemd update timer'), + ]); + await requireSystemctl(context.timer.runSystemctl, ['daemon-reload'], 'Reloading systemd failed'); + await context.timer.runSystemctl(['reset-failed', context.service.unitName]); + await context.timer.runSystemctl(['reset-failed', context.timer.unitName]); +} + +async function stopSystemdUpdateScheduler(context: SystemdUpdateSchedulerContext): Promise { + const [serviceUnit, timerUnit, serviceStatus, timerStatus] = await Promise.all([ + readOptionalFile(context.service.unitPath), + readOptionalFile(context.timer.unitPath), + readSystemdStatus(context.service), + readSystemdStatus(context.timer), + ]); + const units = [ + ...(timerUnit !== null || timerStatus.loadState !== 'not-found' + ? [context.timer.unitName] + : []), + ...(serviceUnit !== null || serviceStatus.loadState !== 'not-found' + ? [context.service.unitName] + : []), + ]; + if (units.length === 0) return; + await requireSystemctl( + context.timer.runSystemctl, + ['stop', ...units], + 'Stopping Runtime Host update scheduling failed', + ); +} + +async function restoreSystemdUpdateScheduler( + snapshot: SystemdUpdateSchedulerSnapshot, + context: SystemdUpdateSchedulerContext, +): Promise { + await removeSystemdUpdateScheduler(context); + if (snapshot.serviceUnit === null && snapshot.timerUnit === null) return; + await Promise.all([ + snapshot.serviceUnit === null + ? removeRuntimeHostServiceFile(context.service.unitPath, 'systemd update service') + : writeRuntimeHostServiceFile(context.service.unitPath, snapshot.serviceUnit, 0o600), + snapshot.timerUnit === null + ? removeRuntimeHostServiceFile(context.timer.unitPath, 'systemd update timer') + : writeRuntimeHostServiceFile(context.timer.unitPath, snapshot.timerUnit, 0o600), + ]); + await requireSystemctl(context.timer.runSystemctl, ['daemon-reload'], 'Reloading systemd failed'); + if (snapshot.timerUnit === null) return; + await requireSystemctl( + context.timer.runSystemctl, + [ + snapshot.timerStatus.unitFileState === 'enabled' || + snapshot.timerStatus.unitFileState === 'enabled-runtime' + ? 'enable' + : 'disable', + context.timer.unitName, + ], + 'Restoring Runtime Host update scheduling failed', + ); + await requireSystemctl( + context.timer.runSystemctl, + [snapshot.timerStatus.activeState === 'active' ? 'restart' : 'stop', context.timer.unitName], + 'Restoring Runtime Host update scheduler state failed', + ); +} + +function isLoadedManagedSystemdUnit(status: SystemdStatus, path: string): boolean { + return ( + status.loadState === 'loaded' && + status.fragmentPath === path && + status.needDaemonReload === 'no' && + !status.dropInPaths?.trim() + ); +} + +function isSystemdUnitRunning(status: SystemdStatus): boolean { + return status.activeState !== 'inactive' && status.activeState !== 'failed'; +} + +function schedulerMismatch(): RuntimeHostServiceManagerError { + return new RuntimeHostServiceManagerError( + 'target_mismatch', + 'The Runtime Host update scheduler does not match its managed deployment', + ); +} + +function isTargetMismatch(error: unknown): boolean { + return error instanceof RuntimeHostServiceManagerError && error.code === 'target_mismatch'; +} + +async function readOptionalFile(path: string): Promise { + return readFile(path, 'utf8').catch((error: unknown) => { + if (isNodeError(error, 'ENOENT')) return null; + throw error; + }); +} + async function captureSystemdDeployment( unitPath: string, readStatus: () => Promise, @@ -295,14 +782,16 @@ async function applySystemdDeployment( async function restoreFailedSystemdDeployment( snapshot: SystemdDeploymentSnapshot, + schedulerSnapshot: SystemdUpdateSchedulerSnapshot | undefined, context: SystemdUnitContext, + schedulerContext: SystemdUpdateSchedulerContext, originalError: unknown, recoveryFailureCode: | 'service_manager_operation_failed' | 'update_incomplete' = 'service_manager_operation_failed', ): Promise { try { - await restoreSystemdDeployment(snapshot, context); + await restoreSystemdManagedDeployment(snapshot, schedulerSnapshot, context, schedulerContext); } catch (rollbackError) { throw new RuntimeHostServiceManagerError( recoveryFailureCode, @@ -313,6 +802,30 @@ async function restoreFailedSystemdDeployment( throw originalError; } +async function restoreSystemdManagedDeployment( + snapshot: SystemdDeploymentSnapshot, + schedulerSnapshot: SystemdUpdateSchedulerSnapshot | undefined, + context: SystemdUnitContext, + schedulerContext: SystemdUpdateSchedulerContext, +): Promise { + const errors: unknown[] = []; + try { + await restoreSystemdDeployment(snapshot, context); + } catch (error) { + errors.push(error); + } + if (schedulerSnapshot) { + try { + await restoreSystemdUpdateScheduler(schedulerSnapshot, schedulerContext); + } catch (error) { + errors.push(error); + } + } + if (errors.length > 0) { + throw new AggregateError(errors, 'Unable to restore the previous systemd deployment'); + } +} + async function restoreSystemdDeployment( snapshot: SystemdDeploymentSnapshot, context: SystemdUnitContext, @@ -557,3 +1070,7 @@ function actionPresentParticiple(action: 'start' | 'stop' | 'restart'): string { if (action === 'stop') return 'Stopping'; return 'Restarting'; } + +function assertServiceId(serviceId: string): void { + if (!/^[0-9a-f]{64}$/u.test(serviceId)) throw new TypeError('Invalid Runtime Host service ID'); +} diff --git a/packages/cli/src/runtime-host-update-command.ts b/packages/cli/src/runtime-host-update-command.ts index 18a634b51a..f9c733451d 100644 --- a/packages/cli/src/runtime-host-update-command.ts +++ b/packages/cli/src/runtime-host-update-command.ts @@ -210,6 +210,7 @@ export async function runManagedRuntimeHostUpdateCli( 'The Runtime Host service is not owned by a Maka managed deployment', ); } + await backend.verifyReplacementPreconditions(serviceConfig); const deploymentRoot = serviceConfig.managedDeploymentRoot; const currentCliPath = resolve(serviceConfig.launch.cliPath); const targetCliPath = resolveRuntimeHostManagedPackageCliPath( @@ -344,9 +345,10 @@ export async function runManagedRuntimeHostUpdateCli( if (!options.allowInterruptActiveTasks) { retirement = activeTasksRetirementFrame(status); } else { - const forced = await deps.withLifecycleLock(options.clientDataRoot, () => - deps.manage({ ...common, action: 'stop' }, backend), - ); + const forced = await deps.withLifecycleLock(options.clientDataRoot, async () => { + await backend.retire(); + return deps.manage({ ...common, action: 'status' }, backend); + }); if ( forced.service.active || forced.service.pid !== null ||