Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
36 changes: 25 additions & 11 deletions modules/jarvos-control-plane/src/storage/index.js
Original file line number Diff line number Diff line change
Expand Up @@ -150,6 +150,18 @@ function createFileStore(rootDir, options = {}) {
throw error;
}

try {
const text = fs.readFileSync(lockPath, 'utf8');
if (text) lock = JSON.parse(text);
} catch (error) {
if (error.code === 'ENOENT') return false;
if (!(error instanceof SyntaxError)) throw error;
}
const ageMs = Date.now() - stat.mtimeMs;
// Never remove the canonical lock until the observed owner is known to be
// dead and the lock is old enough to recover.
if (lockOwnerIsRunning(lock) || ageMs < staleLockMs) return false;

const takeoverPath = `${lockPath}.takeover-${stat.dev}-${stat.ino}-${process.pid}-${Math.random().toString(16).slice(2)}`;
try {
fs.renameSync(lockPath, takeoverPath);
Expand All @@ -163,7 +175,7 @@ function createFileStore(rootDir, options = {}) {
const takeoverStat = fs.statSync(takeoverPath);
if (takeoverStat.dev !== stat.dev || takeoverStat.ino !== stat.ino) {
try {
fs.renameSync(takeoverPath, lockPath);
fs.linkSync(takeoverPath, lockPath);
} catch (error) {
if (error.code !== 'EEXIST') throw error;
preserveTakeoverPath = true;
Expand All @@ -177,13 +189,6 @@ function createFileStore(rootDir, options = {}) {
if (error.code === 'ENOENT') return false;
if (!(error instanceof SyntaxError)) throw error;
}
const ageMs = Date.now() - fs.statSync(takeoverPath).mtimeMs;
// A matching process-start marker proves this is a live owner; a recycled
// PID fails the comparison and is recoverable once the stale interval passes.
if (lockOwnerIsRunning(lock) || ageMs < staleLockMs) {
try { fs.renameSync(takeoverPath, lockPath); } catch (error) { if (error.code !== 'EEXIST') throw error; }
return false;
}
fs.unlinkSync(takeoverPath);
return true;
} catch (error) {
Expand All @@ -196,19 +201,28 @@ function createFileStore(rootDir, options = {}) {
}
}
function withLock(fn) {
let fd; const deadline = Date.now() + (options.lockTimeoutMs || 5000);
let fd; let token; const deadline = Date.now() + (options.lockTimeoutMs || 5000);
while (!fd) {
try {
fd = fs.openSync(lockPath, 'wx', 0o600);
fs.writeFileSync(fd, JSON.stringify({ pid: process.pid, processStart: processStartMarker(process.pid), token: `${process.pid}-${Date.now()}-${Math.random().toString(16).slice(2)}`, createdAt: nowIso() }), 'utf8');
token = `${process.pid}-${Date.now()}-${Math.random().toString(16).slice(2)}`;
fs.writeFileSync(fd, JSON.stringify({ pid: process.pid, processStart: processStartMarker(process.pid), token, createdAt: nowIso() }), 'utf8');
fs.fsyncSync(fd);
} catch (error) {
if (fd) { fs.closeSync(fd); fs.rmSync(lockPath, { force: true }); fd = undefined; throw error; }
if (error.code !== 'EEXIST' || Date.now() >= deadline) throw new Error('Timed out acquiring file store lock');
if (!recoverStaleLock()) Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, retryMs);
}
}
try { return fn(); } finally { fs.closeSync(fd); fs.rmSync(lockPath, { force: true }); }
try { return fn(); } finally {
fs.closeSync(fd);
try {
const current = JSON.parse(fs.readFileSync(lockPath, 'utf8'));
if (current.token === token) fs.unlinkSync(lockPath);
} catch (error) {
if (error.code !== 'ENOENT') throw error;
}
}
}
function load() {
const hasCheckpoint = fs.existsSync(statePath);
Expand Down
26 changes: 26 additions & 0 deletions modules/jarvos-control-plane/test/storage.test.js
Original file line number Diff line number Diff line change
Expand Up @@ -93,6 +93,32 @@ test('file store recovers from a malformed stale lock', () => {
} finally { fs.rmSync(tmp, { recursive: true, force: true }); }
});

test('stale-lock recovery does not move a live lock', () => {
const tmp = fs.mkdtempSync(path.join(os.tmpdir(), 'jarvos-control-plane-live-lock-'));
const lockPath = path.join(tmp, 'state.lock');
const originalRename = fs.renameSync;
let lockRenameAttempted = false;
try {
fs.writeFileSync(lockPath, JSON.stringify({ pid: process.pid, token: 'live-owner' }));
const staleAt = new Date(Date.now() - 1000);
fs.utimesSync(lockPath, staleAt, staleAt);
fs.renameSync = (from, to) => {
if (from === lockPath) lockRenameAttempted = true;
return originalRename(from, to);
};

assert.throws(
() => createFileStore(tmp, { staleLockMs: 10, lockTimeoutMs: 20 }).acquireLease({ key: 'live-lock', holder: 'contender' }),
/Timed out acquiring file store lock/
);
assert.equal(lockRenameAttempted, false);
assert.equal(JSON.parse(fs.readFileSync(lockPath, 'utf8')).token, 'live-owner');
} finally {
fs.renameSync = originalRename;
fs.rmSync(tmp, { recursive: true, force: true });
}
});

test('stale-lock recovery never unlinks a fresh lock installed after takeover', () => {
const tmp = fs.mkdtempSync(path.join(os.tmpdir(), 'jarvos-control-plane-lock-race-'));
const lockPath = path.join(tmp, 'state.lock');
Expand Down
Loading