Skip to content

Keep journal recovery snapshots and audit backups outside the synced vault #183

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
levineam wants to merge 1 commit into
base: main
Choose a base branch
from
Open

Keep journal recovery snapshots and audit backups outside the synced vault #183

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
52 changes: 44 additions & 8 deletions modules/jarvos-secondbrain/packages/jarvos-secondbrain-journal/src/journal-maintenance.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,7 @@ const PROJECTS_PROJECTION_MODULE = path.join(
const SIGNATURE = '— Edited by Jarvis';
const DEFAULT_TIMEZONE = 'America/New_York';
const LEGACY_SALIENCE_LINE_RE = /^-\s*📌\s*\*\(([^,]+),\s*(\d+)%\)\*\s*(.+)$/i;
const JOURNAL_STATE_DIR = '.jarvos/journal-maintenance';
const JOURNAL_STATE_DIR = 'journal-maintenance';

function parseArgs(argv) {
const out = {
Expand Down Expand Up @@ -173,7 +173,39 @@ function safeTimestamp(date = new Date()) {
}

function journalStateRoot(journalDir) {
return path.join(path.dirname(journalDir), JOURNAL_STATE_DIR);
const configured = process.env.JARVOS_JOURNAL_STATE_DIR;
if (configured) return path.resolve(resolveTilde(configured));

// Recovery snapshots contain the complete journal and must not live inside
// the vault, where sync/index/export tools can retain text the user deleted.
// Scope the local state by vault path so multiple vaults do not collide.
const stateHome = process.env.XDG_STATE_HOME
? path.resolve(resolveTilde(process.env.XDG_STATE_HOME))
: path.join(process.env.HOME || os.homedir(), '.local', 'state');
const vaultId = contentHash(path.resolve(path.dirname(journalDir))).slice(0, 16);
return path.join(stateHome, 'jarvos', JOURNAL_STATE_DIR, vaultId);
}

function migrateLegacyJournalSnapshots(journalDir) {
const legacyRoot = path.join(path.dirname(journalDir), '.jarvos', JOURNAL_STATE_DIR);
const stateRoot = journalStateRoot(journalDir);
if (path.resolve(legacyRoot) === path.resolve(stateRoot) || !fs.existsSync(legacyRoot)) return;

fs.mkdirSync(stateRoot, { recursive: true, mode: 0o700 });
for (const name of ['state.json', 'known-good', 'audit-backups']) {
const source = path.join(legacyRoot, name);
const destination = path.join(stateRoot, name);
if (!fs.existsSync(source)) continue;
if (!fs.existsSync(destination)) fs.cpSync(source, destination, { recursive: true });
const secure = (candidate) => {
const stat = fs.statSync(candidate);
fs.chmodSync(candidate, stat.isDirectory() ? 0o700 : 0o600);
if (stat.isDirectory()) fs.readdirSync(candidate).forEach((entry) => secure(path.join(candidate, entry)));
};
secure(destination);
// Always remove the synced copy, including when local state already exists.
fs.rmSync(source, { recursive: true, force: true });
}
}

function journalStatePath(journalDir) {
Expand All @@ -200,8 +232,9 @@ function loadJournalState(journalDir) {

function writeJournalState(journalDir, state) {
const statePath = journalStatePath(journalDir);
fs.mkdirSync(path.dirname(statePath), { recursive: true });
fs.writeFileSync(statePath, `${JSON.stringify(state, null, 2)}\n`, 'utf8');
fs.mkdirSync(path.dirname(statePath), { recursive: true, mode: 0o700 });
fs.writeFileSync(statePath, `${JSON.stringify(state, null, 2)}\n`, { encoding: 'utf8', mode: 0o600 });
fs.chmodSync(statePath, 0o600);
}

/**
Expand Down Expand Up @@ -1091,6 +1124,7 @@ function resolveJournalDir(config) {

function syncOneDate(date, config, opts = {}) {
const journalDir = resolveJournalDir(config);
if (!opts.dryRun) migrateLegacyJournalSnapshots(journalDir);
const journalPath = path.join(journalDir, `${date}.md`);
const existed = fs.existsSync(journalPath);
const original = existed ? fs.readFileSync(journalPath, 'utf8') : '';
Expand Down Expand Up @@ -1127,8 +1161,9 @@ function syncOneDate(date, config, opts = {}) {
}
if (existed) {
backupPath = auditBackupPath(journalDir, date, backupReason);
fs.mkdirSync(path.dirname(backupPath), { recursive: true });
fs.writeFileSync(backupPath, original, 'utf8');
fs.mkdirSync(path.dirname(backupPath), { recursive: true, mode: 0o700 });
fs.writeFileSync(backupPath, original, { encoding: 'utf8', mode: 0o600 });
fs.chmodSync(backupPath, 0o600);
}
mutationReceipt = mutate({
filePath: journalPath,
Expand Down Expand Up @@ -1206,8 +1241,9 @@ function syncOneDate(date, config, opts = {}) {

if (!opts.dryRun && acknowledged && (healthAfter.status === 'healthy' || authoredContentIntact)) {
const updatedKnownGoodPath = knownGoodPath(journalDir, date);
fs.mkdirSync(path.dirname(updatedKnownGoodPath), { recursive: true });
fs.writeFileSync(updatedKnownGoodPath, effectiveContent, 'utf8');
fs.mkdirSync(path.dirname(updatedKnownGoodPath), { recursive: true, mode: 0o700 });
fs.writeFileSync(updatedKnownGoodPath, effectiveContent, { encoding: 'utf8', mode: 0o600 });
fs.chmodSync(updatedKnownGoodPath, 0o600);
const metrics = journalMetrics(effectiveContent, config);
state.version = 1;
state.dates = state.dates || {};
Expand Down
11 changes: 9 additions & 2 deletions modules/jarvos-secondbrain/tests/journal-projects-section.test.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -455,8 +455,10 @@ test('the known-good snapshot refreshes across a contract migration', () => {

const previousJournalDir = process.env.JARVOS_JOURNAL_DIR;
const previousProjectsDir = process.env.JARVOS_PROJECTS_DIR;
const previousStateDir = process.env.JARVOS_JOURNAL_STATE_DIR;
process.env.JARVOS_JOURNAL_DIR = journalDir;
process.env.JARVOS_PROJECTS_DIR = path.join(tmp, 'Vault', 'Projects');
process.env.JARVOS_JOURNAL_STATE_DIR = path.join(tmp, 'local-state');

try {
const config = loadConfig();
Expand All @@ -474,7 +476,7 @@ test('the known-good snapshot refreshes across a contract migration', () => {
fs.writeFileSync(path.join(journalDir, `${date}.md`), preMigration, 'utf8');

// A snapshot exactly as the previous contract would have recorded it.
const stateDir = path.join(tmp, 'Vault', '.jarvos', 'journal-maintenance');
const stateDir = process.env.JARVOS_JOURNAL_STATE_DIR;
fs.mkdirSync(path.join(stateDir, 'known-good'), { recursive: true });
fs.writeFileSync(path.join(stateDir, 'known-good', `${date}.md`), preMigration, 'utf8');
fs.writeFileSync(path.join(stateDir, 'state.json'), JSON.stringify({
Expand Down Expand Up @@ -507,6 +509,8 @@ test('the known-good snapshot refreshes across a contract migration', () => {
else process.env.JARVOS_JOURNAL_DIR = previousJournalDir;
if (previousProjectsDir === undefined) delete process.env.JARVOS_PROJECTS_DIR;
else process.env.JARVOS_PROJECTS_DIR = previousProjectsDir;
if (previousStateDir === undefined) delete process.env.JARVOS_JOURNAL_STATE_DIR;
else process.env.JARVOS_JOURNAL_STATE_DIR = previousStateDir;
}
});

Expand All @@ -526,8 +530,10 @@ test('a damaged entry must not overwrite a good pre-migration snapshot', () => {

const prevJournal = process.env.JARVOS_JOURNAL_DIR;
const prevProjects = process.env.JARVOS_PROJECTS_DIR;
const prevState = process.env.JARVOS_JOURNAL_STATE_DIR;
process.env.JARVOS_JOURNAL_DIR = journalDir;
process.env.JARVOS_PROJECTS_DIR = path.join(tmp, 'Vault', 'Projects');
process.env.JARVOS_JOURNAL_STATE_DIR = path.join(tmp, 'local-state');

try {
const config = loadConfig();
Expand Down Expand Up @@ -557,7 +563,7 @@ test('a damaged entry must not overwrite a good pre-migration snapshot', () => {
].join('\n');
fs.writeFileSync(path.join(journalDir, `${date}.md`), gutted, 'utf8');

const stateDir = path.join(tmp, 'Vault', '.jarvos', 'journal-maintenance');
const stateDir = process.env.JARVOS_JOURNAL_STATE_DIR;
const kgPath = path.join(stateDir, 'known-good', `${date}.md`);
fs.mkdirSync(path.dirname(kgPath), { recursive: true });
fs.writeFileSync(kgPath, intact, 'utf8');
Expand Down Expand Up @@ -586,5 +592,6 @@ test('a damaged entry must not overwrite a good pre-migration snapshot', () => {
} finally {
if (prevJournal === undefined) delete process.env.JARVOS_JOURNAL_DIR; else process.env.JARVOS_JOURNAL_DIR = prevJournal;
if (prevProjects === undefined) delete process.env.JARVOS_PROJECTS_DIR; else process.env.JARVOS_PROJECTS_DIR = prevProjects;
if (prevState === undefined) delete process.env.JARVOS_JOURNAL_STATE_DIR; else process.env.JARVOS_JOURNAL_STATE_DIR = prevState;
}
});
47 changes: 43 additions & 4 deletions modules/jarvos-secondbrain/tests/journal-stub-guard.test.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -88,10 +88,15 @@ function makeVault() {
return { vault, journalDir };
}

function stateDirFor(journalDir) {
return `${path.dirname(journalDir)}-local-state`;
}

function inVault(journalDir, fn) {
return withEnv(
{
JARVOS_JOURNAL_DIR: journalDir,
JARVOS_JOURNAL_STATE_DIR: stateDirFor(journalDir),
JOURNAL_DIR: undefined,
JARVOS_VAULT_DIR: undefined,
JARVOS_CONFIG_PATH: undefined,
Expand Down Expand Up @@ -152,7 +157,7 @@ test('frontmatter-only stub overwrite after a populated journal is restored from
fs.writeFileSync(journalPath, populatedJournal(), 'utf8');
const first = syncOneDate(DATE, TEST_CONFIG, {});
assert.equal(first.healthAfter.status, 'healthy');
const knownGoodFile = path.join(vault, '.jarvos/journal-maintenance/known-good', `${DATE}.md`);
const knownGoodFile = path.join(stateDirFor(journalDir), 'known-good', `${DATE}.md`);
assert.ok(fs.existsSync(knownGoodFile), 'known-good snapshot is recorded');

// External writer clobbers the populated journal with a stub.
Expand Down Expand Up @@ -310,9 +315,9 @@ test('partial clobber (stale) is detected and does not poison the known-good sna
fs.writeFileSync(journalPath, populatedJournal(), 'utf8');
const first = syncOneDate(DATE, TEST_CONFIG, {});
assert.equal(first.healthAfter.status, 'healthy');
const knownGoodFile = path.join(vault, '.jarvos/journal-maintenance/known-good', `${DATE}.md`);
const knownGoodFile = path.join(stateDirFor(journalDir), 'known-good', `${DATE}.md`);
const goodSnapshot = fs.readFileSync(knownGoodFile, 'utf8');
const statePath = path.join(vault, '.jarvos/journal-maintenance/state.json');
const statePath = path.join(stateDirFor(journalDir), 'state.json');
const goodState = fs.readFileSync(statePath, 'utf8');

// External writer clobbers it with a partial file: still has a section
Expand Down Expand Up @@ -352,7 +357,7 @@ test('healthy journal refresh keeps known-good state in sync', () => {
fs.writeFileSync(journalPath, populatedJournal(), 'utf8');

syncOneDate(DATE, TEST_CONFIG, {});
const statePath = path.join(vault, '.jarvos/journal-maintenance/state.json');
const statePath = path.join(stateDirFor(journalDir), 'state.json');
const state = JSON.parse(fs.readFileSync(statePath, 'utf8'));
const entry = state.dates[DATE];
const onDisk = fs.readFileSync(journalPath, 'utf8');
Expand All @@ -363,3 +368,37 @@ test('healthy journal refresh keeps known-good state in sync', () => {
fs.rmSync(vault, { recursive: true, force: true });
}
});

test('full-content recovery files stay outside the vault and legacy synced copies are removed', () => {
const { journalDir, vault } = makeVault();
const localState = stateDirFor(journalDir);
try {
const legacyRoot = path.join(vault, '.jarvos', 'journal-maintenance');
const legacyKnownGood = path.join(legacyRoot, 'known-good', `${DATE}.md`);
fs.mkdirSync(path.dirname(legacyKnownGood), { recursive: true });
fs.writeFileSync(legacyKnownGood, populatedJournal(), 'utf8');
fs.writeFileSync(path.join(legacyRoot, 'state.json'), JSON.stringify({
version: 1,
dates: {
[DATE]: {
...journalModule().journalMetrics(populatedJournal()),
date: DATE,
},
},
}), 'utf8');
fs.writeFileSync(path.join(journalDir, `${DATE}.md`), STUB, 'utf8');

inVault(journalDir, () => {
const result = journalModule().syncOneDate(DATE, TEST_CONFIG, {});
assert.equal(result.restoredKnownGood, true);
assert.ok(result.backupPath.startsWith(localState));
assert.ok(fs.existsSync(path.join(localState, 'known-good', `${DATE}.md`)));
assert.equal(fs.existsSync(path.join(legacyRoot, 'known-good')), false);
assert.equal(fs.existsSync(path.join(legacyRoot, 'audit-backups')), false);
assert.equal(fs.statSync(result.backupPath).mode & 0o777, 0o600);
});
} finally {
fs.rmSync(vault, { recursive: true, force: true });
fs.rmSync(localState, { recursive: true, force: true });
}
});
Loading