diff --git a/.jules/bolt.md b/.jules/bolt.md new file mode 100644 index 00000000..d42e621a --- /dev/null +++ b/.jules/bolt.md @@ -0,0 +1,3 @@ +## 2026-07-21 - Optimize ArtifactStore Garbage Collection +**Learning:** Optimizing operations that iterate over large numbers of files (e.g. ArtifactStore.gc) using batched concurrent fs.stat and fs.rm checks (e.g. Promise.all with chunk sizes of 10) prevents severe performance bottlenecks and EMFILE limits. +**Action:** When performing file system operations on a potentially large list of directories/files, batch operations in chunks (e.g., of 10) utilizing Promise.all rather than awaiting them sequentially. diff --git a/src/core/sub-agent/artifacts/store.ts b/src/core/sub-agent/artifacts/store.ts index 8a7aff5c..164f15a9 100644 --- a/src/core/sub-agent/artifacts/store.ts +++ b/src/core/sub-agent/artifacts/store.ts @@ -93,13 +93,19 @@ export class ArtifactStore { const entries = await fs.readdir(root, { withFileTypes: true }).catch(() => []); const files: Array<{ name: string; path: string; mtimeMs: number; size: number }> = []; - for (const entry of entries) { - if (!entry.isFile()) continue; - const filePath = path.join(root, entry.name); - if (!isWithinDir(root, filePath)) continue; - const stat = await fs.stat(filePath).catch(() => null); - if (!stat) continue; - files.push({ name: entry.name, path: filePath, mtimeMs: stat.mtimeMs, size: stat.size }); + const validEntries = entries.filter((e) => e.isFile()); + + for (let i = 0; i < validEntries.length; i += 10) { + const chunk = validEntries.slice(i, i + 10); + await Promise.all( + chunk.map(async (entry) => { + const filePath = path.join(root, entry.name); + if (!isWithinDir(root, filePath)) return; + const stat = await fs.stat(filePath).catch(() => null); + if (!stat) return; + files.push({ name: entry.name, path: filePath, mtimeMs: stat.mtimeMs, size: stat.size }); + }), + ); } const maxAgeMs = options?.maxAgeMs ?? LIMITS.artifactTtlMs; @@ -112,15 +118,7 @@ export class ArtifactStore { let removedFiles = 0; let removedBytes = 0; - const removeFile = async (file: { path: string; size: number }) => { - await fs.rm(file.path, { force: true }).catch(() => null); - removedFiles += 1; - removedBytes += file.size; - }; - - for (const file of expired) { - await removeFile(file); - } + const toRemove: Array<{ path: string; size: number }> = [...expired]; // Recompute remaining after TTL removal (newest first). const remaining = files @@ -136,11 +134,25 @@ export class ArtifactStore { if (!tooManyFiles && !tooManyBytes) break; const oldest = remaining[i]; - await removeFile(oldest); + toRemove.push(oldest); currentFiles -= 1; currentBytes -= oldest.size; } + for (let i = 0; i < toRemove.length; i += 10) { + const chunk = toRemove.slice(i, i + 10); + await Promise.all( + chunk.map(async (file) => { + await fs.rm(file.path, { force: true }).catch(() => null); + }), + ); + } + + for (const file of toRemove) { + removedFiles += 1; + removedBytes += file.size; + } + return { removedFiles, removedBytes }; } diff --git a/tests/integration/prompt_templates.test.ts b/tests/integration/prompt_templates.test.ts index 21905259..f1360d78 100644 --- a/tests/integration/prompt_templates.test.ts +++ b/tests/integration/prompt_templates.test.ts @@ -36,7 +36,9 @@ describe('Prompt templates', () => { expect(planSystem).toContain('You are SalmonLoop.'); expect(patchSystem).toContain('You are PATCH, a phase-native diff compiler.'); - expect(autopilotSystem).toContain('You are a senior software engineer running in "autopilot" mode.'); + expect(autopilotSystem).toContain( + 'You are a senior software engineer running in "autopilot" mode.', + ); expect(answerSystem).toContain('You are a coding assistant in "answer" mode.'); expect(researchSystem).toContain('You are a research assistant.'); });