diff --git a/.jules/bolt.md b/.jules/bolt.md new file mode 100644 index 00000000..c0910c5f --- /dev/null +++ b/.jules/bolt.md @@ -0,0 +1,3 @@ +## 2024-05-24 - Batching concurrent fs operations for ArtifactStore.gc +**Learning:** Sequential fs operations (e.g. fs.stat, fs.rm) over a large number of files within `ArtifactStore.gc` introduces a significant performance bottleneck due to excessive async-await blocking. This leads to slow artifact cleanup. +**Action:** When performing file operations sequentially over many files, batch the fs operations using `Promise.all` in chunks of 10 to speed up throughput and circumvent EMFILE limits. \ No newline at end of file diff --git a/src/core/sub-agent/artifacts/store.ts b/src/core/sub-agent/artifacts/store.ts index 8a7aff5c..c8cc6b32 100644 --- a/src/core/sub-agent/artifacts/store.ts +++ b/src/core/sub-agent/artifacts/store.ts @@ -93,13 +93,35 @@ export class ArtifactStore { const entries = await fs.readdir(root, { withFileTypes: true }).catch(() => []); const files: Array<{ name: string; path: string; mtimeMs: number; size: number }> = []; - for (const entry of entries) { - if (!entry.isFile()) continue; - const filePath = path.join(root, entry.name); - if (!isWithinDir(root, filePath)) continue; - const stat = await fs.stat(filePath).catch(() => null); - if (!stat) continue; - files.push({ name: entry.name, path: filePath, mtimeMs: stat.mtimeMs, size: stat.size }); + + // Valid entry preparation + const validEntries = entries.reduce>((acc, entry) => { + if (entry.isFile()) { + const filePath = path.join(root, entry.name); + if (isWithinDir(root, filePath)) { + acc.push({ name: entry.name, path: filePath }); + } + } + return acc; + }, []); + + // ⚡ Bolt: Batch fs.stat concurrently in chunks of 10 to speed up stat retrieval without hitting EMFILE limits. + const chunkSize = 10; + for (let i = 0; i < validEntries.length; i += chunkSize) { + const chunk = validEntries.slice(i, i + chunkSize); + await Promise.all( + chunk.map(async (entry) => { + const stat = await fs.stat(entry.path).catch(() => null); + if (stat) { + files.push({ + name: entry.name, + path: entry.path, + mtimeMs: stat.mtimeMs, + size: stat.size, + }); + } + }), + ); } const maxAgeMs = options?.maxAgeMs ?? LIMITS.artifactTtlMs; @@ -109,18 +131,8 @@ export class ArtifactStore { const nowMs = Date.now(); const expired = files.filter((f) => nowMs - f.mtimeMs > maxAgeMs); - let removedFiles = 0; - let removedBytes = 0; - - const removeFile = async (file: { path: string; size: number }) => { - await fs.rm(file.path, { force: true }).catch(() => null); - removedFiles += 1; - removedBytes += file.size; - }; - - for (const file of expired) { - await removeFile(file); - } + // Build removal queue safely without async race conditions during logic evaluation + const removalQueue: Array<{ path: string; size: number }> = [...expired]; // Recompute remaining after TTL removal (newest first). const remaining = files @@ -136,11 +148,26 @@ export class ArtifactStore { if (!tooManyFiles && !tooManyBytes) break; const oldest = remaining[i]; - await removeFile(oldest); + removalQueue.push(oldest); currentFiles -= 1; currentBytes -= oldest.size; } + let removedFiles = 0; + let removedBytes = 0; + + // ⚡ Bolt: Batch fs.rm concurrently in chunks of 10 to vastly improve file deletion speed. + for (let i = 0; i < removalQueue.length; i += chunkSize) { + const chunk = removalQueue.slice(i, i + chunkSize); + await Promise.all( + chunk.map(async (file) => { + await fs.rm(file.path, { force: true }).catch(() => null); + removedFiles += 1; + removedBytes += file.size; + }), + ); + } + return { removedFiles, removedBytes }; } diff --git a/tests/integration/prompt_templates.test.ts b/tests/integration/prompt_templates.test.ts index 21905259..f1360d78 100644 --- a/tests/integration/prompt_templates.test.ts +++ b/tests/integration/prompt_templates.test.ts @@ -36,7 +36,9 @@ describe('Prompt templates', () => { expect(planSystem).toContain('You are SalmonLoop.'); expect(patchSystem).toContain('You are PATCH, a phase-native diff compiler.'); - expect(autopilotSystem).toContain('You are a senior software engineer running in "autopilot" mode.'); + expect(autopilotSystem).toContain( + 'You are a senior software engineer running in "autopilot" mode.', + ); expect(answerSystem).toContain('You are a coding assistant in "answer" mode.'); expect(researchSystem).toContain('You are a research assistant.'); });