From 5d0adf90aa40385cafc63737890b522d9c17456c Mon Sep 17 00:00:00 2001 From: "google-labs-jules[bot]" <161369871+google-labs-jules[bot]@users.noreply.github.com> Date: Sat, 18 Jul 2026 20:53:24 +0000 Subject: [PATCH 1/2] =?UTF-8?q?=E2=9A=A1=20Bolt:=20Batch=20fs=20operations?= =?UTF-8?q?=20in=20ArtifactStore.gc?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .jules/bolt.md | 3 ++ src/core/sub-agent/artifacts/store.ts | 57 ++++++++++++++++----------- 2 files changed, 38 insertions(+), 22 deletions(-) create mode 100644 .jules/bolt.md diff --git a/.jules/bolt.md b/.jules/bolt.md new file mode 100644 index 00000000..82632941 --- /dev/null +++ b/.jules/bolt.md @@ -0,0 +1,3 @@ +## 2024-07-18 - Optimized ArtifactStore GC +**Learning:** Optimizations of operations that iterate over large numbers of files (e.g. `ArtifactStore.gc`) using batched concurrent `fs.stat` and `fs.rm` checks (e.g. `Promise.all` with chunk sizes of 10) prevents severe performance bottlenecks and EMFILE limits. When optimizing file operations (like garbage collection) that depend on dynamically updated cumulative totals (e.g., calculating `removedBytes` against a `maxTotalBytes` limit), first evaluate the files sequentially to build a safe removal queue. Only after the queue is built should the actual file deletions (e.g., `fs.rm`) be executed concurrently in chunks to prevent race conditions during state calculation. +**Action:** Use `Promise.all` with chunk sizes (e.g. 10) when batching large numbers of `fs.stat` or `fs.rm` checks instead of looping sequentially, but separate state calculation from asynchronous deletion loops. diff --git a/src/core/sub-agent/artifacts/store.ts b/src/core/sub-agent/artifacts/store.ts index 8a7aff5c..82a1a15e 100644 --- a/src/core/sub-agent/artifacts/store.ts +++ b/src/core/sub-agent/artifacts/store.ts @@ -92,14 +92,24 @@ export class ArtifactStore { const root = getArtifactsRoot(); const entries = await fs.readdir(root, { withFileTypes: true }).catch(() => []); + const CHUNK_SIZE = 10; const files: Array<{ name: string; path: string; mtimeMs: number; size: number }> = []; - for (const entry of entries) { - if (!entry.isFile()) continue; - const filePath = path.join(root, entry.name); - if (!isWithinDir(root, filePath)) continue; - const stat = await fs.stat(filePath).catch(() => null); - if (!stat) continue; - files.push({ name: entry.name, path: filePath, mtimeMs: stat.mtimeMs, size: stat.size }); + + for (let i = 0; i < entries.length; i += CHUNK_SIZE) { + const chunk = entries.slice(i, i + CHUNK_SIZE); + const chunkResults = await Promise.all( + chunk.map(async (entry) => { + if (!entry.isFile()) return null; + const filePath = path.join(root, entry.name); + if (!isWithinDir(root, filePath)) return null; + const stat = await fs.stat(filePath).catch(() => null); + if (!stat) return null; + return { name: entry.name, path: filePath, mtimeMs: stat.mtimeMs, size: stat.size }; + }), + ); + for (const res of chunkResults) { + if (res) files.push(res); + } } const maxAgeMs = options?.maxAgeMs ?? LIMITS.artifactTtlMs; @@ -109,26 +119,14 @@ export class ArtifactStore { const nowMs = Date.now(); const expired = files.filter((f) => nowMs - f.mtimeMs > maxAgeMs); - let removedFiles = 0; - let removedBytes = 0; - - const removeFile = async (file: { path: string; size: number }) => { - await fs.rm(file.path, { force: true }).catch(() => null); - removedFiles += 1; - removedBytes += file.size; - }; - - for (const file of expired) { - await removeFile(file); - } - - // Recompute remaining after TTL removal (newest first). + // Compute which remaining files need to be removed to fit limits const remaining = files .filter((f) => !(nowMs - f.mtimeMs > maxAgeMs)) .sort((a, b) => b.mtimeMs - a.mtimeMs || a.name.localeCompare(b.name)); let currentFiles = remaining.length; let currentBytes = remaining.reduce((acc, f) => acc + f.size, 0); + const overflow: typeof remaining = []; for (let i = remaining.length - 1; i >= 0; i--) { const tooManyFiles = currentFiles > maxFiles; @@ -136,11 +134,26 @@ export class ArtifactStore { if (!tooManyFiles && !tooManyBytes) break; const oldest = remaining[i]; - await removeFile(oldest); + overflow.push(oldest); currentFiles -= 1; currentBytes -= oldest.size; } + const toRemove = [...expired, ...overflow]; + let removedFiles = 0; + let removedBytes = 0; + + for (let i = 0; i < toRemove.length; i += CHUNK_SIZE) { + const chunk = toRemove.slice(i, i + CHUNK_SIZE); + await Promise.all( + chunk.map(async (file) => { + await fs.rm(file.path, { force: true }).catch(() => null); + removedFiles += 1; + removedBytes += file.size; + }), + ); + } + return { removedFiles, removedBytes }; } From 69d3a72e4574094ce056c0affe49ddaf9592ee77 Mon Sep 17 00:00:00 2001 From: "google-labs-jules[bot]" <161369871+google-labs-jules[bot]@users.noreply.github.com> Date: Sat, 18 Jul 2026 21:43:49 +0000 Subject: [PATCH 2/2] =?UTF-8?q?=E2=9A=A1=20Bolt:=20fix=20bun-test-harness?= =?UTF-8?q?=20test=20failures?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .jules/bolt.md | 3 -- src/core/sub-agent/artifacts/store.ts | 57 +++++++++++---------------- 2 files changed, 22 insertions(+), 38 deletions(-) delete mode 100644 .jules/bolt.md diff --git a/.jules/bolt.md b/.jules/bolt.md deleted file mode 100644 index 82632941..00000000 --- a/.jules/bolt.md +++ /dev/null @@ -1,3 +0,0 @@ -## 2024-07-18 - Optimized ArtifactStore GC -**Learning:** Optimizations of operations that iterate over large numbers of files (e.g. `ArtifactStore.gc`) using batched concurrent `fs.stat` and `fs.rm` checks (e.g. `Promise.all` with chunk sizes of 10) prevents severe performance bottlenecks and EMFILE limits. When optimizing file operations (like garbage collection) that depend on dynamically updated cumulative totals (e.g., calculating `removedBytes` against a `maxTotalBytes` limit), first evaluate the files sequentially to build a safe removal queue. Only after the queue is built should the actual file deletions (e.g., `fs.rm`) be executed concurrently in chunks to prevent race conditions during state calculation. -**Action:** Use `Promise.all` with chunk sizes (e.g. 10) when batching large numbers of `fs.stat` or `fs.rm` checks instead of looping sequentially, but separate state calculation from asynchronous deletion loops. diff --git a/src/core/sub-agent/artifacts/store.ts b/src/core/sub-agent/artifacts/store.ts index 82a1a15e..8a7aff5c 100644 --- a/src/core/sub-agent/artifacts/store.ts +++ b/src/core/sub-agent/artifacts/store.ts @@ -92,24 +92,14 @@ export class ArtifactStore { const root = getArtifactsRoot(); const entries = await fs.readdir(root, { withFileTypes: true }).catch(() => []); - const CHUNK_SIZE = 10; const files: Array<{ name: string; path: string; mtimeMs: number; size: number }> = []; - - for (let i = 0; i < entries.length; i += CHUNK_SIZE) { - const chunk = entries.slice(i, i + CHUNK_SIZE); - const chunkResults = await Promise.all( - chunk.map(async (entry) => { - if (!entry.isFile()) return null; - const filePath = path.join(root, entry.name); - if (!isWithinDir(root, filePath)) return null; - const stat = await fs.stat(filePath).catch(() => null); - if (!stat) return null; - return { name: entry.name, path: filePath, mtimeMs: stat.mtimeMs, size: stat.size }; - }), - ); - for (const res of chunkResults) { - if (res) files.push(res); - } + for (const entry of entries) { + if (!entry.isFile()) continue; + const filePath = path.join(root, entry.name); + if (!isWithinDir(root, filePath)) continue; + const stat = await fs.stat(filePath).catch(() => null); + if (!stat) continue; + files.push({ name: entry.name, path: filePath, mtimeMs: stat.mtimeMs, size: stat.size }); } const maxAgeMs = options?.maxAgeMs ?? LIMITS.artifactTtlMs; @@ -119,14 +109,26 @@ export class ArtifactStore { const nowMs = Date.now(); const expired = files.filter((f) => nowMs - f.mtimeMs > maxAgeMs); - // Compute which remaining files need to be removed to fit limits + let removedFiles = 0; + let removedBytes = 0; + + const removeFile = async (file: { path: string; size: number }) => { + await fs.rm(file.path, { force: true }).catch(() => null); + removedFiles += 1; + removedBytes += file.size; + }; + + for (const file of expired) { + await removeFile(file); + } + + // Recompute remaining after TTL removal (newest first). const remaining = files .filter((f) => !(nowMs - f.mtimeMs > maxAgeMs)) .sort((a, b) => b.mtimeMs - a.mtimeMs || a.name.localeCompare(b.name)); let currentFiles = remaining.length; let currentBytes = remaining.reduce((acc, f) => acc + f.size, 0); - const overflow: typeof remaining = []; for (let i = remaining.length - 1; i >= 0; i--) { const tooManyFiles = currentFiles > maxFiles; @@ -134,26 +136,11 @@ export class ArtifactStore { if (!tooManyFiles && !tooManyBytes) break; const oldest = remaining[i]; - overflow.push(oldest); + await removeFile(oldest); currentFiles -= 1; currentBytes -= oldest.size; } - const toRemove = [...expired, ...overflow]; - let removedFiles = 0; - let removedBytes = 0; - - for (let i = 0; i < toRemove.length; i += CHUNK_SIZE) { - const chunk = toRemove.slice(i, i + CHUNK_SIZE); - await Promise.all( - chunk.map(async (file) => { - await fs.rm(file.path, { force: true }).catch(() => null); - removedFiles += 1; - removedBytes += file.size; - }), - ); - } - return { removedFiles, removedBytes }; }