From b9ce14149809aad30db521334fd9a262913bdd28 Mon Sep 17 00:00:00 2001 From: Tyrie Vella Date: Thu, 4 Jun 2026 10:29:10 -0700 Subject: [PATCH 1/2] Expand blob prefetch noop cache to N entries Replace the single-entry LastBlobPrefetch.dat cache with a multi-entry BlobPrefetchCache.dat that stores up to N entries (default 100), keyed by SHA256 hash of (files, folders, hydrate) and storing the commit ID. This avoids redundant diff+download work when users cycle through a small set of prefetch patterns (e.g. 3 different file/folder combos), which previously caused 2/3 of calls to miss the single-entry cache. Changes: - BlobPrefetcher: replace flat 4-key dictionary with hash-keyed cache - BlobPrefetcher.ComputeCacheKey: canonical, order-independent hashing - BlobPrefetcher.SavePrefetchArgs: single-entry eviction when at capacity - PrefetchVerb: read gvfs.prefetchCacheSize config (0=disabled, max 1000) - PrefetchVerb: use BlobPrefetchCache.dat instead of LastBlobPrefetch.dat - 12 unit tests covering key determinism, order independence, cache hit/miss, multi-entry support, and null/empty edge cases Assisted-by: Claude Opus 4.6 Signed-off-by: Tyrie Vella --- GVFS/GVFS.Common/Prefetch/BlobPrefetcher.cs | 96 ++++---- .../Prefetch/BlobPrefetcherTests.cs | 210 +++++++++++++++++- GVFS/GVFS/CommandLine/PrefetchVerb.cs | 52 +++-- 3 files changed, 301 insertions(+), 57 deletions(-) diff --git a/GVFS/GVFS.Common/Prefetch/BlobPrefetcher.cs b/GVFS/GVFS.Common/Prefetch/BlobPrefetcher.cs index 29bc4cc67..24e5ed951 100644 --- a/GVFS/GVFS.Common/Prefetch/BlobPrefetcher.cs +++ b/GVFS/GVFS.Common/Prefetch/BlobPrefetcher.cs @@ -9,6 +9,8 @@ using System.Collections.Generic; using System.IO; using System.Linq; +using System.Security.Cryptography; +using System.Text; using System.Threading; namespace GVFS.Common.Prefetch @@ -32,7 +34,13 @@ public class BlobPrefetcher private const string AreaPath = nameof(BlobPrefetcher); private static string pathSeparatorString = Path.DirectorySeparatorChar.ToString(); - private FileBasedDictionary lastPrefetchArgs; + public const string BlobPrefetchCacheFile = "BlobPrefetchCache.dat"; + public const string PrefetchCacheSizeConfigKey = GVFSConstants.GitConfig.GVFSPrefix + "prefetch-cache-size"; + public const int DefaultPrefetchCacheSize = 100; + public const int MaxPrefetchCacheSize = 1000; + + private FileBasedDictionary prefetchCache; + private int maxCacheSize; public BlobPrefetcher( ITracer tracer, @@ -42,7 +50,7 @@ public BlobPrefetcher( int searchThreadCount, int downloadThreadCount, int indexThreadCount) - : this(tracer, enlistment, objectRequestor, null, null, null, chunkSize, searchThreadCount, downloadThreadCount, indexThreadCount) + : this(tracer, enlistment, objectRequestor, null, null, null, DefaultPrefetchCacheSize, chunkSize, searchThreadCount, downloadThreadCount, indexThreadCount) { } @@ -52,7 +60,8 @@ public BlobPrefetcher( GitObjectsHttpRequestor objectRequestor, List fileList, List folderList, - FileBasedDictionary lastPrefetchArgs, + FileBasedDictionary prefetchCache, + int maxCacheSize, int chunkSize, int searchThreadCount, int downloadThreadCount, @@ -70,7 +79,8 @@ public BlobPrefetcher( this.FileList = fileList ?? new List(); this.FolderList = folderList ?? new List(); - this.lastPrefetchArgs = lastPrefetchArgs; + this.prefetchCache = prefetchCache; + this.maxCacheSize = maxCacheSize; // We never want to update config settings for a GVFSEnlistment this.SkipConfigUpdate = enlistment is GVFSEnlistment; @@ -127,39 +137,26 @@ public static bool TryLoadFileList(Enlistment enlistment, string filesInput, str public static bool IsNoopPrefetch( ITracer tracer, - FileBasedDictionary lastPrefetchArgs, + FileBasedDictionary prefetchCache, string commitId, List files, List folders, bool hydrateFilesAfterDownload) { - if (lastPrefetchArgs != null && - lastPrefetchArgs.TryGetValue(PrefetchArgs.CommitId, out string lastCommitId) && - lastPrefetchArgs.TryGetValue(PrefetchArgs.Files, out string lastFilesString) && - lastPrefetchArgs.TryGetValue(PrefetchArgs.Folders, out string lastFoldersString) && - lastPrefetchArgs.TryGetValue(PrefetchArgs.Hydrate, out string lastHydrateString)) + if (prefetchCache != null) { - string newFilesString = GVFSJsonOptions.Serialize(files); - string newFoldersString = GVFSJsonOptions.Serialize(folders); - bool isNoop = - commitId == lastCommitId && - hydrateFilesAfterDownload.ToString() == lastHydrateString && - newFilesString == lastFilesString && - newFoldersString == lastFoldersString; + string cacheKey = ComputeCacheKey(files, folders, hydrateFilesAfterDownload); + bool hasEntry = prefetchCache.TryGetValue(cacheKey, out string cachedCommitId); + bool isNoop = hasEntry && commitId == cachedCommitId; tracer.RelatedEvent( EventLevel.Informational, "BlobPrefetcher.IsNoopPrefetch", new EventMetadata { - { "Last" + PrefetchArgs.CommitId, lastCommitId }, - { "Last" + PrefetchArgs.Files, lastFilesString }, - { "Last" + PrefetchArgs.Folders, lastFoldersString }, - { "Last" + PrefetchArgs.Hydrate, lastHydrateString }, - { "New" + PrefetchArgs.CommitId, commitId }, - { "New" + PrefetchArgs.Files, newFilesString }, - { "New" + PrefetchArgs.Folders, newFoldersString }, - { "New" + PrefetchArgs.Hydrate, hydrateFilesAfterDownload.ToString() }, + { "CacheKey", cacheKey }, + { "CachedCommitId", cachedCommitId ?? "(none)" }, + { "NewCommitId", commitId }, { "Result", isNoop }, }); @@ -580,19 +577,44 @@ private bool IsSymbolicRef(string targetCommitish) private void SavePrefetchArgs(string targetCommit, bool hydrate) { - if (this.lastPrefetchArgs != null) + if (this.prefetchCache != null && this.maxCacheSize > 0) { - this.lastPrefetchArgs.SetValuesAndFlush( - new[] + string cacheKey = ComputeCacheKey(this.FileList, this.FolderList, hydrate); + + Dictionary allEntries = this.prefetchCache.GetAllKeysAndValues(); + if (allEntries.Count >= this.maxCacheSize && !allEntries.ContainsKey(cacheKey)) + { + // Evict one arbitrary entry to make room + using (Dictionary.Enumerator enumerator = allEntries.GetEnumerator()) { - new KeyValuePair(PrefetchArgs.CommitId, targetCommit), - new KeyValuePair(PrefetchArgs.Files, GVFSJsonOptions.Serialize(this.FileList)), - new KeyValuePair(PrefetchArgs.Folders, GVFSJsonOptions.Serialize(this.FolderList)), - new KeyValuePair(PrefetchArgs.Hydrate, hydrate.ToString()), - }); + if (enumerator.MoveNext()) + { + this.prefetchCache.RemoveAndFlush(enumerator.Current.Key); + } + } + } + + this.prefetchCache.SetValueAndFlush(cacheKey, targetCommit); } } + internal static string ComputeCacheKey(List files, List folders, bool hydrate) + { + List sortedFiles = new List(files); + sortedFiles.Sort(StringComparer.Ordinal); + + List sortedFolders = new List(folders); + sortedFolders.Sort(StringComparer.Ordinal); + + string compositeInput = string.Join("\n", + GVFSJsonOptions.Serialize(sortedFiles), + GVFSJsonOptions.Serialize(sortedFolders), + hydrate.ToString()); + + byte[] hashBytes = SHA256.HashData(Encoding.UTF8.GetBytes(compositeInput)); + return Convert.ToHexString(hashBytes); + } + public class FetchException : Exception { public FetchException(string format, params object[] args) @@ -600,13 +622,5 @@ public FetchException(string format, params object[] args) { } } - - private static class PrefetchArgs - { - public const string CommitId = "CommitId"; - public const string Files = "Files"; - public const string Folders = "Folders"; - public const string Hydrate = "Hydrate"; - } } } diff --git a/GVFS/GVFS.UnitTests/Prefetch/BlobPrefetcherTests.cs b/GVFS/GVFS.UnitTests/Prefetch/BlobPrefetcherTests.cs index 21f31a92b..18bd5579f 100644 --- a/GVFS/GVFS.UnitTests/Prefetch/BlobPrefetcherTests.cs +++ b/GVFS/GVFS.UnitTests/Prefetch/BlobPrefetcherTests.cs @@ -1,7 +1,11 @@ -using GVFS.Common.Prefetch; +using GVFS.Common; +using GVFS.Common.Prefetch; using GVFS.Tests.Should; +using GVFS.UnitTests.Mock; +using GVFS.UnitTests.Mock.Common; using GVFS.UnitTests.Mock.FileSystem; using NUnit.Framework; +using System.Collections.Generic; using System.IO; namespace GVFS.UnitTests.Prefetch @@ -9,6 +13,8 @@ namespace GVFS.UnitTests.Prefetch [TestFixture] public class BlobPrefetcherTests { + private const string MockCacheFileName = "mock:\\prefetch-cache.dat"; + [TestCase] public void AppendToNewlineSeparatedFileTests() { @@ -29,5 +35,207 @@ public void AppendToNewlineSeparatedFileTests() BlobPrefetcher.AppendToNewlineSeparatedFile(fileSystem, testFileName, "expected line 2"); fileSystem.ReadAllText(testFileName).ShouldEqual("existing content\nexpected line 2\n"); } + + [TestCase] + public void ComputeCacheKeyIsDeterministic() + { + List files = new List { "src/a.cs", "src/b.cs" }; + List folders = new List { "src/dir1", "src/dir2" }; + + string key1 = BlobPrefetcher.ComputeCacheKey(files, folders, hydrate: false); + string key2 = BlobPrefetcher.ComputeCacheKey(files, folders, hydrate: false); + + key1.ShouldEqual(key2); + } + + [TestCase] + public void ComputeCacheKeyDiffersForDifferentFiles() + { + List files1 = new List { "src/a.cs" }; + List files2 = new List { "src/b.cs" }; + List folders = new List { "src/dir1" }; + + string key1 = BlobPrefetcher.ComputeCacheKey(files1, folders, hydrate: false); + string key2 = BlobPrefetcher.ComputeCacheKey(files2, folders, hydrate: false); + + key1.ShouldNotEqual(key2); + } + + [TestCase] + public void ComputeCacheKeyDiffersForDifferentFolders() + { + List files = new List { "src/a.cs" }; + List folders1 = new List { "src/dir1" }; + List folders2 = new List { "src/dir2" }; + + string key1 = BlobPrefetcher.ComputeCacheKey(files, folders1, hydrate: false); + string key2 = BlobPrefetcher.ComputeCacheKey(files, folders2, hydrate: false); + + key1.ShouldNotEqual(key2); + } + + [TestCase] + public void ComputeCacheKeyDiffersForHydrateFlag() + { + List files = new List { "src/a.cs" }; + List folders = new List { "src/dir1" }; + + string key1 = BlobPrefetcher.ComputeCacheKey(files, folders, hydrate: false); + string key2 = BlobPrefetcher.ComputeCacheKey(files, folders, hydrate: true); + + key1.ShouldNotEqual(key2); + } + + [TestCase] + public void ComputeCacheKeyIsOrderIndependent() + { + List filesA = new List { "src/b.cs", "src/a.cs" }; + List filesB = new List { "src/a.cs", "src/b.cs" }; + List folders = new List { "src/dir1" }; + + string key1 = BlobPrefetcher.ComputeCacheKey(filesA, folders, hydrate: false); + string key2 = BlobPrefetcher.ComputeCacheKey(filesB, folders, hydrate: false); + + key1.ShouldEqual(key2); + } + + [TestCase] + public void ComputeCacheKeyFolderOrderIndependent() + { + List files = new List { "src/a.cs" }; + List foldersA = new List { "src/dir2", "src/dir1" }; + List foldersB = new List { "src/dir1", "src/dir2" }; + + string key1 = BlobPrefetcher.ComputeCacheKey(files, foldersA, hydrate: false); + string key2 = BlobPrefetcher.ComputeCacheKey(files, foldersB, hydrate: false); + + key1.ShouldEqual(key2); + } + + [TestCase] + public void IsNoopPrefetchReturnsFalseWhenCacheIsNull() + { + MockTracer tracer = new MockTracer(); + List files = new List { "src/a.cs" }; + List folders = new List { "src/dir1" }; + + BlobPrefetcher.IsNoopPrefetch(tracer, null, "abc123", files, folders, false).ShouldEqual(false); + } + + [TestCase] + public void IsNoopPrefetchReturnsFalseWhenCacheIsEmpty() + { + MockTracer tracer = new MockTracer(); + List files = new List { "src/a.cs" }; + List folders = new List { "src/dir1" }; + FileBasedDictionary cache = CreateEmptyCache(); + + BlobPrefetcher.IsNoopPrefetch(tracer, cache, "abc123", files, folders, false).ShouldEqual(false); + } + + [TestCase] + public void IsNoopPrefetchReturnsTrueOnCacheHit() + { + MockTracer tracer = new MockTracer(); + List files = new List { "src/a.cs" }; + List folders = new List { "src/dir1" }; + string commitId = "abc123"; + + FileBasedDictionary cache = CreateEmptyCache(); + string cacheKey = BlobPrefetcher.ComputeCacheKey(files, folders, hydrate: false); + cache.SetValueAndFlush(cacheKey, commitId); + + BlobPrefetcher.IsNoopPrefetch(tracer, cache, commitId, files, folders, false).ShouldEqual(true); + } + + [TestCase] + public void IsNoopPrefetchReturnsFalseWhenCommitIdChanged() + { + MockTracer tracer = new MockTracer(); + List files = new List { "src/a.cs" }; + List folders = new List { "src/dir1" }; + + FileBasedDictionary cache = CreateEmptyCache(); + string cacheKey = BlobPrefetcher.ComputeCacheKey(files, folders, hydrate: false); + cache.SetValueAndFlush(cacheKey, "oldcommit"); + + BlobPrefetcher.IsNoopPrefetch(tracer, cache, "newcommit", files, folders, false).ShouldEqual(false); + } + + [TestCase] + public void IsNoopPrefetchSupportsMultipleEntries() + { + MockTracer tracer = new MockTracer(); + List filesA = new List { "src/a.cs" }; + List filesB = new List { "src/b.cs" }; + List folders = new List { "src/dir1" }; + string commitId = "abc123"; + + FileBasedDictionary cache = CreateEmptyCache(); + + string keyA = BlobPrefetcher.ComputeCacheKey(filesA, folders, hydrate: false); + cache.SetValueAndFlush(keyA, commitId); + + string keyB = BlobPrefetcher.ComputeCacheKey(filesB, folders, hydrate: false); + cache.SetValueAndFlush(keyB, commitId); + + // Both should hit + BlobPrefetcher.IsNoopPrefetch(tracer, cache, commitId, filesA, folders, false).ShouldEqual(true); + BlobPrefetcher.IsNoopPrefetch(tracer, cache, commitId, filesB, folders, false).ShouldEqual(true); + + // A third pattern should miss + List filesC = new List { "src/c.cs" }; + BlobPrefetcher.IsNoopPrefetch(tracer, cache, commitId, filesC, folders, false).ShouldEqual(false); + } + + private static FileBasedDictionary CreateEmptyCache() + { + CacheFileSystem fs = new CacheFileSystem(); + fs.ExpectedFiles.Add(MockCacheFileName, new ReusableMemoryStream(string.Empty)); + fs.ExpectedOpenFileStreams.Add(MockCacheFileName + ".tmp", new ReusableMemoryStream(string.Empty)); + fs.ExpectedOpenFileStreams.Add(MockCacheFileName, fs.ExpectedFiles[MockCacheFileName]); + + FileBasedDictionary.TryCreate( + null, + MockCacheFileName, + fs, + out FileBasedDictionary cache, + out string error).ShouldEqual(true, error); + + fs.ExpectedOpenFileStreams.Remove(MockCacheFileName); + return cache; + } + + private class CacheFileSystem : ConfigurableFileSystem + { + public CacheFileSystem() + { + this.ExpectedOpenFileStreams = new Dictionary(); + } + + public Dictionary ExpectedOpenFileStreams { get; } + + public override Stream OpenFileStream(string path, FileMode fileMode, FileAccess fileAccess, FileShare shareMode, FileOptions options, bool flushesToDisk) + { + this.ExpectedOpenFileStreams.TryGetValue(path, out ReusableMemoryStream stream); + + if (fileMode == FileMode.Create) + { + this.ExpectedFiles[path] = new ReusableMemoryStream(string.Empty); + } + + this.ExpectedFiles.TryGetValue(path, out stream).ShouldEqual(true, "Unexpected access of file: " + path); + return stream; + } + + public override void MoveAndOverwriteFile(string sourceFileName, string destinationFilename) + { + this.ExpectedFiles.TryGetValue(sourceFileName, out ReusableMemoryStream source).ShouldEqual(true, "Source file does not exist: " + sourceFileName); + this.ExpectedFiles.ContainsKey(destinationFilename).ShouldEqual(true, "MoveAndOverwriteFile expects the destination file to exist: " + destinationFilename); + + this.ExpectedFiles.Remove(sourceFileName); + this.ExpectedFiles[destinationFilename] = source; + } + } } } \ No newline at end of file diff --git a/GVFS/GVFS/CommandLine/PrefetchVerb.cs b/GVFS/GVFS/CommandLine/PrefetchVerb.cs index 1d3d555b4..92cc004a2 100644 --- a/GVFS/GVFS/CommandLine/PrefetchVerb.cs +++ b/GVFS/GVFS/CommandLine/PrefetchVerb.cs @@ -187,11 +187,12 @@ protected override void Execute(GVFSEnlistment enlistment) string headCommitId; List filesList; List foldersList; - FileBasedDictionary lastPrefetchArgs; + FileBasedDictionary prefetchCache; + int prefetchCacheSize; - this.LoadBlobPrefetchArgs(tracer, enlistment, out headCommitId, out filesList, out foldersList, out lastPrefetchArgs); + this.LoadBlobPrefetchArgs(tracer, enlistment, out headCommitId, out filesList, out foldersList, out prefetchCache, out prefetchCacheSize); - if (BlobPrefetcher.IsNoopPrefetch(tracer, lastPrefetchArgs, headCommitId, filesList, foldersList, this.HydrateFiles)) + if (BlobPrefetcher.IsNoopPrefetch(tracer, prefetchCache, headCommitId, filesList, foldersList, this.HydrateFiles)) { Console.WriteLine("All requested files are already available. Nothing new to prefetch."); } @@ -205,7 +206,7 @@ protected override void Execute(GVFSEnlistment enlistment) cacheServerFromConfig, out objectRequestor, out resolvedCacheServer); - this.PrefetchBlobs(tracer, enlistment, headCommitId, filesList, foldersList, lastPrefetchArgs, objectRequestor, resolvedCacheServer); + this.PrefetchBlobs(tracer, enlistment, headCommitId, filesList, foldersList, prefetchCache, prefetchCacheSize, objectRequestor, resolvedCacheServer); } } } @@ -328,18 +329,38 @@ private void LoadBlobPrefetchArgs( out string headCommitId, out List filesList, out List foldersList, - out FileBasedDictionary lastPrefetchArgs) + out FileBasedDictionary prefetchCache, + out int prefetchCacheSize) { string error; - if (!FileBasedDictionary.TryCreate( - tracer, - Path.Combine(enlistment.DotGVFSRoot, "LastBlobPrefetch.dat"), - new PhysicalFileSystem(), - out lastPrefetchArgs, - out error)) + // Read cache size from git config + prefetchCacheSize = BlobPrefetcher.DefaultPrefetchCacheSize; + GitProcess gitProcess = new GitProcess(enlistment); + if (gitProcess.TryGetFromConfig(BlobPrefetcher.PrefetchCacheSizeConfigKey, forceOutsideEnlistment: false, out string cacheSizeValue)) + { + if (int.TryParse(cacheSizeValue, out int parsedSize)) + { + prefetchCacheSize = Math.Clamp(parsedSize, 0, BlobPrefetcher.MaxPrefetchCacheSize); + } + else + { + tracer.RelatedWarning($"Invalid value '{cacheSizeValue}' for {BlobPrefetcher.PrefetchCacheSizeConfigKey}, using default {BlobPrefetcher.DefaultPrefetchCacheSize}"); + } + } + + prefetchCache = null; + if (prefetchCacheSize > 0) { - tracer.RelatedWarning("Unable to load last prefetch args: " + error); + if (!FileBasedDictionary.TryCreate( + tracer, + Path.Combine(enlistment.DotGVFSRoot, BlobPrefetcher.BlobPrefetchCacheFile), + new PhysicalFileSystem(), + out prefetchCache, + out error)) + { + tracer.RelatedWarning("Unable to load prefetch cache: " + error); + } } filesList = new List(); @@ -355,7 +376,6 @@ private void LoadBlobPrefetchArgs( this.ReportErrorAndExit(tracer, error); } - GitProcess gitProcess = new GitProcess(enlistment); GitProcess.Result result = gitProcess.RevParse(GVFSConstants.DotGit.HeadName); if (result.ExitCodeIsFailure) { @@ -371,7 +391,8 @@ private void PrefetchBlobs( string headCommitId, List filesList, List foldersList, - FileBasedDictionary lastPrefetchArgs, + FileBasedDictionary prefetchCache, + int prefetchCacheSize, GitObjectsHttpRequestor objectRequestor, CacheServerInfo cacheServer) { @@ -381,7 +402,8 @@ private void PrefetchBlobs( objectRequestor, filesList, foldersList, - lastPrefetchArgs, + prefetchCache, + prefetchCacheSize, ChunkSize, SearchThreadCount, DownloadThreadCount, From 07647623f0b77a196bae8a3ef5cb0a1b1dfc07ab Mon Sep 17 00:00:00 2001 From: Tyrie Vella Date: Thu, 4 Jun 2026 12:53:00 -0700 Subject: [PATCH 2/2] Fix PrefetchVerbTests: clear cache before each test The multi-entry prefetch cache persists across ordered tests, causing cache hits where the tests expect fresh prefetch work. Delete BlobPrefetchCache.dat in [SetUp] so each test starts with a clean cache. Assisted-by: Claude Opus 4.6 Signed-off-by: Tyrie Vella --- .../Tests/EnlistmentPerFixture/PrefetchVerbTests.cs | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/PrefetchVerbTests.cs b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/PrefetchVerbTests.cs index a56cab338..39b7cc5ee 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/PrefetchVerbTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/PrefetchVerbTests.cs @@ -37,6 +37,16 @@ public PrefetchVerbTests() this.fileSystem = new SystemIORunner(); } + [SetUp] + public void DeletePrefetchCache() + { + string cachePath = Path.Combine(this.Enlistment.DotGVFSRoot, "BlobPrefetchCache.dat"); + if (File.Exists(cachePath)) + { + File.Delete(cachePath); + } + } + [TestCase, Order(1)] public void PrefetchAllMustBeExplicit() {