Skip to content

Expand blob prefetch noop cache to N entries #2004

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
tyrielv wants to merge 2 commits into
base: master
Choose a base branch
from
Open

Expand blob prefetch noop cache to N entries #2004

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
b9ce141
Expand blob prefetch noop cache to N entries
tyrielv Jun 4, 2026
0764762
Fix PrefetchVerbTests: clear cache before each test
tyrielv Jun 4, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
96 changes: 55 additions & 41 deletions GVFS/GVFS.Common/Prefetch/BlobPrefetcher.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,8 @@
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Security.Cryptography;
using System.Text;
using System.Threading;

namespace GVFS.Common.Prefetch
Expand All @@ -32,7 +34,13 @@ public class BlobPrefetcher
private const string AreaPath = nameof(BlobPrefetcher);
private static string pathSeparatorString = Path.DirectorySeparatorChar.ToString();

private FileBasedDictionary<string, string> lastPrefetchArgs;
public const string BlobPrefetchCacheFile = "BlobPrefetchCache.dat";
public const string PrefetchCacheSizeConfigKey = GVFSConstants.GitConfig.GVFSPrefix + "prefetch-cache-size";
public const int DefaultPrefetchCacheSize = 100;
public const int MaxPrefetchCacheSize = 1000;

private FileBasedDictionary<string, string> prefetchCache;
private int maxCacheSize;

public BlobPrefetcher(
ITracer tracer,
Expand All @@ -42,7 +50,7 @@ public BlobPrefetcher(
int searchThreadCount,
int downloadThreadCount,
int indexThreadCount)
: this(tracer, enlistment, objectRequestor, null, null, null, chunkSize, searchThreadCount, downloadThreadCount, indexThreadCount)
: this(tracer, enlistment, objectRequestor, null, null, null, DefaultPrefetchCacheSize, chunkSize, searchThreadCount, downloadThreadCount, indexThreadCount)
{
}

Expand All @@ -52,7 +60,8 @@ public BlobPrefetcher(
GitObjectsHttpRequestor objectRequestor,
List<string> fileList,
List<string> folderList,
FileBasedDictionary<string, string> lastPrefetchArgs,
FileBasedDictionary<string, string> prefetchCache,
int maxCacheSize,
int chunkSize,
int searchThreadCount,
int downloadThreadCount,
Expand All @@ -70,7 +79,8 @@ public BlobPrefetcher(
this.FileList = fileList ?? new List<string>();
this.FolderList = folderList ?? new List<string>();

this.lastPrefetchArgs = lastPrefetchArgs;
this.prefetchCache = prefetchCache;
this.maxCacheSize = maxCacheSize;

// We never want to update config settings for a GVFSEnlistment
this.SkipConfigUpdate = enlistment is GVFSEnlistment;
Expand Down Expand Up @@ -127,39 +137,26 @@ public static bool TryLoadFileList(Enlistment enlistment, string filesInput, str

public static bool IsNoopPrefetch(
ITracer tracer,
FileBasedDictionary<string, string> lastPrefetchArgs,
FileBasedDictionary<string, string> prefetchCache,
string commitId,
List<string> files,
List<string> folders,
bool hydrateFilesAfterDownload)
{
if (lastPrefetchArgs != null &&
lastPrefetchArgs.TryGetValue(PrefetchArgs.CommitId, out string lastCommitId) &&
lastPrefetchArgs.TryGetValue(PrefetchArgs.Files, out string lastFilesString) &&
lastPrefetchArgs.TryGetValue(PrefetchArgs.Folders, out string lastFoldersString) &&
lastPrefetchArgs.TryGetValue(PrefetchArgs.Hydrate, out string lastHydrateString))
if (prefetchCache != null)
{
string newFilesString = GVFSJsonOptions.Serialize(files);
string newFoldersString = GVFSJsonOptions.Serialize(folders);
bool isNoop =
commitId == lastCommitId &&
hydrateFilesAfterDownload.ToString() == lastHydrateString &&
newFilesString == lastFilesString &&
newFoldersString == lastFoldersString;
string cacheKey = ComputeCacheKey(files, folders, hydrateFilesAfterDownload);
bool hasEntry = prefetchCache.TryGetValue(cacheKey, out string cachedCommitId);
bool isNoop = hasEntry && commitId == cachedCommitId;

tracer.RelatedEvent(
EventLevel.Informational,
"BlobPrefetcher.IsNoopPrefetch",
new EventMetadata
{
{ "Last" + PrefetchArgs.CommitId, lastCommitId },
{ "Last" + PrefetchArgs.Files, lastFilesString },
{ "Last" + PrefetchArgs.Folders, lastFoldersString },
{ "Last" + PrefetchArgs.Hydrate, lastHydrateString },
{ "New" + PrefetchArgs.CommitId, commitId },
{ "New" + PrefetchArgs.Files, newFilesString },
{ "New" + PrefetchArgs.Folders, newFoldersString },
{ "New" + PrefetchArgs.Hydrate, hydrateFilesAfterDownload.ToString() },
{ "CacheKey", cacheKey },
{ "CachedCommitId", cachedCommitId ?? "(none)" },
{ "NewCommitId", commitId },
{ "Result", isNoop },
});

Expand Down Expand Up @@ -580,33 +577,50 @@ private bool IsSymbolicRef(string targetCommitish)

private void SavePrefetchArgs(string targetCommit, bool hydrate)
{
if (this.lastPrefetchArgs != null)
if (this.prefetchCache != null && this.maxCacheSize > 0)
{
this.lastPrefetchArgs.SetValuesAndFlush(
new[]
string cacheKey = ComputeCacheKey(this.FileList, this.FolderList, hydrate);

Dictionary<string, string> allEntries = this.prefetchCache.GetAllKeysAndValues();
if (allEntries.Count >= this.maxCacheSize && !allEntries.ContainsKey(cacheKey))
{
// Evict one arbitrary entry to make room
using (Dictionary<string, string>.Enumerator enumerator = allEntries.GetEnumerator())
{
new KeyValuePair<string, string>(PrefetchArgs.CommitId, targetCommit),
new KeyValuePair<string, string>(PrefetchArgs.Files, GVFSJsonOptions.Serialize(this.FileList)),
new KeyValuePair<string, string>(PrefetchArgs.Folders, GVFSJsonOptions.Serialize(this.FolderList)),
new KeyValuePair<string, string>(PrefetchArgs.Hydrate, hydrate.ToString()),
});
if (enumerator.MoveNext())
{
this.prefetchCache.RemoveAndFlush(enumerator.Current.Key);
}
}
}

this.prefetchCache.SetValueAndFlush(cacheKey, targetCommit);
}
}

internal static string ComputeCacheKey(List<string> files, List<string> folders, bool hydrate)
{
List<string> sortedFiles = new List<string>(files);
sortedFiles.Sort(StringComparer.Ordinal);

List<string> sortedFolders = new List<string>(folders);
sortedFolders.Sort(StringComparer.Ordinal);

string compositeInput = string.Join("\n",
GVFSJsonOptions.Serialize(sortedFiles),
GVFSJsonOptions.Serialize(sortedFolders),
hydrate.ToString());

byte[] hashBytes = SHA256.HashData(Encoding.UTF8.GetBytes(compositeInput));
return Convert.ToHexString(hashBytes);
}

public class FetchException : Exception
{
public FetchException(string format, params object[] args)
: base(string.Format(format, args))
{
}
}

private static class PrefetchArgs
{
public const string CommitId = "CommitId";
public const string Files = "Files";
public const string Folders = "Folders";
public const string Hydrate = "Hydrate";
}
}
}
10 changes: 10 additions & 0 deletions GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/PrefetchVerbTests.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,16 @@ public PrefetchVerbTests()
this.fileSystem = new SystemIORunner();
}

[SetUp]
public void DeletePrefetchCache()
{
string cachePath = Path.Combine(this.Enlistment.DotGVFSRoot, "BlobPrefetchCache.dat");
if (File.Exists(cachePath))
{
File.Delete(cachePath);
}
}

[TestCase, Order(1)]
public void PrefetchAllMustBeExplicit()
{
Expand Down
Loading
Loading