Skip to content

Commit

Permalink
Improve folder enumeration speed (#4922)
Browse files Browse the repository at this point in the history
  • Loading branch information
@gave92
gave92 authored May 23, 2021
1 parent de5dd1b commit ea7d908
Show file tree
Hide file tree
Showing 62 changed files with 122 additions and 1,103 deletions.
1 change: 0 additions & 1 deletion Files/Files.csproj
View file
Original file line number Diff line number Diff line change
Expand Up @@ -261,7 +261,6 @@
<Compile Include="Helpers\CollectionDebugView.cs" />
<Compile Include="Helpers\DynamicDialogFactory.cs" />
<Compile Include="Helpers\Extension.cs" />
<Compile Include="Helpers\FileListCache\CacheEntry.cs" />
<Compile Include="Helpers\FileListCache\FileListCacheController.cs" />
<Compile Include="Helpers\FileListCache\IFileListCache.cs" />
<Compile Include="Helpers\FileListCache\PersistentSQLiteCacheAdapter.cs" />
Expand Down
19 changes: 2 additions & 17 deletions Files/Filesystem/StorageEnumerators/UniversalStorageEnumerator.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,6 @@ public static async Task<List<ListedItem>> ListEntries(
string returnformat,
Type sourcePageType,
CancellationToken cancellationToken,
List<string> skipItems,
int countLimit,
Func<List<ListedItem>, Task> intermediateAction
)
Expand Down Expand Up @@ -74,14 +73,7 @@ ex is UnauthorizedAccessException
var folder = await AddFolderAsync(item as StorageFolder, currentStorageFolder, returnformat, cancellationToken);
if (folder != null)
{
if (skipItems?.Contains(folder.ItemPath) ?? false)
{
skipItems.Remove(folder.ItemPath);
}
else
{
tempList.Add(folder);
}
tempList.Add(folder);
}
}
else
Expand All @@ -90,14 +82,7 @@ ex is UnauthorizedAccessException
var fileEntry = await AddFileAsync(file, currentStorageFolder, returnformat, true, sourcePageType, cancellationToken);
if (fileEntry != null)
{
if (skipItems?.Contains(fileEntry.ItemPath) ?? false)
{
skipItems.Remove(fileEntry.ItemPath);
}
else
{
tempList.Add(fileEntry);
}
tempList.Add(fileEntry);
}
}
if (cancellationToken.IsCancellationRequested)
Expand Down
19 changes: 2 additions & 17 deletions Files/Filesystem/StorageEnumerators/Win32StorageEnumerator.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,6 @@ public static async Task<List<ListedItem>> ListEntries(
WIN32_FIND_DATA findData,
NamedPipeAsAppServiceConnection connection,
CancellationToken cancellationToken,
List<string> skipItems,
int countLimit,
Func<List<ListedItem>, Task> intermediateAction
)
Expand All @@ -49,14 +48,7 @@ Func<List<ListedItem>, Task> intermediateAction
var file = await GetFile(findData, path, returnformat, connection, cancellationToken);
if (file != null)
{
if (skipItems?.Contains(file.ItemPath) ?? false)
{
skipItems.Remove(file.ItemPath);
}
else
{
tempList.Add(file);
}
tempList.Add(file);
++count;
}
}
Expand All @@ -67,14 +59,7 @@ Func<List<ListedItem>, Task> intermediateAction
var folder = await GetFolder(findData, path, returnformat, cancellationToken);
if (folder != null)
{
if (skipItems?.Contains(folder.ItemPath) ?? false)
{
skipItems.Remove(folder.ItemPath);
}
else
{
tempList.Add(folder);
}
tempList.Add(folder);
++count;
}
}
Expand Down
11 changes: 0 additions & 11 deletions Files/Helpers/FileListCache/CacheEntry.cs
View file

This file was deleted.

48 changes: 0 additions & 48 deletions Files/Helpers/FileListCache/FileListCacheController.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,59 +20,11 @@ private FileListCacheController()
persistentAdapter = new PersistentSQLiteCacheAdapter();
}

private readonly IMemoryCache filesCache = new MemoryCache(new MemoryCacheOptions
{
SizeLimit = 1_000_000
});

private readonly IMemoryCache fileNamesCache = new MemoryCache(new MemoryCacheOptions
{
SizeLimit = 1_000_000
});

public Task SaveFileListToCache(string path, CacheEntry cacheEntry)
{
if (!App.AppSettings.UseFileListCache)
{
return Task.CompletedTask;
}

if (cacheEntry == null)
{
filesCache.Remove(path);
return persistentAdapter.SaveFileListToCache(path, cacheEntry);
}
filesCache.Set(path, cacheEntry, new MemoryCacheEntryOptions
{
Size = cacheEntry.FileList.Count
});

// save entry to persistent cache in background
return persistentAdapter.SaveFileListToCache(path, cacheEntry);
}

public async Task<CacheEntry> ReadFileListFromCache(string path, CancellationToken cancellationToken)
{
if (!App.AppSettings.UseFileListCache)
{
return null;
}

var entry = filesCache.Get<CacheEntry>(path);
if (entry == null)
{
entry = await persistentAdapter.ReadFileListFromCache(path, cancellationToken);
if (entry?.FileList != null)
{
filesCache.Set(path, entry, new MemoryCacheEntryOptions
{
Size = entry.FileList.Count
});
}
}
return entry;
}

public async Task<string> ReadFileDisplayNameFromCache(string path, CancellationToken cancellationToken)
{
var displayName = fileNamesCache.Get<string>(path);
Expand Down
4 changes: 0 additions & 4 deletions Files/Helpers/FileListCache/IFileListCache.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,10 +5,6 @@ namespace Files.Helpers.FileListCache
{
internal interface IFileListCache
{
public Task<CacheEntry> ReadFileListFromCache(string path, CancellationToken cancellationToken);

public Task SaveFileListToCache(string path, CacheEntry cacheEntry);

public Task<string> ReadFileDisplayNameFromCache(string path, CancellationToken cancellationToken);

public Task SaveFileDisplayNameToCache(string path, string displayName);
Expand Down
123 changes: 0 additions & 123 deletions Files/Helpers/FileListCache/PersistentSQLiteCacheAdapter.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,98 +15,6 @@ internal class PersistentSQLiteCacheAdapter : IFileListCache, IDisposable
private SqliteConnection connection;
private bool disposedValue;

public async Task SaveFileListToCache(string path, CacheEntry cacheEntry)
{
if (!await InitializeIfNeeded())
{
return;
}
const int maxCachedEntries = 128;
try
{
if (cacheEntry == null)
{
using var deleteCommand = new SqliteCommand("DELETE FROM FileListCache WHERE Id = @Id", connection);
deleteCommand.Parameters.Add("@Id", SqliteType.Text).Value = path;
await deleteCommand.ExecuteNonQueryAsync();
return;
}

if (cacheEntry.FileList.Count > maxCachedEntries)
{
cacheEntry.FileList = cacheEntry.FileList.Take(maxCachedEntries).ToList();
}

using var cmd = new SqliteCommand("SELECT Id FROM FileListCache WHERE Id = @Id", connection);
cmd.Parameters.Add("@Id", SqliteType.Text).Value = path;
using var reader = await cmd.ExecuteReaderAsync();
if (reader.HasRows)
{
// need to update entry
using var updateCommand = new SqliteCommand("UPDATE FileListCache SET Timestamp = @Timestamp, Entry = @Entry WHERE Id = @Id", connection);
updateCommand.Parameters.Add("@Id", SqliteType.Text).Value = path;
updateCommand.Parameters.Add("@Timestamp", SqliteType.Integer).Value = GetTimestamp(DateTime.UtcNow);
var settings = new JsonSerializerSettings
{
TypeNameHandling = TypeNameHandling.Auto
};
updateCommand.Parameters.Add("@Entry", SqliteType.Text).Value = JsonConvert.SerializeObject(cacheEntry, settings);
await updateCommand.ExecuteNonQueryAsync();
}
else
{
// need to insert entry
using var insertCommand = new SqliteCommand("INSERT INTO FileListCache (Id, Timestamp, Entry) VALUES (@Id, @Timestamp, @Entry)", connection);
insertCommand.Parameters.Add("@Id", SqliteType.Text).Value = path;
insertCommand.Parameters.Add("@Timestamp", SqliteType.Integer).Value = GetTimestamp(DateTime.UtcNow);
var settings = new JsonSerializerSettings
{
TypeNameHandling = TypeNameHandling.Auto
};
insertCommand.Parameters.Add("@Entry", SqliteType.Text).Value = JsonConvert.SerializeObject(cacheEntry, settings);
await insertCommand.ExecuteNonQueryAsync();
}
}
catch (Exception ex)
{
NLog.LogManager.GetCurrentClassLogger().Warn(ex, ex.Message);
}
}

public async Task<CacheEntry> ReadFileListFromCache(string path, CancellationToken cancellationToken)
{
if (!await InitializeIfNeeded())
{
return null;
}
try
{
using var cmd = new SqliteCommand("SELECT Timestamp, Entry FROM FileListCache WHERE Id = @Id", connection);
cmd.Parameters.Add("@Id", SqliteType.Text).Value = path;

using var reader = await cmd.ExecuteReaderAsync(cancellationToken);
if (!await reader.ReadAsync())
{
return null;
}
var timestamp = reader.GetInt64(0);
var entryAsJson = reader.GetString(1);
var settings = new JsonSerializerSettings
{
TypeNameHandling = TypeNameHandling.Auto
};
var entry = JsonConvert.DeserializeObject<CacheEntry>(entryAsJson, settings);
entry.CurrentFolder.ItemPropertiesInitialized = false;
entry.FileList.ForEach((item) => item.ItemPropertiesInitialized = false);
return entry;
}
catch (Exception ex)
{
NLog.LogManager.GetCurrentClassLogger().Warn(ex, ex.Message);
return null;
}
}

public async Task SaveFileDisplayNameToCache(string path, string displayName)
{
if (!await InitializeIfNeeded())
Expand Down Expand Up @@ -174,11 +82,6 @@ public async Task<string> ReadFileDisplayNameFromCache(string path, Cancellation
}
}

private long GetTimestamp(DateTime dateTime)
{
return new DateTimeOffset(dateTime).ToUnixTimeSeconds();
}

public void Dispose()
{
if (!disposedValue)
Expand All @@ -190,23 +93,6 @@ public void Dispose()

private void RunCleanupRoutine()
{
Task.Run(async () =>
{
try
{
// remove entries that are 1 month old (timestamp is updated every time the cache is set)
var limitTimestamp = GetTimestamp(DateTime.Now.AddMonths(-1));
using var cmd = new SqliteCommand("DELETE FROM FileListCache WHERE Timestamp < @Timestamp", connection);
cmd.Parameters.Add("@Timestamp", SqliteType.Integer).Value = limitTimestamp;

var count = await cmd.ExecuteNonQueryAsync().ConfigureAwait(false);
Debug.WriteLine($"Removed {count} old entries from cache database");
}
catch (Exception ex)
{
NLog.LogManager.GetCurrentClassLogger().Warn(ex, ex.Message);
}
});
}

private async Task<bool> InitializeIfNeeded()
Expand All @@ -227,15 +113,6 @@ private async Task<bool> InitializeIfNeeded()
connection.Open();

// create db schema
var createFileListCacheTable = @"CREATE TABLE IF NOT EXISTS ""FileListCache"" (
""Id"" VARCHAR(5000) NOT NULL,
""Timestamp"" INTEGER NOT NULL,
""Entry"" TEXT NOT NULL,
PRIMARY KEY(""Id"")
)";
using var cmdFileListCacheTable = new SqliteCommand(createFileListCacheTable, connection);
cmdFileListCacheTable.ExecuteNonQuery();

var createFileDisplayNameCacheTable = @"CREATE TABLE IF NOT EXISTS ""FileDisplayNameCache"" (
""Id"" VARCHAR(5000) NOT NULL,
""DisplayName"" TEXT NOT NULL,
Expand Down
12 changes: 0 additions & 12 deletions Files/MultilingualResources/Files.ar.xlf
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1845,10 +1845,6 @@
<source>Original path</source>
<target state="translated">المسار الأصلي</target>
</trans-unit>
<trans-unit id="SettingsUseFileListCache.Header" translate="yes" xml:space="preserve">
<source>Cache files and folders for better performance</source>
<target state="translated">ملفات ومجلدات ذاكرة التخزين المؤقت لأداء أفضل</target>
</trans-unit>
<trans-unit id="AppBarButtonAdd.Label" translate="yes" xml:space="preserve">
<source>Add</source>
<target state="translated" state-qualifier="tm-suggestion">إضافة</target>
Expand Down Expand Up @@ -2249,14 +2245,6 @@
<source>Disconnect</source>
<target state="needs-review-translation" state-qualifier="tm-suggestion">قطع الاتصال</target>
</trans-unit>
<trans-unit id="PreemptiveCacheParallelLimit.Header" translate="yes" xml:space="preserve">
<source>Preemptive cache parallel limit (smaller numbers should work better for hard drives)</source>
<target state="new">Preemptive cache parallel limit (smaller numbers should work better for hard drives)</target>
</trans-unit>
<trans-unit id="SettingsUsePreemptiveCache.Header" translate="yes" xml:space="preserve">
<source>Use preemptive cache (preload entries in child directories on navigation)</source>
<target state="new">Use preemptive cache (preload entries in child directories on navigation)</target>
</trans-unit>
<trans-unit id="BundlesWidgetMoreOptionsButton.AutomationProperties.Name" translate="yes" xml:space="preserve">
<source>More bundles options</source>
<target state="new">More bundles options</target>
Expand Down
12 changes: 0 additions & 12 deletions Files/MultilingualResources/Files.cs-CZ.xlf
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1860,10 +1860,6 @@
<source>Original path</source>
<target state="translated">Umístění</target>
</trans-unit>
<trans-unit id="SettingsUseFileListCache.Header" translate="yes" xml:space="preserve">
<source>Cache files and folders for better performance</source>
<target state="translated">Indexovat soubory a složky pro lepší výkon</target>
</trans-unit>
<trans-unit id="AppBarButtonAdd.Label" translate="yes" xml:space="preserve">
<source>Add</source>
<target state="translated">Přidat</target>
Expand Down Expand Up @@ -2262,14 +2258,6 @@
<source>Disconnect</source>
<target state="needs-review-translation" state-qualifier="tm-suggestion">Odpojit</target>
</trans-unit>
<trans-unit id="PreemptiveCacheParallelLimit.Header" translate="yes" xml:space="preserve">
<source>Preemptive cache parallel limit (smaller numbers should work better for hard drives)</source>
<target state="new">Preemptive cache parallel limit (smaller numbers should work better for hard drives)</target>
</trans-unit>
<trans-unit id="SettingsUsePreemptiveCache.Header" translate="yes" xml:space="preserve">
<source>Use preemptive cache (preload entries in child directories on navigation)</source>
<target state="new">Use preemptive cache (preload entries in child directories on navigation)</target>
</trans-unit>
<trans-unit id="BundlesWidgetMoreOptionsButton.AutomationProperties.Name" translate="yes" xml:space="preserve">
<source>More bundles options</source>
<target state="new">More bundles options</target>
Expand Down
Loading

0 comments on commit ea7d908

Please sign in to comment.