From 9f48540fb3048439021236578dedea1fc75ad7cb Mon Sep 17 00:00:00 2001 From: Steve Date: Thu, 2 Sep 2021 15:06:24 +0800 Subject: [PATCH 01/23] Improve performance of watcher --- Files/ViewModels/ItemViewModel.cs | 62 ++++++++++++++++++------------- 1 file changed, 37 insertions(+), 25 deletions(-) diff --git a/Files/ViewModels/ItemViewModel.cs b/Files/ViewModels/ItemViewModel.cs index 4bae49dda194..a875fb77e566 100644 --- a/Files/ViewModels/ItemViewModel.cs +++ b/Files/ViewModels/ItemViewModel.cs @@ -1714,8 +1714,10 @@ private void WatchForDirectoryChanges(string path, CloudDriveSyncStatus syncStat return; } + var hasSyncStatus = syncStatus != CloudDriveSyncStatus.NotSynced && syncStatus != CloudDriveSyncStatus.Unknown; + var cts = new CancellationTokenSource(); - _ = Windows.System.Threading.ThreadPool.RunAsync((x) => ProcessOperationQueue(cts.Token)); + _ = Windows.System.Threading.ThreadPool.RunAsync((x) => ProcessOperationQueue(cts.Token, hasSyncStatus)); aWatcherAction = Windows.System.Threading.ThreadPool.RunAsync((x) => { @@ -1724,7 +1726,7 @@ private void WatchForDirectoryChanges(string path, CloudDriveSyncStatus syncStat buff = new byte[4096]; int notifyFilters = FILE_NOTIFY_CHANGE_DIR_NAME | FILE_NOTIFY_CHANGE_FILE_NAME | FILE_NOTIFY_CHANGE_LAST_WRITE | FILE_NOTIFY_CHANGE_SIZE; - if (syncStatus != CloudDriveSyncStatus.NotSynced && syncStatus != CloudDriveSyncStatus.Unknown) + if (hasSyncStatus) { notifyFilters |= FILE_NOTIFY_CHANGE_ATTRIBUTES; } @@ -1804,7 +1806,7 @@ private void WatchForDirectoryChanges(string path, CloudDriveSyncStatus syncStat Debug.WriteLine("Task exiting..."); } - private async void ProcessOperationQueue(CancellationToken cancellationToken) + private async void ProcessOperationQueue(CancellationToken cancellationToken, bool hasSyncStatus) { ApplicationDataContainer localSettings = ApplicationData.Current.LocalSettings; string returnformat = Enum.Parse(localSettings.Values[Constants.LocalSettings.DateTimeFormat].ToString()) == TimeStyle.Application ? "D" : "g"; @@ -1816,6 +1818,7 @@ private async void ProcessOperationQueue(CancellationToken cancellationToken) const uint FILE_ACTION_RENAMED_NEW_NAME = 0x00000005; var sampler = new IntervalSampler(200); + var updateList = new HashSet(); bool anyEdits = false; try @@ -1825,6 +1828,8 @@ private async void ProcessOperationQueue(CancellationToken cancellationToken) if (operationEvent.Wait(200, cancellationToken)) { operationEvent.Reset(); + + processQueue: while (operationQueue.TryDequeue(out var operation)) { if (cancellationToken.IsCancellationRequested) break; @@ -1839,7 +1844,7 @@ private async void ProcessOperationQueue(CancellationToken cancellationToken) break; case FILE_ACTION_MODIFIED: - await UpdateFileOrFolderAsync(operation.FileName); + updateList.Add(operation.FileName); break; case FILE_ACTION_REMOVED: @@ -1861,9 +1866,20 @@ private async void ProcessOperationQueue(CancellationToken cancellationToken) anyEdits = false; } } + + foreach (var entry in updateList.ToList()) + { + await UpdateFileOrFolderAsync(entry, hasSyncStatus); + updateList.Remove(entry); + + if (sampler.CheckNow() && operationQueue.Any(i => i.Action != FILE_ACTION_MODIFIED)) + { + goto processQueue; + } + } } - if (anyEdits && sampler.CheckNow()) + if (anyEdits) { await OrderFilesAndFoldersAsync(); await ApplyFilesAndFoldersChangesAsync(); @@ -1959,26 +1975,22 @@ public ListedItem AddFileOrFolderFromShellFile(ShellFileItem item, string dateRe private async Task AddFileOrFolderAsync(ListedItem item) { - try - { - await enumFolderSemaphore.WaitAsync(semaphoreCTS.Token); - } - catch (OperationCanceledException) + if (item == null) { return; } try { - if (item != null) - { - filesAndFolders.Add(item); - } + await enumFolderSemaphore.WaitAsync(semaphoreCTS.Token); } - finally + catch (OperationCanceledException) { - enumFolderSemaphore.Release(); + return; } + + filesAndFolders.Add(item); + enumFolderSemaphore.Release(); } private async Task AddFileOrFolderAsync(string fileOrFolderPath, string dateReturnFormat) @@ -2015,13 +2027,10 @@ private async Task AddFileOrFolderAsync(string fileOrFolderPath, string dateRetu listedItem = await Win32StorageEnumerator.GetFile(findData, Directory.GetParent(fileOrFolderPath).FullName, dateReturnFormat, Connection, addFilesCTS.Token); } - if (listedItem != null) - { - filesAndFolders.Add(listedItem); - } + await AddFileOrFolderAsync(listedItem); } - private async Task UpdateFileOrFolderAsync(ListedItem item) + private async Task UpdateFileOrFolderAsync(ListedItem item, bool hasSyncStatus = false) { IStorageItem storageItem = null; if (item.PrimaryItemAttribute == StorageItemTypes.File) @@ -2034,10 +2043,13 @@ private async Task UpdateFileOrFolderAsync(ListedItem item) } if (storageItem != null) { - var syncStatus = await CheckCloudDriveSyncStatusAsync(storageItem); + var syncStatus = hasSyncStatus ? await CheckCloudDriveSyncStatusAsync(storageItem) : CloudDriveSyncStatus.NotSynced; await CoreApplication.MainView.DispatcherQueue.EnqueueAsync(async () => { - item.SyncStatusUI = CloudDriveSyncStatusUI.FromCloudDriveSyncStatus(syncStatus); + if (hasSyncStatus) + { + item.SyncStatusUI = CloudDriveSyncStatusUI.FromCloudDriveSyncStatus(syncStatus); + } if (storageItem.IsOfType(StorageItemTypes.File)) { @@ -2057,7 +2069,7 @@ await CoreApplication.MainView.DispatcherQueue.EnqueueAsync(async () => } } - private async Task UpdateFileOrFolderAsync(string path) + private async Task UpdateFileOrFolderAsync(string path, bool hasSyncStatus = false) { try { @@ -2074,7 +2086,7 @@ private async Task UpdateFileOrFolderAsync(string path) if (matchingItem != null) { - await UpdateFileOrFolderAsync(matchingItem); + await UpdateFileOrFolderAsync(matchingItem, hasSyncStatus); } } finally From d750af1a817ac2b7179df040811f632d9cf9891b Mon Sep 17 00:00:00 2001 From: Steve Date: Thu, 2 Sep 2021 15:29:17 +0800 Subject: [PATCH 02/23] Bulk update --- Files/ViewModels/ItemViewModel.cs | 85 +++++++++++++++++-------------- 1 file changed, 46 insertions(+), 39 deletions(-) diff --git a/Files/ViewModels/ItemViewModel.cs b/Files/ViewModels/ItemViewModel.cs index a875fb77e566..90c46f80a551 100644 --- a/Files/ViewModels/ItemViewModel.cs +++ b/Files/ViewModels/ItemViewModel.cs @@ -1818,7 +1818,6 @@ private async void ProcessOperationQueue(CancellationToken cancellationToken, bo const uint FILE_ACTION_RENAMED_NEW_NAME = 0x00000005; var sampler = new IntervalSampler(200); - var updateList = new HashSet(); bool anyEdits = false; try @@ -1828,8 +1827,8 @@ private async void ProcessOperationQueue(CancellationToken cancellationToken, bo if (operationEvent.Wait(200, cancellationToken)) { operationEvent.Reset(); + var updateList = new HashSet(); - processQueue: while (operationQueue.TryDequeue(out var operation)) { if (cancellationToken.IsCancellationRequested) break; @@ -1867,16 +1866,7 @@ private async void ProcessOperationQueue(CancellationToken cancellationToken, bo } } - foreach (var entry in updateList.ToList()) - { - await UpdateFileOrFolderAsync(entry, hasSyncStatus); - updateList.Remove(entry); - - if (sampler.CheckNow() && operationQueue.Any(i => i.Action != FILE_ACTION_MODIFIED)) - { - goto processQueue; - } - } + await UpdateFilesOrFoldersAsync(updateList, hasSyncStatus); } if (anyEdits) @@ -2030,7 +2020,7 @@ private async Task AddFileOrFolderAsync(string fileOrFolderPath, string dateRetu await AddFileOrFolderAsync(listedItem); } - private async Task UpdateFileOrFolderAsync(ListedItem item, bool hasSyncStatus = false) + private async Task<(ListedItem Item, CloudDriveSyncStatusUI SyncUI, long? Size, DateTimeOffset Created, DateTimeOffset Modified)?> UpdateFileOrFolderAsync(ListedItem item, bool hasSyncStatus = false) { IStorageItem storageItem = null; if (item.PrimaryItemAttribute == StorageItemTypes.File) @@ -2043,33 +2033,31 @@ private async Task UpdateFileOrFolderAsync(ListedItem item, bool hasSyncStatus = } if (storageItem != null) { - var syncStatus = hasSyncStatus ? await CheckCloudDriveSyncStatusAsync(storageItem) : CloudDriveSyncStatus.NotSynced; - await CoreApplication.MainView.DispatcherQueue.EnqueueAsync(async () => + CloudDriveSyncStatusUI syncUI = hasSyncStatus ? CloudDriveSyncStatusUI.FromCloudDriveSyncStatus(await CheckCloudDriveSyncStatusAsync(storageItem)) : null; + long? size = null; + DateTimeOffset created = default, modified = default; + + if (storageItem.IsOfType(StorageItemTypes.File)) { - if (hasSyncStatus) - { - item.SyncStatusUI = CloudDriveSyncStatusUI.FromCloudDriveSyncStatus(syncStatus); - } + var properties = await storageItem.AsBaseStorageFile().GetBasicPropertiesAsync(); + size = (long)properties.Size; + modified = properties.DateModified; + created = properties.ItemDate; + } + else if (storageItem.IsOfType(StorageItemTypes.Folder)) + { + var properties = await storageItem.AsBaseStorageFolder().GetBasicPropertiesAsync(); + modified = properties.DateModified; + created = properties.ItemDate; + } - if (storageItem.IsOfType(StorageItemTypes.File)) - { - var properties = await storageItem.AsBaseStorageFile().GetBasicPropertiesAsync(); - item.FileSizeBytes = (long)properties.Size; - item.FileSize = ByteSizeLib.ByteSize.FromBytes(item.FileSizeBytes).ToBinaryString().ConvertSizeAbbreviation(); - item.ItemDateModifiedReal = properties.DateModified; - item.ItemDateCreatedReal = properties.ItemDate; - } - else if (storageItem.IsOfType(StorageItemTypes.Folder)) - { - var properties = await storageItem.AsBaseStorageFolder().GetBasicPropertiesAsync(); - item.ItemDateModifiedReal = properties.DateModified; - item.ItemDateCreatedReal = properties.ItemDate; - } - }); + return (item, syncUI, size, created, modified); } + + return null; } - private async Task UpdateFileOrFolderAsync(string path, bool hasSyncStatus = false) + private async Task UpdateFilesOrFoldersAsync(IEnumerable paths, bool hasSyncStatus = false) { try { @@ -2082,12 +2070,31 @@ private async Task UpdateFileOrFolderAsync(string path, bool hasSyncStatus = fal try { - var matchingItem = filesAndFolders.FirstOrDefault(x => x.ItemPath.Equals(path, StringComparison.OrdinalIgnoreCase)); + var matchingItems = filesAndFolders.Where(x => paths.Any(p => p.Equals(x.ItemPath, StringComparison.OrdinalIgnoreCase))); + var results = await Task.WhenAll(matchingItems.Select(x => UpdateFileOrFolderAsync(x, hasSyncStatus))); - if (matchingItem != null) + await CoreApplication.MainView.DispatcherQueue.EnqueueAsync(() => { - await UpdateFileOrFolderAsync(matchingItem, hasSyncStatus); - } + foreach (var result in results) + { + if (result.HasValue) + { + var item = result.Value.Item; + item.ItemDateModifiedReal = result.Value.Modified; + item.ItemDateCreatedReal = result.Value.Created; + + if (result.Value.SyncUI != null) + { + item.SyncStatusUI = result.Value.SyncUI; + } + + if (result.Value.Size.HasValue) + { + item.FileSize = ByteSizeLib.ByteSize.FromBytes(item.FileSizeBytes).ToBinaryString().ConvertSizeAbbreviation(); + } + } + } + }); } finally { From 3739fefcd5761dcd88fc9a471f0867e61df966ee Mon Sep 17 00:00:00 2001 From: Steve Date: Thu, 2 Sep 2021 15:44:53 +0800 Subject: [PATCH 03/23] Rename --- Files/ViewModels/ItemViewModel.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Files/ViewModels/ItemViewModel.cs b/Files/ViewModels/ItemViewModel.cs index 90c46f80a551..b749a6c122e3 100644 --- a/Files/ViewModels/ItemViewModel.cs +++ b/Files/ViewModels/ItemViewModel.cs @@ -2020,7 +2020,7 @@ private async Task AddFileOrFolderAsync(string fileOrFolderPath, string dateRetu await AddFileOrFolderAsync(listedItem); } - private async Task<(ListedItem Item, CloudDriveSyncStatusUI SyncUI, long? Size, DateTimeOffset Created, DateTimeOffset Modified)?> UpdateFileOrFolderAsync(ListedItem item, bool hasSyncStatus = false) + private async Task<(ListedItem Item, CloudDriveSyncStatusUI SyncUI, long? Size, DateTimeOffset Created, DateTimeOffset Modified)?> GetFileOrFolderUpdateInfoAsync(ListedItem item, bool hasSyncStatus = false) { IStorageItem storageItem = null; if (item.PrimaryItemAttribute == StorageItemTypes.File) @@ -2071,7 +2071,7 @@ private async Task UpdateFilesOrFoldersAsync(IEnumerable paths, bool has try { var matchingItems = filesAndFolders.Where(x => paths.Any(p => p.Equals(x.ItemPath, StringComparison.OrdinalIgnoreCase))); - var results = await Task.WhenAll(matchingItems.Select(x => UpdateFileOrFolderAsync(x, hasSyncStatus))); + var results = await Task.WhenAll(matchingItems.Select(x => GetFileOrFolderUpdateInfoAsync(x, hasSyncStatus))); await CoreApplication.MainView.DispatcherQueue.EnqueueAsync(() => { From 66626e6db62ffa03ab0524c240554726b69dc5bf Mon Sep 17 00:00:00 2001 From: Steve Date: Thu, 2 Sep 2021 15:52:46 +0800 Subject: [PATCH 04/23] Remove optional parameter --- Files/ViewModels/ItemViewModel.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Files/ViewModels/ItemViewModel.cs b/Files/ViewModels/ItemViewModel.cs index b749a6c122e3..edf441566ed1 100644 --- a/Files/ViewModels/ItemViewModel.cs +++ b/Files/ViewModels/ItemViewModel.cs @@ -2020,7 +2020,7 @@ private async Task AddFileOrFolderAsync(string fileOrFolderPath, string dateRetu await AddFileOrFolderAsync(listedItem); } - private async Task<(ListedItem Item, CloudDriveSyncStatusUI SyncUI, long? Size, DateTimeOffset Created, DateTimeOffset Modified)?> GetFileOrFolderUpdateInfoAsync(ListedItem item, bool hasSyncStatus = false) + private async Task<(ListedItem Item, CloudDriveSyncStatusUI SyncUI, long? Size, DateTimeOffset Created, DateTimeOffset Modified)?> GetFileOrFolderUpdateInfoAsync(ListedItem item, bool hasSyncStatus) { IStorageItem storageItem = null; if (item.PrimaryItemAttribute == StorageItemTypes.File) @@ -2057,7 +2057,7 @@ private async Task AddFileOrFolderAsync(string fileOrFolderPath, string dateRetu return null; } - private async Task UpdateFilesOrFoldersAsync(IEnumerable paths, bool hasSyncStatus = false) + private async Task UpdateFilesOrFoldersAsync(IEnumerable paths, bool hasSyncStatus) { try { From b60d854af7e19399f6b26c6555f5fc951688d5cf Mon Sep 17 00:00:00 2001 From: Steve Date: Fri, 3 Sep 2021 15:07:31 +0800 Subject: [PATCH 05/23] Improve performance of properties loading --- Files/Files.csproj | 1 + Files/Helpers/AsyncManualResetEvent.cs | 67 +++++ Files/ViewModels/ItemViewModel.cs | 251 ++++++++---------- .../Views/LayoutModes/ColumnViewBase.xaml.cs | 4 - .../LayoutModes/ColumnViewBrowser.xaml.cs | 4 - .../LayoutModes/DetailsLayoutBrowser.xaml.cs | 4 - .../Views/LayoutModes/GridViewBrowser.xaml.cs | 4 - 7 files changed, 178 insertions(+), 157 deletions(-) create mode 100644 Files/Helpers/AsyncManualResetEvent.cs diff --git a/Files/Files.csproj b/Files/Files.csproj index 16dcae0cc0d5..60cd0baaab41 100644 --- a/Files/Files.csproj +++ b/Files/Files.csproj @@ -275,6 +275,7 @@ + diff --git a/Files/Helpers/AsyncManualResetEvent.cs b/Files/Helpers/AsyncManualResetEvent.cs new file mode 100644 index 000000000000..92ba496e9f1e --- /dev/null +++ b/Files/Helpers/AsyncManualResetEvent.cs @@ -0,0 +1,67 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading; +using System.Threading.Tasks; + +namespace Files.Helpers +{ + public class AsyncManualResetEvent + { + private volatile TaskCompletionSource m_tcs = new TaskCompletionSource(); + + public Task WaitAsync(CancellationToken cancellationToken = default) + { + var tcs = m_tcs; + using (cancellationToken.Register( + s => ((TaskCompletionSource)s).TrySetCanceled(), tcs)) { } + + return m_tcs.Task; + } + + public Task WaitAsync(int milliseconds, CancellationToken cancellationToken = default) + { + var tcs = m_tcs; + + using (cancellationToken.Register( + s => ((TaskCompletionSource)s).TrySetCanceled(), tcs)) { } + + var cancellationTokenSource = new CancellationTokenSource(); + cancellationTokenSource.CancelAfter(milliseconds); + using (cancellationTokenSource.Token.Register( + s => + { + var l_tcs = (TaskCompletionSource)s; + + if (!l_tcs.Task.IsCanceled) + { + l_tcs.TrySetResult(false); + } + + cancellationTokenSource.Dispose(); + }, tcs)) { } + + return m_tcs.Task; + } + + public void Set() + { + var tcs = m_tcs; + Task.Factory.StartNew(s => ((TaskCompletionSource)s).TrySetResult(true), + tcs, CancellationToken.None, TaskCreationOptions.PreferFairness, TaskScheduler.Default); + tcs.Task.Wait(); + } + + public void Reset() + { + while (true) + { + var tcs = m_tcs; + if (!tcs.Task.IsCompleted || + Interlocked.CompareExchange(ref m_tcs, new TaskCompletionSource(), tcs) == tcs) + return; + } + } + } +} diff --git a/Files/ViewModels/ItemViewModel.cs b/Files/ViewModels/ItemViewModel.cs index 4bae49dda194..3cedf0b78948 100644 --- a/Files/ViewModels/ItemViewModel.cs +++ b/Files/ViewModels/ItemViewModel.cs @@ -44,10 +44,9 @@ namespace Files.ViewModels { public class ItemViewModel : ObservableObject, IDisposable { - private readonly SemaphoreSlim enumFolderSemaphore, loadExtendedPropsSemaphore; + private readonly SemaphoreSlim enumFolderSemaphore; private readonly ConcurrentQueue<(uint Action, string FileName)> operationQueue; - private readonly ConcurrentDictionary itemLoadQueue; - private readonly ManualResetEventSlim operationEvent, itemLoadEvent; + private readonly AsyncManualResetEvent operationEvent, loadPropsEvent; private IntPtr hWatchDir; private IAsyncAction aWatcherAction; @@ -345,14 +344,12 @@ public ItemViewModel(FolderSettingsViewModel folderSettingsViewModel) filesAndFolders = new List(); FilesAndFolders = new BulkConcurrentObservableCollection(); operationQueue = new ConcurrentQueue<(uint Action, string FileName)>(); - itemLoadQueue = new ConcurrentDictionary(); addFilesCTS = new CancellationTokenSource(); semaphoreCTS = new CancellationTokenSource(); loadPropsCTS = new CancellationTokenSource(); - operationEvent = new ManualResetEventSlim(); - itemLoadEvent = new ManualResetEventSlim(); + operationEvent = new AsyncManualResetEvent(); enumFolderSemaphore = new SemaphoreSlim(1, 1); - loadExtendedPropsSemaphore = new SemaphoreSlim(Environment.ProcessorCount, Environment.ProcessorCount); + loadPropsEvent = new AsyncManualResetEvent(); shouldDisplayFileExtensions = App.AppSettings.ShowFileExtensions; AppServiceConnectionHelper.ConnectionChanged += AppServiceConnectionHelper_ConnectionChanged; @@ -455,11 +452,6 @@ public void CancelExtendedPropertiesLoading() loadPropsCTS = new CancellationTokenSource(); } - public void CancelExtendedPropertiesLoadingForItem(ListedItem item) - { - itemLoadQueue.TryUpdate(item.ItemPath, true, false); - } - public async Task ApplySingleFileChangeAsync(ListedItem item) { var newIndex = filesAndFolders.IndexOf(item); @@ -907,161 +899,139 @@ public async Task LoadExtendedItemProperties(ListedItem item, uint thumbnailSize return; } - try - { - itemLoadQueue[item.ItemPath] = false; - await loadExtendedPropsSemaphore.WaitAsync(loadPropsCTS.Token); - if (itemLoadQueue.TryGetValue(item.ItemPath, out var canceled) && canceled) - { - loadExtendedPropsSemaphore.Release(); - return; - } - } - catch (OperationCanceledException) - { - return; - } - finally - { - itemLoadQueue.TryRemove(item.ItemPath, out _); - } - item.ItemPropertiesInitialized = true; - await Task.Run(async () => + try { - try - { - itemLoadEvent.Wait(loadPropsCTS.Token); - } - catch (OperationCanceledException) + await Task.Run(async () => { - loadExtendedPropsSemaphore.Release(); - return; - } + await loadPropsEvent.WaitAsync(loadPropsCTS.Token); - var wasSyncStatusLoaded = false; - ImageSource groupImage = null; - bool loadGroupHeaderInfo = false; - GroupedCollection gp = null; - try - { - bool isFileTypeGroupMode = folderSettings.DirectoryGroupOption == GroupOption.FileType; - BaseStorageFile matchingStorageFile = null; - if (item.Key != null && FilesAndFolders.IsGrouped && FilesAndFolders.GetExtendedGroupHeaderInfo != null) + var wasSyncStatusLoaded = false; + ImageSource groupImage = null; + bool loadGroupHeaderInfo = false; + GroupedCollection gp = null; + try { - gp = FilesAndFolders.GroupedCollection.Where(x => x.Model.Key == item.Key).FirstOrDefault(); - loadGroupHeaderInfo = !(gp is null) && !gp.Model.Initialized && !(gp.GetExtendedGroupHeaderInfo is null); - } + bool isFileTypeGroupMode = folderSettings.DirectoryGroupOption == GroupOption.FileType; + BaseStorageFile matchingStorageFile = null; + if (item.Key != null && FilesAndFolders.IsGrouped && FilesAndFolders.GetExtendedGroupHeaderInfo != null) + { + gp = FilesAndFolders.GroupedCollection.Where(x => x.Model.Key == item.Key).FirstOrDefault(); + loadGroupHeaderInfo = !(gp is null) && !gp.Model.Initialized && !(gp.GetExtendedGroupHeaderInfo is null); + } - if (item.IsLibraryItem || item.PrimaryItemAttribute == StorageItemTypes.File || item.IsZipItem) - { - if (!item.IsShortcutItem && !item.IsHiddenItem && !FtpHelpers.IsFtpPath(item.ItemPath)) + if (item.IsLibraryItem || item.PrimaryItemAttribute == StorageItemTypes.File || item.IsZipItem) { - matchingStorageFile = await GetFileFromPathAsync(item.ItemPath); - if (matchingStorageFile != null) + if (!item.IsShortcutItem && !item.IsHiddenItem && !FtpHelpers.IsFtpPath(item.ItemPath)) { - await LoadItemThumbnail(item, thumbnailSize, matchingStorageFile, true); - - var syncStatus = await CheckCloudDriveSyncStatusAsync(matchingStorageFile); - var fileFRN = await FileTagsHelper.GetFileFRN(matchingStorageFile); - var fileTag = FileTagsHelper.ReadFileTag(item.ItemPath); - await CoreApplication.MainView.DispatcherQueue.EnqueueAsync(() => + matchingStorageFile = await GetFileFromPathAsync(item.ItemPath); + if (matchingStorageFile != null) { - item.FolderRelativeId = matchingStorageFile.FolderRelativeId; - item.ItemType = matchingStorageFile.DisplayType; - item.SyncStatusUI = CloudDriveSyncStatusUI.FromCloudDriveSyncStatus(syncStatus); - item.FileFRN = fileFRN; - item.FileTag = fileTag; - }, Windows.System.DispatcherQueuePriority.Low); - FileTagsHelper.DbInstance.SetTag(item.ItemPath, item.FileFRN, item.FileTag); - wasSyncStatusLoaded = true; + await LoadItemThumbnail(item, thumbnailSize, matchingStorageFile, true); + + var syncStatus = await CheckCloudDriveSyncStatusAsync(matchingStorageFile); + var fileFRN = await FileTagsHelper.GetFileFRN(matchingStorageFile); + var fileTag = FileTagsHelper.ReadFileTag(item.ItemPath); + await CoreApplication.MainView.DispatcherQueue.EnqueueAsync(() => + { + item.FolderRelativeId = matchingStorageFile.FolderRelativeId; + item.ItemType = matchingStorageFile.DisplayType; + item.SyncStatusUI = CloudDriveSyncStatusUI.FromCloudDriveSyncStatus(syncStatus); + item.FileFRN = fileFRN; + item.FileTag = fileTag; + }, Windows.System.DispatcherQueuePriority.Low); + FileTagsHelper.DbInstance.SetTag(item.ItemPath, item.FileFRN, item.FileTag); + wasSyncStatusLoaded = true; + } + } + if (!wasSyncStatusLoaded) + { + await LoadItemThumbnail(item, thumbnailSize, null, true); } } - if (!wasSyncStatusLoaded) - { - await LoadItemThumbnail(item, thumbnailSize, null, true); - } - } - else - { - if (!item.IsShortcutItem && !item.IsHiddenItem && !FtpHelpers.IsFtpPath(item.ItemPath)) + else { - BaseStorageFolder matchingStorageFolder = await GetFolderFromPathAsync(item.ItemPath); - if (matchingStorageFolder != null) + if (!item.IsShortcutItem && !item.IsHiddenItem && !FtpHelpers.IsFtpPath(item.ItemPath)) { - await LoadItemThumbnail(item, thumbnailSize, matchingStorageFolder, true); - if (matchingStorageFolder.DisplayName != item.ItemName && !matchingStorageFolder.DisplayName.StartsWith("$R")) + BaseStorageFolder matchingStorageFolder = await GetFolderFromPathAsync(item.ItemPath); + if (matchingStorageFolder != null) { - await CoreApplication.MainView.DispatcherQueue.EnqueueAsync(() => - { - item.ItemName = matchingStorageFolder.DisplayName; - }); - await fileListCache.SaveFileDisplayNameToCache(item.ItemPath, matchingStorageFolder.DisplayName); - if (folderSettings.DirectorySortOption == SortOption.Name && !isLoadingItems) + await LoadItemThumbnail(item, thumbnailSize, matchingStorageFolder, true); + if (matchingStorageFolder.DisplayName != item.ItemName && !matchingStorageFolder.DisplayName.StartsWith("$R")) { - await OrderFilesAndFoldersAsync(); - await ApplySingleFileChangeAsync(item); + await CoreApplication.MainView.DispatcherQueue.EnqueueAsync(() => + { + item.ItemName = matchingStorageFolder.DisplayName; + }); + await fileListCache.SaveFileDisplayNameToCache(item.ItemPath, matchingStorageFolder.DisplayName); + if (folderSettings.DirectorySortOption == SortOption.Name && !isLoadingItems) + { + await OrderFilesAndFoldersAsync(); + await ApplySingleFileChangeAsync(item); + } } - } - var syncStatus = await CheckCloudDriveSyncStatusAsync(matchingStorageFolder); - var fileFRN = await FileTagsHelper.GetFileFRN(matchingStorageFolder); - var fileTag = FileTagsHelper.ReadFileTag(item.ItemPath); - await CoreApplication.MainView.DispatcherQueue.EnqueueAsync(() => - { - item.FolderRelativeId = matchingStorageFolder.FolderRelativeId; - item.ItemType = matchingStorageFolder.DisplayType; - item.SyncStatusUI = CloudDriveSyncStatusUI.FromCloudDriveSyncStatus(syncStatus); - item.FileFRN = fileFRN; - item.FileTag = fileTag; - }, Windows.System.DispatcherQueuePriority.Low); - FileTagsHelper.DbInstance.SetTag(item.ItemPath, item.FileFRN, item.FileTag); - wasSyncStatusLoaded = true; + var syncStatus = await CheckCloudDriveSyncStatusAsync(matchingStorageFolder); + var fileFRN = await FileTagsHelper.GetFileFRN(matchingStorageFolder); + var fileTag = FileTagsHelper.ReadFileTag(item.ItemPath); + await CoreApplication.MainView.DispatcherQueue.EnqueueAsync(() => + { + item.FolderRelativeId = matchingStorageFolder.FolderRelativeId; + item.ItemType = matchingStorageFolder.DisplayType; + item.SyncStatusUI = CloudDriveSyncStatusUI.FromCloudDriveSyncStatus(syncStatus); + item.FileFRN = fileFRN; + item.FileTag = fileTag; + }, Windows.System.DispatcherQueuePriority.Low); + FileTagsHelper.DbInstance.SetTag(item.ItemPath, item.FileFRN, item.FileTag); + wasSyncStatusLoaded = true; + } + } + if (!wasSyncStatusLoaded) + { + await LoadItemThumbnail(item, thumbnailSize, null, true); } } - if (!wasSyncStatusLoaded) + + if (loadGroupHeaderInfo && isFileTypeGroupMode) { - await LoadItemThumbnail(item, thumbnailSize, null, true); + groupImage = await GetItemTypeGroupIcon(item, matchingStorageFile); } } - - if (loadGroupHeaderInfo && isFileTypeGroupMode) + catch (Exception ex) { - groupImage = await GetItemTypeGroupIcon(item, matchingStorageFile); } - } - catch (Exception ex) - { - } - finally - { - if (!wasSyncStatusLoaded) + finally { - await FilesystemTasks.Wrap(async () => + if (!wasSyncStatusLoaded) { - var fileTag = FileTagsHelper.ReadFileTag(item.ItemPath); - await CoreApplication.MainView.DispatcherQueue.EnqueueAsync(() => + await FilesystemTasks.Wrap(async () => { - item.SyncStatusUI = new CloudDriveSyncStatusUI() { LoadSyncStatus = false }; // Reset cloud sync status icon - item.FileTag = fileTag; - }, Windows.System.DispatcherQueuePriority.Low); - FileTagsHelper.DbInstance.SetTag(item.ItemPath, item.FileFRN, item.FileTag); - }); - } + var fileTag = FileTagsHelper.ReadFileTag(item.ItemPath); + await CoreApplication.MainView.DispatcherQueue.EnqueueAsync(() => + { + item.SyncStatusUI = new CloudDriveSyncStatusUI() { LoadSyncStatus = false }; // Reset cloud sync status icon + item.FileTag = fileTag; + }, Windows.System.DispatcherQueuePriority.Low); + FileTagsHelper.DbInstance.SetTag(item.ItemPath, item.FileFRN, item.FileTag); + }); + } - if (loadGroupHeaderInfo) - { - await FilesystemTasks.Wrap(() => CoreApplication.MainView.DispatcherQueue.EnqueueAsync(() => + if (loadGroupHeaderInfo) { - gp.Model.ImageSource = groupImage; - gp.InitializeExtendedGroupHeaderInfoAsync(); - })); + await FilesystemTasks.Wrap(() => CoreApplication.MainView.DispatcherQueue.EnqueueAsync(() => + { + gp.Model.ImageSource = groupImage; + gp.InitializeExtendedGroupHeaderInfoAsync(); + })); + } } - - loadExtendedPropsSemaphore.Release(); - } - }); + }, loadPropsCTS.Token); + } + catch (OperationCanceledException) + { + // ignored + } } private async Task GetItemTypeGroupIcon(ListedItem item, BaseStorageFile matchingStorageItem = null) @@ -1141,6 +1111,8 @@ private async void RapidAddItemsToCollectionAsync(string path, string previousDi return; } + loadPropsEvent.Reset(); + try { // Drop all the other waiting instances @@ -1148,7 +1120,6 @@ private async void RapidAddItemsToCollectionAsync(string path, string previousDi semaphoreCTS = new CancellationTokenSource(); IsLoadingItems = true; - itemLoadEvent.Reset(); filesAndFolders.Clear(); FilesAndFolders.Clear(); @@ -1198,7 +1169,7 @@ private async void RapidAddItemsToCollectionAsync(string path, string previousDi { DirectoryInfoUpdated?.Invoke(this, EventArgs.Empty); // Make sure item count is updated enumFolderSemaphore.Release(); - itemLoadEvent.Set(); + loadPropsEvent.Set(); } postLoadCallback?.Invoke(); @@ -1822,7 +1793,7 @@ private async void ProcessOperationQueue(CancellationToken cancellationToken) { while (!cancellationToken.IsCancellationRequested) { - if (operationEvent.Wait(200, cancellationToken)) + if (await operationEvent.WaitAsync(200, cancellationToken)) { operationEvent.Reset(); while (operationQueue.TryDequeue(out var operation)) @@ -2133,7 +2104,6 @@ public async Task SearchAsync(FolderSearch search) filesAndFolders.Clear(); IsLoadingItems = true; IsSearchResults = true; - itemLoadEvent.Reset(); await ApplyFilesAndFoldersChangesAsync(); EmptyTextType = EmptyTextType.None; @@ -2154,7 +2124,6 @@ public async Task SearchAsync(FolderSearch search) ItemLoadStatusChanged?.Invoke(this, new ItemLoadStatusChangedEventArgs() { Status = ItemLoadStatusChangedEventArgs.ItemLoadStatus.Complete }); IsLoadingItems = false; - itemLoadEvent.Set(); } public void CancelSearch() diff --git a/Files/Views/LayoutModes/ColumnViewBase.xaml.cs b/Files/Views/LayoutModes/ColumnViewBase.xaml.cs index 9d44c71e560b..86cf0edc77b5 100644 --- a/Files/Views/LayoutModes/ColumnViewBase.xaml.cs +++ b/Files/Views/LayoutModes/ColumnViewBase.xaml.cs @@ -564,10 +564,6 @@ private void FileList_ContainerContentChanging(ListViewBase sender, ContainerCon { UninitializeDrag(args.ItemContainer); args.ItemContainer.PointerPressed -= FileListListItem_PointerPressed; - if (args.Item is ListedItem item) - { - ParentShellPageInstance.FilesystemViewModel.CancelExtendedPropertiesLoadingForItem(item); - } } } diff --git a/Files/Views/LayoutModes/ColumnViewBrowser.xaml.cs b/Files/Views/LayoutModes/ColumnViewBrowser.xaml.cs index a14c2a21d391..8a982d91f85c 100644 --- a/Files/Views/LayoutModes/ColumnViewBrowser.xaml.cs +++ b/Files/Views/LayoutModes/ColumnViewBrowser.xaml.cs @@ -740,10 +740,6 @@ private void FileList_ContainerContentChanging(ListViewBase sender, ContainerCon { UninitializeDrag(args.ItemContainer); args.ItemContainer.PointerPressed -= FileListListItem_PointerPressed; - if (args.Item is ListedItem item) - { - ParentShellPageInstance.FilesystemViewModel.CancelExtendedPropertiesLoadingForItem(item); - } } } diff --git a/Files/Views/LayoutModes/DetailsLayoutBrowser.xaml.cs b/Files/Views/LayoutModes/DetailsLayoutBrowser.xaml.cs index 174b9fca33c2..03c9c79517e7 100644 --- a/Files/Views/LayoutModes/DetailsLayoutBrowser.xaml.cs +++ b/Files/Views/LayoutModes/DetailsLayoutBrowser.xaml.cs @@ -698,10 +698,6 @@ private void FileList_ContainerContentChanging(ListViewBase sender, ContainerCon { UninitializeDrag(args.ItemContainer); args.ItemContainer.PointerPressed -= FileListGridItem_PointerPressed; - if (args.Item is ListedItem item) - { - ParentShellPageInstance.FilesystemViewModel.CancelExtendedPropertiesLoadingForItem(item); - } } } diff --git a/Files/Views/LayoutModes/GridViewBrowser.xaml.cs b/Files/Views/LayoutModes/GridViewBrowser.xaml.cs index d7a29b578070..f3240e9146ab 100644 --- a/Files/Views/LayoutModes/GridViewBrowser.xaml.cs +++ b/Files/Views/LayoutModes/GridViewBrowser.xaml.cs @@ -587,10 +587,6 @@ private void FileList_ContainerContentChanging(ListViewBase sender, ContainerCon { UninitializeDrag(args.ItemContainer); args.ItemContainer.PointerPressed -= FileListGridItem_PointerPressed; - if (args.Item is ListedItem item) - { - ParentShellPageInstance.FilesystemViewModel.CancelExtendedPropertiesLoadingForItem(item); - } } } } From 5a63665046558405d9b85ec94743d7f839610f6e Mon Sep 17 00:00:00 2001 From: Steve Date: Fri, 3 Sep 2021 15:22:52 +0800 Subject: [PATCH 06/23] Fix cancellation registration --- Files/Helpers/AsyncManualResetEvent.cs | 28 +++++++++++++------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/Files/Helpers/AsyncManualResetEvent.cs b/Files/Helpers/AsyncManualResetEvent.cs index 92ba496e9f1e..0d99bdeb00f3 100644 --- a/Files/Helpers/AsyncManualResetEvent.cs +++ b/Files/Helpers/AsyncManualResetEvent.cs @@ -14,8 +14,8 @@ public class AsyncManualResetEvent public Task WaitAsync(CancellationToken cancellationToken = default) { var tcs = m_tcs; - using (cancellationToken.Register( - s => ((TaskCompletionSource)s).TrySetCanceled(), tcs)) { } + cancellationToken.Register( + s => ((TaskCompletionSource)s).TrySetCanceled(), tcs); return m_tcs.Task; } @@ -24,23 +24,23 @@ public Task WaitAsync(int milliseconds, CancellationToken cancellationToke { var tcs = m_tcs; - using (cancellationToken.Register( - s => ((TaskCompletionSource)s).TrySetCanceled(), tcs)) { } + cancellationToken.Register( + s => ((TaskCompletionSource)s).TrySetCanceled(), tcs); var cancellationTokenSource = new CancellationTokenSource(); cancellationTokenSource.CancelAfter(milliseconds); - using (cancellationTokenSource.Token.Register( - s => - { - var l_tcs = (TaskCompletionSource)s; + cancellationTokenSource.Token.Register( + s => + { + var l_tcs = (TaskCompletionSource)s; - if (!l_tcs.Task.IsCanceled) - { - l_tcs.TrySetResult(false); - } + if (!l_tcs.Task.IsCanceled) + { + l_tcs.TrySetResult(false); + } - cancellationTokenSource.Dispose(); - }, tcs)) { } + cancellationTokenSource.Dispose(); + }, tcs); return m_tcs.Task; } From af61580cf81e3712ca27bf2bf391fd454b0002ba Mon Sep 17 00:00:00 2001 From: Steve Date: Fri, 3 Sep 2021 15:44:45 +0800 Subject: [PATCH 07/23] Fix AsyncManualResetEvent timeout implementation --- Files/Helpers/AsyncManualResetEvent.cs | 22 +++++++--------------- 1 file changed, 7 insertions(+), 15 deletions(-) diff --git a/Files/Helpers/AsyncManualResetEvent.cs b/Files/Helpers/AsyncManualResetEvent.cs index 0d99bdeb00f3..8f32f1cd39bd 100644 --- a/Files/Helpers/AsyncManualResetEvent.cs +++ b/Files/Helpers/AsyncManualResetEvent.cs @@ -20,6 +20,12 @@ public Task WaitAsync(CancellationToken cancellationToken = default) return m_tcs.Task; } + private async Task Delay(int milliseconds) + { + await Task.Delay(milliseconds); + return false; + } + public Task WaitAsync(int milliseconds, CancellationToken cancellationToken = default) { var tcs = m_tcs; @@ -27,22 +33,8 @@ public Task WaitAsync(int milliseconds, CancellationToken cancellationToke cancellationToken.Register( s => ((TaskCompletionSource)s).TrySetCanceled(), tcs); - var cancellationTokenSource = new CancellationTokenSource(); - cancellationTokenSource.CancelAfter(milliseconds); - cancellationTokenSource.Token.Register( - s => - { - var l_tcs = (TaskCompletionSource)s; - if (!l_tcs.Task.IsCanceled) - { - l_tcs.TrySetResult(false); - } - - cancellationTokenSource.Dispose(); - }, tcs); - - return m_tcs.Task; + return Task.WhenAny(m_tcs.Task, Delay(milliseconds)).Result; } public void Set() From 37433feb4dfee9f5957086b3505ed98c46b21c34 Mon Sep 17 00:00:00 2001 From: Steve Date: Fri, 3 Sep 2021 15:51:18 +0800 Subject: [PATCH 08/23] Add back propoerty load cancellation --- Files/ViewModels/ItemViewModel.cs | 17 +++++++++++++++++ Files/Views/LayoutModes/ColumnViewBase.xaml.cs | 4 ++++ .../Views/LayoutModes/ColumnViewBrowser.xaml.cs | 4 ++++ .../LayoutModes/DetailsLayoutBrowser.xaml.cs | 4 ++++ Files/Views/LayoutModes/GridViewBrowser.xaml.cs | 4 ++++ 5 files changed, 33 insertions(+) diff --git a/Files/ViewModels/ItemViewModel.cs b/Files/ViewModels/ItemViewModel.cs index 3cedf0b78948..9d16736c4cbe 100644 --- a/Files/ViewModels/ItemViewModel.cs +++ b/Files/ViewModels/ItemViewModel.cs @@ -46,6 +46,7 @@ public class ItemViewModel : ObservableObject, IDisposable { private readonly SemaphoreSlim enumFolderSemaphore; private readonly ConcurrentQueue<(uint Action, string FileName)> operationQueue; + private readonly ConcurrentDictionary itemLoadQueue; private readonly AsyncManualResetEvent operationEvent, loadPropsEvent; private IntPtr hWatchDir; private IAsyncAction aWatcherAction; @@ -344,6 +345,7 @@ public ItemViewModel(FolderSettingsViewModel folderSettingsViewModel) filesAndFolders = new List(); FilesAndFolders = new BulkConcurrentObservableCollection(); operationQueue = new ConcurrentQueue<(uint Action, string FileName)>(); + itemLoadQueue = new ConcurrentDictionary(); addFilesCTS = new CancellationTokenSource(); semaphoreCTS = new CancellationTokenSource(); loadPropsCTS = new CancellationTokenSource(); @@ -452,6 +454,11 @@ public void CancelExtendedPropertiesLoading() loadPropsCTS = new CancellationTokenSource(); } + public void CancelExtendedPropertiesLoadingForItem(ListedItem item) + { + itemLoadQueue.TryUpdate(item.ItemPath, true, false); + } + public async Task ApplySingleFileChangeAsync(ListedItem item) { var newIndex = filesAndFolders.IndexOf(item); @@ -900,6 +907,7 @@ public async Task LoadExtendedItemProperties(ListedItem item, uint thumbnailSize } item.ItemPropertiesInitialized = true; + itemLoadQueue[item.ItemPath] = false; try { @@ -907,6 +915,11 @@ await Task.Run(async () => { await loadPropsEvent.WaitAsync(loadPropsCTS.Token); + if (itemLoadQueue.TryGetValue(item.ItemPath, out var canceled) && canceled) + { + return; + } + var wasSyncStatusLoaded = false; ImageSource groupImage = null; bool loadGroupHeaderInfo = false; @@ -1032,6 +1045,10 @@ await FilesystemTasks.Wrap(() => CoreApplication.MainView.DispatcherQueue.Enqueu { // ignored } + finally + { + itemLoadQueue.TryRemove(item.ItemPath, out _); + } } private async Task GetItemTypeGroupIcon(ListedItem item, BaseStorageFile matchingStorageItem = null) diff --git a/Files/Views/LayoutModes/ColumnViewBase.xaml.cs b/Files/Views/LayoutModes/ColumnViewBase.xaml.cs index 86cf0edc77b5..9d44c71e560b 100644 --- a/Files/Views/LayoutModes/ColumnViewBase.xaml.cs +++ b/Files/Views/LayoutModes/ColumnViewBase.xaml.cs @@ -564,6 +564,10 @@ private void FileList_ContainerContentChanging(ListViewBase sender, ContainerCon { UninitializeDrag(args.ItemContainer); args.ItemContainer.PointerPressed -= FileListListItem_PointerPressed; + if (args.Item is ListedItem item) + { + ParentShellPageInstance.FilesystemViewModel.CancelExtendedPropertiesLoadingForItem(item); + } } } diff --git a/Files/Views/LayoutModes/ColumnViewBrowser.xaml.cs b/Files/Views/LayoutModes/ColumnViewBrowser.xaml.cs index 8a982d91f85c..a14c2a21d391 100644 --- a/Files/Views/LayoutModes/ColumnViewBrowser.xaml.cs +++ b/Files/Views/LayoutModes/ColumnViewBrowser.xaml.cs @@ -740,6 +740,10 @@ private void FileList_ContainerContentChanging(ListViewBase sender, ContainerCon { UninitializeDrag(args.ItemContainer); args.ItemContainer.PointerPressed -= FileListListItem_PointerPressed; + if (args.Item is ListedItem item) + { + ParentShellPageInstance.FilesystemViewModel.CancelExtendedPropertiesLoadingForItem(item); + } } } diff --git a/Files/Views/LayoutModes/DetailsLayoutBrowser.xaml.cs b/Files/Views/LayoutModes/DetailsLayoutBrowser.xaml.cs index 03c9c79517e7..174b9fca33c2 100644 --- a/Files/Views/LayoutModes/DetailsLayoutBrowser.xaml.cs +++ b/Files/Views/LayoutModes/DetailsLayoutBrowser.xaml.cs @@ -698,6 +698,10 @@ private void FileList_ContainerContentChanging(ListViewBase sender, ContainerCon { UninitializeDrag(args.ItemContainer); args.ItemContainer.PointerPressed -= FileListGridItem_PointerPressed; + if (args.Item is ListedItem item) + { + ParentShellPageInstance.FilesystemViewModel.CancelExtendedPropertiesLoadingForItem(item); + } } } diff --git a/Files/Views/LayoutModes/GridViewBrowser.xaml.cs b/Files/Views/LayoutModes/GridViewBrowser.xaml.cs index f3240e9146ab..d7a29b578070 100644 --- a/Files/Views/LayoutModes/GridViewBrowser.xaml.cs +++ b/Files/Views/LayoutModes/GridViewBrowser.xaml.cs @@ -587,6 +587,10 @@ private void FileList_ContainerContentChanging(ListViewBase sender, ContainerCon { UninitializeDrag(args.ItemContainer); args.ItemContainer.PointerPressed -= FileListGridItem_PointerPressed; + if (args.Item is ListedItem item) + { + ParentShellPageInstance.FilesystemViewModel.CancelExtendedPropertiesLoadingForItem(item); + } } } } From b4e360c566bfc45b208cc33cf9038aae84fd5e33 Mon Sep 17 00:00:00 2001 From: Steve Date: Fri, 3 Sep 2021 15:54:41 +0800 Subject: [PATCH 09/23] Minor fixes --- Files/Helpers/AsyncManualResetEvent.cs | 17 ++++++++++------- Files/ViewModels/ItemViewModel.cs | 6 ++++-- 2 files changed, 14 insertions(+), 9 deletions(-) diff --git a/Files/Helpers/AsyncManualResetEvent.cs b/Files/Helpers/AsyncManualResetEvent.cs index 8f32f1cd39bd..9a69f892ab1c 100644 --- a/Files/Helpers/AsyncManualResetEvent.cs +++ b/Files/Helpers/AsyncManualResetEvent.cs @@ -11,13 +11,15 @@ public class AsyncManualResetEvent { private volatile TaskCompletionSource m_tcs = new TaskCompletionSource(); - public Task WaitAsync(CancellationToken cancellationToken = default) + public async Task WaitAsync(CancellationToken cancellationToken = default) { var tcs = m_tcs; + var cancelTcs = new TaskCompletionSource(); + cancellationToken.Register( - s => ((TaskCompletionSource)s).TrySetCanceled(), tcs); + s => ((TaskCompletionSource)s).TrySetCanceled(), cancelTcs); - return m_tcs.Task; + await await Task.WhenAny(tcs.Task, cancelTcs.Task); } private async Task Delay(int milliseconds) @@ -26,15 +28,15 @@ private async Task Delay(int milliseconds) return false; } - public Task WaitAsync(int milliseconds, CancellationToken cancellationToken = default) + public async Task WaitAsync(int milliseconds, CancellationToken cancellationToken = default) { var tcs = m_tcs; + var cancelTcs = new TaskCompletionSource(); cancellationToken.Register( - s => ((TaskCompletionSource)s).TrySetCanceled(), tcs); - + s => ((TaskCompletionSource)s).TrySetCanceled(), cancelTcs); - return Task.WhenAny(m_tcs.Task, Delay(milliseconds)).Result; + return await await Task.WhenAny(tcs.Task, cancelTcs.Task, Delay(milliseconds)); } public void Set() @@ -43,6 +45,7 @@ public void Set() Task.Factory.StartNew(s => ((TaskCompletionSource)s).TrySetResult(true), tcs, CancellationToken.None, TaskCreationOptions.PreferFairness, TaskScheduler.Default); tcs.Task.Wait(); + } public void Reset() diff --git a/Files/ViewModels/ItemViewModel.cs b/Files/ViewModels/ItemViewModel.cs index 9d16736c4cbe..86921cb86e69 100644 --- a/Files/ViewModels/ItemViewModel.cs +++ b/Files/ViewModels/ItemViewModel.cs @@ -909,11 +909,13 @@ public async Task LoadExtendedItemProperties(ListedItem item, uint thumbnailSize item.ItemPropertiesInitialized = true; itemLoadQueue[item.ItemPath] = false; + var cts = loadPropsCTS; + try { await Task.Run(async () => { - await loadPropsEvent.WaitAsync(loadPropsCTS.Token); + await loadPropsEvent.WaitAsync(cts.Token); if (itemLoadQueue.TryGetValue(item.ItemPath, out var canceled) && canceled) { @@ -1039,7 +1041,7 @@ await FilesystemTasks.Wrap(() => CoreApplication.MainView.DispatcherQueue.Enqueu })); } } - }, loadPropsCTS.Token); + }, cts.Token); } catch (OperationCanceledException) { From 03ce94d2ac3d783aec66b3fbbb6b712895dbe846 Mon Sep 17 00:00:00 2001 From: Steve Date: Fri, 3 Sep 2021 17:29:07 +0800 Subject: [PATCH 10/23] Uninit if cancelled --- Files/ViewModels/ItemViewModel.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Files/ViewModels/ItemViewModel.cs b/Files/ViewModels/ItemViewModel.cs index 86921cb86e69..968ab6ddf9d7 100644 --- a/Files/ViewModels/ItemViewModel.cs +++ b/Files/ViewModels/ItemViewModel.cs @@ -1045,7 +1045,7 @@ await FilesystemTasks.Wrap(() => CoreApplication.MainView.DispatcherQueue.Enqueu } catch (OperationCanceledException) { - // ignored + item.ItemPropertiesInitialized = false; } finally { From e5797222527cb4df4e42c14c90e0ed8d3536fdb3 Mon Sep 17 00:00:00 2001 From: Steve Date: Fri, 3 Sep 2021 17:30:28 +0800 Subject: [PATCH 11/23] Revert event renaming --- Files/ViewModels/ItemViewModel.cs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/Files/ViewModels/ItemViewModel.cs b/Files/ViewModels/ItemViewModel.cs index 968ab6ddf9d7..ae55e916e771 100644 --- a/Files/ViewModels/ItemViewModel.cs +++ b/Files/ViewModels/ItemViewModel.cs @@ -47,7 +47,7 @@ public class ItemViewModel : ObservableObject, IDisposable private readonly SemaphoreSlim enumFolderSemaphore; private readonly ConcurrentQueue<(uint Action, string FileName)> operationQueue; private readonly ConcurrentDictionary itemLoadQueue; - private readonly AsyncManualResetEvent operationEvent, loadPropsEvent; + private readonly AsyncManualResetEvent operationEvent, itemLoadEvent; private IntPtr hWatchDir; private IAsyncAction aWatcherAction; @@ -351,7 +351,7 @@ public ItemViewModel(FolderSettingsViewModel folderSettingsViewModel) loadPropsCTS = new CancellationTokenSource(); operationEvent = new AsyncManualResetEvent(); enumFolderSemaphore = new SemaphoreSlim(1, 1); - loadPropsEvent = new AsyncManualResetEvent(); + itemLoadEvent = new AsyncManualResetEvent(); shouldDisplayFileExtensions = App.AppSettings.ShowFileExtensions; AppServiceConnectionHelper.ConnectionChanged += AppServiceConnectionHelper_ConnectionChanged; @@ -915,7 +915,7 @@ public async Task LoadExtendedItemProperties(ListedItem item, uint thumbnailSize { await Task.Run(async () => { - await loadPropsEvent.WaitAsync(cts.Token); + await itemLoadEvent.WaitAsync(cts.Token); if (itemLoadQueue.TryGetValue(item.ItemPath, out var canceled) && canceled) { @@ -1130,7 +1130,7 @@ private async void RapidAddItemsToCollectionAsync(string path, string previousDi return; } - loadPropsEvent.Reset(); + itemLoadEvent.Reset(); try { @@ -1188,7 +1188,7 @@ private async void RapidAddItemsToCollectionAsync(string path, string previousDi { DirectoryInfoUpdated?.Invoke(this, EventArgs.Empty); // Make sure item count is updated enumFolderSemaphore.Release(); - loadPropsEvent.Set(); + itemLoadEvent.Set(); } postLoadCallback?.Invoke(); From e91741fe92c37bc1f8e8b26f561d2b7b0b56c8f9 Mon Sep 17 00:00:00 2001 From: Steve Date: Fri, 3 Sep 2021 16:49:29 +0800 Subject: [PATCH 12/23] Delay event set --- Files/ViewModels/ItemViewModel.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Files/ViewModels/ItemViewModel.cs b/Files/ViewModels/ItemViewModel.cs index edf441566ed1..d40f310aed36 100644 --- a/Files/ViewModels/ItemViewModel.cs +++ b/Files/ViewModels/ItemViewModel.cs @@ -1787,11 +1787,11 @@ private void WatchForDirectoryChanges(string path, CloudDriveSyncStatus syncStat operationQueue.Enqueue((action, FileName)); - operationEvent.Set(); - offset += notifyInfo.NextEntryOffset; } while (notifyInfo.NextEntryOffset != 0 && x.Status != AsyncStatus.Canceled); + operationEvent.Set(); + //ResetEvent(overlapped.hEvent); Debug.WriteLine("Task running..."); } From ef1792b83a2763ef1e9d76a08e0f0ff4b872fd08 Mon Sep 17 00:00:00 2001 From: Steve Date: Fri, 3 Sep 2021 17:21:19 +0800 Subject: [PATCH 13/23] Low priority --- Files/ViewModels/ItemViewModel.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Files/ViewModels/ItemViewModel.cs b/Files/ViewModels/ItemViewModel.cs index d40f310aed36..09e461463ad7 100644 --- a/Files/ViewModels/ItemViewModel.cs +++ b/Files/ViewModels/ItemViewModel.cs @@ -2094,7 +2094,7 @@ await CoreApplication.MainView.DispatcherQueue.EnqueueAsync(() => } } } - }); + }, Windows.System.DispatcherQueuePriority.Low); } finally { From c6ca8249b43937f332a9988f86ed3a9543c63c4f Mon Sep 17 00:00:00 2001 From: Steve Date: Fri, 3 Sep 2021 17:47:13 +0800 Subject: [PATCH 14/23] Fix sometimes props not loading --- Files/ViewModels/ItemViewModel.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Files/ViewModels/ItemViewModel.cs b/Files/ViewModels/ItemViewModel.cs index ae55e916e771..73722af70a9f 100644 --- a/Files/ViewModels/ItemViewModel.cs +++ b/Files/ViewModels/ItemViewModel.cs @@ -906,7 +906,6 @@ public async Task LoadExtendedItemProperties(ListedItem item, uint thumbnailSize return; } - item.ItemPropertiesInitialized = true; itemLoadQueue[item.ItemPath] = false; var cts = loadPropsCTS; @@ -916,6 +915,7 @@ public async Task LoadExtendedItemProperties(ListedItem item, uint thumbnailSize await Task.Run(async () => { await itemLoadEvent.WaitAsync(cts.Token); + item.ItemPropertiesInitialized = true; if (itemLoadQueue.TryGetValue(item.ItemPath, out var canceled) && canceled) { @@ -1045,7 +1045,7 @@ await FilesystemTasks.Wrap(() => CoreApplication.MainView.DispatcherQueue.Enqueu } catch (OperationCanceledException) { - item.ItemPropertiesInitialized = false; + // ignored } finally { From a362883ea19bd8206558d7bbf5407af92d8832c7 Mon Sep 17 00:00:00 2001 From: Steve Date: Fri, 3 Sep 2021 18:05:45 +0800 Subject: [PATCH 15/23] Fix itemPropertiesInitialized visibility --- Files/Filesystem/ListedItem.cs | 12 +++++++++++- Files/ViewModels/ItemViewModel.cs | 2 +- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/Files/Filesystem/ListedItem.cs b/Files/Filesystem/ListedItem.cs index 66b00378ecc6..9f8d92b120cd 100644 --- a/Files/Filesystem/ListedItem.cs +++ b/Files/Filesystem/ListedItem.cs @@ -13,6 +13,7 @@ using System.Collections.ObjectModel; using System.IO; using System.Linq; +using System.Threading; using Windows.Storage; using Windows.UI.Xaml.Media.Imaging; @@ -22,7 +23,16 @@ public class ListedItem : ObservableObject, IGroupableItem { public bool IsHiddenItem { get; set; } = false; public StorageItemTypes PrimaryItemAttribute { get; set; } - public bool ItemPropertiesInitialized { get; set; } = false; + + private volatile int itemPropertiesInitialized = 0; + public bool ItemPropertiesInitialized + { + get => itemPropertiesInitialized == 1; + set + { + Interlocked.Exchange(ref itemPropertiesInitialized, value ? 1 : 0); + } + } public string ItemTooltipText { diff --git a/Files/ViewModels/ItemViewModel.cs b/Files/ViewModels/ItemViewModel.cs index 73722af70a9f..7ff74a249755 100644 --- a/Files/ViewModels/ItemViewModel.cs +++ b/Files/ViewModels/ItemViewModel.cs @@ -915,13 +915,13 @@ public async Task LoadExtendedItemProperties(ListedItem item, uint thumbnailSize await Task.Run(async () => { await itemLoadEvent.WaitAsync(cts.Token); - item.ItemPropertiesInitialized = true; if (itemLoadQueue.TryGetValue(item.ItemPath, out var canceled) && canceled) { return; } + item.ItemPropertiesInitialized = true; var wasSyncStatusLoaded = false; ImageSource groupImage = null; bool loadGroupHeaderInfo = false; From dde618262ab915844e42298d7a7cb73adbfc46fa Mon Sep 17 00:00:00 2001 From: Steve Date: Fri, 3 Sep 2021 18:33:13 +0800 Subject: [PATCH 16/23] Styling fixes --- Files/Helpers/AsyncManualResetEvent.cs | 1 - 1 file changed, 1 deletion(-) diff --git a/Files/Helpers/AsyncManualResetEvent.cs b/Files/Helpers/AsyncManualResetEvent.cs index 9a69f892ab1c..79665e7698a7 100644 --- a/Files/Helpers/AsyncManualResetEvent.cs +++ b/Files/Helpers/AsyncManualResetEvent.cs @@ -45,7 +45,6 @@ public void Set() Task.Factory.StartNew(s => ((TaskCompletionSource)s).TrySetResult(true), tcs, CancellationToken.None, TaskCreationOptions.PreferFairness, TaskScheduler.Default); tcs.Task.Wait(); - } public void Reset() From 18e0e877487f47084bd0878ad18bf313b786099b Mon Sep 17 00:00:00 2001 From: Steve Date: Fri, 3 Sep 2021 18:40:15 +0800 Subject: [PATCH 17/23] Prevent props loading while processing operations --- Files/ViewModels/ItemViewModel.cs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Files/ViewModels/ItemViewModel.cs b/Files/ViewModels/ItemViewModel.cs index 7ff74a249755..10c32524910d 100644 --- a/Files/ViewModels/ItemViewModel.cs +++ b/Files/ViewModels/ItemViewModel.cs @@ -1815,6 +1815,8 @@ private async void ProcessOperationQueue(CancellationToken cancellationToken) if (await operationEvent.WaitAsync(200, cancellationToken)) { operationEvent.Reset(); + itemLoadEvent.Reset(); + while (operationQueue.TryDequeue(out var operation)) { if (cancellationToken.IsCancellationRequested) break; @@ -1851,6 +1853,8 @@ private async void ProcessOperationQueue(CancellationToken cancellationToken) anyEdits = false; } } + + itemLoadEvent.Set(); } if (anyEdits && sampler.CheckNow()) From 6f1b29ca05ba900b9dacb5ba87f41e15cfccf474 Mon Sep 17 00:00:00 2001 From: Steve Date: Fri, 3 Sep 2021 19:05:01 +0800 Subject: [PATCH 18/23] Debounce --- Files/ViewModels/ItemViewModel.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Files/ViewModels/ItemViewModel.cs b/Files/ViewModels/ItemViewModel.cs index 09e461463ad7..484b790e26b2 100644 --- a/Files/ViewModels/ItemViewModel.cs +++ b/Files/ViewModels/ItemViewModel.cs @@ -1869,7 +1869,7 @@ private async void ProcessOperationQueue(CancellationToken cancellationToken, bo await UpdateFilesOrFoldersAsync(updateList, hasSyncStatus); } - if (anyEdits) + if (anyEdits && sampler.CheckNow()) { await OrderFilesAndFoldersAsync(); await ApplyFilesAndFoldersChangesAsync(); @@ -2229,4 +2229,4 @@ public enum ItemLoadStatus /// public string Path { get; set; } } -} \ No newline at end of file +} From 1384af8659e6ea63baf26572571c02f30137c8d8 Mon Sep 17 00:00:00 2001 From: Steve Date: Sat, 4 Sep 2021 10:57:50 +0800 Subject: [PATCH 19/23] Fix file not updating and insert a bunch of cancellation points --- Files/ViewModels/ItemViewModel.cs | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/Files/ViewModels/ItemViewModel.cs b/Files/ViewModels/ItemViewModel.cs index 3c0fbc02aa71..29bef892f0f1 100644 --- a/Files/ViewModels/ItemViewModel.cs +++ b/Files/ViewModels/ItemViewModel.cs @@ -940,14 +940,18 @@ await Task.Run(async () => { if (!item.IsShortcutItem && !item.IsHiddenItem && !FtpHelpers.IsFtpPath(item.ItemPath)) { + cts.Token.ThrowIfCancellationRequested(); matchingStorageFile = await GetFileFromPathAsync(item.ItemPath); if (matchingStorageFile != null) { + cts.Token.ThrowIfCancellationRequested(); await LoadItemThumbnail(item, thumbnailSize, matchingStorageFile, true); var syncStatus = await CheckCloudDriveSyncStatusAsync(matchingStorageFile); var fileFRN = await FileTagsHelper.GetFileFRN(matchingStorageFile); var fileTag = FileTagsHelper.ReadFileTag(item.ItemPath); + + cts.Token.ThrowIfCancellationRequested(); await CoreApplication.MainView.DispatcherQueue.EnqueueAsync(() => { item.FolderRelativeId = matchingStorageFile.FolderRelativeId; @@ -969,12 +973,15 @@ await CoreApplication.MainView.DispatcherQueue.EnqueueAsync(() => { if (!item.IsShortcutItem && !item.IsHiddenItem && !FtpHelpers.IsFtpPath(item.ItemPath)) { + cts.Token.ThrowIfCancellationRequested(); BaseStorageFolder matchingStorageFolder = await GetFolderFromPathAsync(item.ItemPath); if (matchingStorageFolder != null) { + cts.Token.ThrowIfCancellationRequested(); await LoadItemThumbnail(item, thumbnailSize, matchingStorageFolder, true); if (matchingStorageFolder.DisplayName != item.ItemName && !matchingStorageFolder.DisplayName.StartsWith("$R")) { + cts.Token.ThrowIfCancellationRequested(); await CoreApplication.MainView.DispatcherQueue.EnqueueAsync(() => { item.ItemName = matchingStorageFolder.DisplayName; @@ -987,9 +994,11 @@ await CoreApplication.MainView.DispatcherQueue.EnqueueAsync(() => } } + cts.Token.ThrowIfCancellationRequested(); var syncStatus = await CheckCloudDriveSyncStatusAsync(matchingStorageFolder); var fileFRN = await FileTagsHelper.GetFileFRN(matchingStorageFolder); var fileTag = FileTagsHelper.ReadFileTag(item.ItemPath); + cts.Token.ThrowIfCancellationRequested(); await CoreApplication.MainView.DispatcherQueue.EnqueueAsync(() => { item.FolderRelativeId = matchingStorageFolder.FolderRelativeId; @@ -1004,12 +1013,14 @@ await CoreApplication.MainView.DispatcherQueue.EnqueueAsync(() => } if (!wasSyncStatusLoaded) { + cts.Token.ThrowIfCancellationRequested(); await LoadItemThumbnail(item, thumbnailSize, null, true); } } if (loadGroupHeaderInfo && isFileTypeGroupMode) { + cts.Token.ThrowIfCancellationRequested(); groupImage = await GetItemTypeGroupIcon(item, matchingStorageFile); } } @@ -1020,6 +1031,7 @@ await CoreApplication.MainView.DispatcherQueue.EnqueueAsync(() => { if (!wasSyncStatusLoaded) { + cts.Token.ThrowIfCancellationRequested(); await FilesystemTasks.Wrap(async () => { var fileTag = FileTagsHelper.ReadFileTag(item.ItemPath); @@ -1034,6 +1046,7 @@ await CoreApplication.MainView.DispatcherQueue.EnqueueAsync(() => if (loadGroupHeaderInfo) { + cts.Token.ThrowIfCancellationRequested(); await FilesystemTasks.Wrap(() => CoreApplication.MainView.DispatcherQueue.EnqueueAsync(() => { gp.Model.ImageSource = groupImage; @@ -2082,6 +2095,7 @@ await CoreApplication.MainView.DispatcherQueue.EnqueueAsync(() => if (result.Value.Size.HasValue) { + item.FileSizeBytes = result.Value.Size.Value; item.FileSize = ByteSizeLib.ByteSize.FromBytes(item.FileSizeBytes).ToBinaryString().ConvertSizeAbbreviation(); } } From ea3985203c56c6ccfe603e2baa7662be7dc7a52a Mon Sep 17 00:00:00 2001 From: Steve Date: Sat, 4 Sep 2021 11:08:47 +0800 Subject: [PATCH 20/23] Remove SQLite cache --- Files/Files.csproj | 1 - .../FileListCache/FileListCacheController.cs | 30 ++-- .../PersistentSQLiteCacheAdapter.cs | 128 ------------------ 3 files changed, 10 insertions(+), 149 deletions(-) delete mode 100644 Files/Helpers/FileListCache/PersistentSQLiteCacheAdapter.cs diff --git a/Files/Files.csproj b/Files/Files.csproj index 60cd0baaab41..c1be3fef9c27 100644 --- a/Files/Files.csproj +++ b/Files/Files.csproj @@ -266,7 +266,6 @@ - diff --git a/Files/Helpers/FileListCache/FileListCacheController.cs b/Files/Helpers/FileListCache/FileListCacheController.cs index ca623bcd55ed..45d8491ee1a1 100644 --- a/Files/Helpers/FileListCache/FileListCacheController.cs +++ b/Files/Helpers/FileListCache/FileListCacheController.cs @@ -1,5 +1,4 @@ -using Files.Common; -using System.Collections.Generic; +using System.Collections.Concurrent; using System.Threading; using System.Threading.Tasks; @@ -14,40 +13,31 @@ public static FileListCacheController GetInstance() return instance ??= new FileListCacheController(); } - private readonly IFileListCache persistentAdapter; - private FileListCacheController() { - persistentAdapter = new PersistentSQLiteCacheAdapter(); } - private readonly Dictionary fileNamesCache = new Dictionary(); + private readonly ConcurrentDictionary fileNamesCache = new ConcurrentDictionary(); - public async Task ReadFileDisplayNameFromCache(string path, CancellationToken cancellationToken) + public Task ReadFileDisplayNameFromCache(string path, CancellationToken cancellationToken) { - var displayName = fileNamesCache.Get(path, (string)null); - if (displayName == null) + if (fileNamesCache.TryGetValue(path, out var displayName)) { - displayName = await persistentAdapter.ReadFileDisplayNameFromCache(path, cancellationToken); - if (displayName != null) - { - fileNamesCache[path] = displayName; - } + return Task.FromResult(displayName); } - return displayName; + + return Task.FromResult(null); } public Task SaveFileDisplayNameToCache(string path, string displayName) { if (displayName == null) { - fileNamesCache.Remove(path); - return persistentAdapter.SaveFileDisplayNameToCache(path, displayName); + fileNamesCache.TryRemove(path, out _); } - fileNamesCache[path] = displayName; - // save entry to persistent cache in background - return persistentAdapter.SaveFileDisplayNameToCache(path, displayName); + fileNamesCache[path] = displayName; + return Task.CompletedTask; } } } \ No newline at end of file diff --git a/Files/Helpers/FileListCache/PersistentSQLiteCacheAdapter.cs b/Files/Helpers/FileListCache/PersistentSQLiteCacheAdapter.cs deleted file mode 100644 index 27779a40d4c5..000000000000 --- a/Files/Helpers/FileListCache/PersistentSQLiteCacheAdapter.cs +++ /dev/null @@ -1,128 +0,0 @@ -using Microsoft.Data.Sqlite; -using System; -using System.Diagnostics; -using System.IO; -using System.Threading; -using System.Threading.Tasks; -using Windows.Storage; - -namespace Files.Helpers.FileListCache -{ - internal class PersistentSQLiteCacheAdapter : IFileListCache, IDisposable - { - private SqliteConnection connection; - private bool disposedValue; - - public async Task SaveFileDisplayNameToCache(string path, string displayName) - { - if (!await InitializeIfNeeded()) - { - return; - } - try - { - if (displayName == null) - { - using var deleteCommand = new SqliteCommand("DELETE FROM FileDisplayNameCache WHERE Id = @Id", connection); - deleteCommand.Parameters.Add("@Id", SqliteType.Text).Value = path; - await deleteCommand.ExecuteNonQueryAsync(); - return; - } - - using var cmd = new SqliteCommand("SELECT Id FROM FileDisplayNameCache WHERE Id = @Id", connection); - cmd.Parameters.Add("@Id", SqliteType.Text).Value = path; - using var reader = await cmd.ExecuteReaderAsync(); - if (reader.HasRows) - { - // need to update entry - using var updateCommand = new SqliteCommand("UPDATE FileDisplayNameCache SET DisplayName = @DisplayName WHERE Id = @Id", connection); - updateCommand.Parameters.Add("@Id", SqliteType.Text).Value = path; - updateCommand.Parameters.Add("@DisplayName", SqliteType.Text).Value = displayName; - await updateCommand.ExecuteNonQueryAsync(); - } - else - { - // need to insert entry - using var insertCommand = new SqliteCommand("INSERT INTO FileDisplayNameCache (Id, DisplayName) VALUES (@Id, @DisplayName)", connection); - insertCommand.Parameters.Add("@Id", SqliteType.Text).Value = path; - insertCommand.Parameters.Add("@DisplayName", SqliteType.Text).Value = displayName; - await insertCommand.ExecuteNonQueryAsync(); - } - } - catch (Exception ex) - { - Debug.WriteLine(ex.ToString()); - } - } - - public async Task ReadFileDisplayNameFromCache(string path, CancellationToken cancellationToken) - { - if (!await InitializeIfNeeded()) - { - return null; - } - try - { - using var cmd = new SqliteCommand("SELECT DisplayName FROM FileDisplayNameCache WHERE Id = @Id", connection); - cmd.Parameters.Add("@Id", SqliteType.Text).Value = path; - - using var reader = await cmd.ExecuteReaderAsync(cancellationToken); - if (!await reader.ReadAsync()) - { - return null; - } - return reader.GetString(0); - } - catch (Exception ex) - { - Debug.WriteLine(ex.ToString()); - return null; - } - } - - public void Dispose() - { - if (!disposedValue) - { - connection?.Dispose(); - disposedValue = true; - } - } - - private void RunCleanupRoutine() - { - } - - private async Task InitializeIfNeeded() - { - if (disposedValue) return false; - if (connection != null) return true; - - string dbPath = Path.Combine(ApplicationData.Current.LocalFolder.Path, "cache.db"); - try - { - SQLitePCL.Batteries_V2.Init(); - - connection = new SqliteConnection($"Data Source='{dbPath}'"); - connection.Open(); - - // create db schema - var createFileDisplayNameCacheTable = @"CREATE TABLE IF NOT EXISTS ""FileDisplayNameCache"" ( - ""Id"" VARCHAR(5000) NOT NULL, - ""DisplayName"" TEXT NOT NULL, - PRIMARY KEY(""Id"") - )"; - using var cmdFileDisplayNameCacheTable = new SqliteCommand(createFileDisplayNameCacheTable, connection); - await cmdFileDisplayNameCacheTable.ExecuteNonQueryAsync(); - - RunCleanupRoutine(); - return true; - } - catch (Exception ex) - { - App.Logger.Warn(ex, $"Failed initializing database with path: {dbPath}"); - return false; - } - } - } -} \ No newline at end of file From 026a5f213e08d263cb48e95f1e5be2f8ca7d449c Mon Sep 17 00:00:00 2001 From: Steve Date: Sun, 5 Sep 2021 20:49:02 +0800 Subject: [PATCH 21/23] Improve performance of file operations --- .../FilesystemItemsOperationDataModel.cs | 23 ++-- Files/Helpers/UIFilesystemHelpers.cs | 128 ++++++++++-------- .../BaseLayoutCommandImplementationModel.cs | 7 +- Files/Views/ColumnShellPage.xaml.cs | 14 +- Files/Views/ModernShellPage.xaml.cs | 14 +- 5 files changed, 101 insertions(+), 85 deletions(-) diff --git a/Files/DataModels/FilesystemItemsOperationDataModel.cs b/Files/DataModels/FilesystemItemsOperationDataModel.cs index 64a542571ae6..58be5f0e7b7e 100644 --- a/Files/DataModels/FilesystemItemsOperationDataModel.cs +++ b/Files/DataModels/FilesystemItemsOperationDataModel.cs @@ -2,6 +2,7 @@ using Files.Helpers; using Files.ViewModels.Dialogs; using System; +using System.Collections.Concurrent; using System.Collections.Generic; using System.Linq; using System.Threading.Tasks; @@ -59,16 +60,16 @@ public FilesystemItemsOperationDataModel(FilesystemOperationType operationType, public async Task> ToItems(Action updatePrimaryButtonEnabled, Action optionGenerateNewName, Action optionReplaceExisting, Action optionSkip) { - List items = new List(); + ConcurrentBag<(int Index, FilesystemOperationItemViewModel Model)> items = new ConcurrentBag<(int Index, FilesystemOperationItemViewModel Model)>(); List nonConflictingItems = IncomingItems.Except(ConflictingItems).ToList(); // Add conflicting items first - foreach (var item in ConflictingItems) + await Task.WhenAll(ConflictingItems.Select(async (item, index) => { var iconData = await FileThumbnailHelper.LoadIconFromPathAsync(item.SourcePath, 64u, Windows.Storage.FileProperties.ThumbnailMode.ListView); - items.Add(new FilesystemOperationItemViewModel(updatePrimaryButtonEnabled, optionGenerateNewName, optionReplaceExisting, optionSkip) + items.Add((index, new FilesystemOperationItemViewModel(updatePrimaryButtonEnabled, optionGenerateNewName, optionReplaceExisting, optionSkip) { IsConflict = true, ItemIcon = iconData != null ? await iconData.ToBitmapAsync() : null, @@ -78,15 +79,17 @@ public async Task> ToItems(Action updateP ConflictResolveOption = FileNameConflictResolveOptionType.GenerateNewName, ItemOperation = item.OperationType, ActionTaken = false - }); - } + })); + })); + + var baseIndex = ConflictingItems.Count; // Then add non-conflicting items - foreach (var item in nonConflictingItems) + await Task.WhenAll(nonConflictingItems.Select(async (item, index) => { var iconData = await FileThumbnailHelper.LoadIconFromPathAsync(item.SourcePath, 64u, Windows.Storage.FileProperties.ThumbnailMode.ListView); - items.Add(new FilesystemOperationItemViewModel(updatePrimaryButtonEnabled, optionGenerateNewName, optionReplaceExisting, optionSkip) + items.Add((baseIndex + index, new FilesystemOperationItemViewModel(updatePrimaryButtonEnabled, optionGenerateNewName, optionReplaceExisting, optionSkip) { IsConflict = false, ItemIcon = iconData != null ? await iconData.ToBitmapAsync() : null, @@ -96,10 +99,10 @@ public async Task> ToItems(Action updateP ConflictResolveOption = FileNameConflictResolveOptionType.NotAConflict, ItemOperation = item.OperationType, ActionTaken = true - }); - } + })); + })); - return items; + return items.OrderBy(i => i.Index).Select(i => i.Model).ToList(); } private string GetOperationIconGlyph(FilesystemOperationType operationType) diff --git a/Files/Helpers/UIFilesystemHelpers.cs b/Files/Helpers/UIFilesystemHelpers.cs index 291970e7804a..480d9187e231 100644 --- a/Files/Helpers/UIFilesystemHelpers.cs +++ b/Files/Helpers/UIFilesystemHelpers.cs @@ -6,7 +6,9 @@ using Files.Interacts; using Microsoft.Toolkit.Uwp; using System; +using System.Collections.Concurrent; using System.Collections.Generic; +using System.IO; using System.Linq; using System.Threading.Tasks; using Windows.ApplicationModel.AppService; @@ -24,7 +26,7 @@ public static async void CutItem(IShellPage associatedInstance) { RequestedOperation = DataPackageOperation.Move }; - List items = new List(); + ConcurrentBag items = new ConcurrentBag(); FilesystemResult result = (FilesystemResult)false; var canFlush = true; @@ -33,46 +35,54 @@ public static async void CutItem(IShellPage associatedInstance) // First, reset DataGrid Rows that may be in "cut" command mode associatedInstance.SlimContentPage.ItemManipulationModel.RefreshItemsOpacity(); - foreach (ListedItem listedItem in associatedInstance.SlimContentPage.SelectedItems.ToList()) + try { + await Task.WhenAll(associatedInstance.SlimContentPage.SelectedItems.ToList().Select(async listedItem => + { // FTP don't support cut, fallback to copy if (listedItem is not FtpItem) - { + { // Dim opacities accordingly listedItem.Opacity = Constants.UI.DimItemOpacity; - } - - if (listedItem is FtpItem ftpItem) - { - canFlush = false; - if (listedItem.PrimaryItemAttribute == StorageItemTypes.File) - { - items.Add(await new FtpStorageFile(ftpItem).ToStorageFileAsync()); } - else if (listedItem.PrimaryItemAttribute == StorageItemTypes.Folder) + + if (listedItem is FtpItem ftpItem) { - items.Add(new FtpStorageFolder(ftpItem)); + canFlush = false; + if (listedItem.PrimaryItemAttribute == StorageItemTypes.File) + { + items.Add(await new FtpStorageFile(ftpItem).ToStorageFileAsync()); + } + else if (listedItem.PrimaryItemAttribute == StorageItemTypes.Folder) + { + items.Add(new FtpStorageFolder(ftpItem)); + } } - } - else if (listedItem.PrimaryItemAttribute == StorageItemTypes.File || listedItem is ZipItem) - { - result = await associatedInstance.FilesystemViewModel.GetFileFromPathAsync(listedItem.ItemPath) - .OnSuccess(t => items.Add(t)); - if (!result) + else if (listedItem.PrimaryItemAttribute == StorageItemTypes.File || listedItem is ZipItem) { - break; + result = await associatedInstance.FilesystemViewModel.GetFileFromPathAsync(listedItem.ItemPath) + .OnSuccess(t => items.Add(t)); + if (!result) + { + throw new IOException($"Failed to process {listedItem.ItemPath}."); + } } - } - else - { - result = await associatedInstance.FilesystemViewModel.GetFolderFromPathAsync(listedItem.ItemPath) - .OnSuccess(t => items.Add(t)); - if (!result) + else { - break; + result = await associatedInstance.FilesystemViewModel.GetFolderFromPathAsync(listedItem.ItemPath) + .OnSuccess(t => items.Add(t)); + if (!result) + { + throw new IOException($"Failed to process {listedItem.ItemPath}."); + } } - } + })); + } + catch + { + return; } + if (result.ErrorCode == FileSystemStatusCode.NotFound) { associatedInstance.SlimContentPage.ItemManipulationModel.RefreshItemsOpacity(); @@ -105,7 +115,7 @@ public static async void CutItem(IShellPage associatedInstance) var onlyStandard = items.All(x => x is StorageFile || x is StorageFolder || x is SystemStorageFile || x is SystemStorageFolder); if (onlyStandard) { - items = await items.ToStandardStorageItemsAsync(); + items = new ConcurrentBag(await items.ToStandardStorageItemsAsync()); } if (!items.Any()) { @@ -132,7 +142,7 @@ public static async Task CopyItem(IShellPage associatedInstance) { RequestedOperation = DataPackageOperation.Copy }; - List items = new List(); + ConcurrentBag items = new ConcurrentBag(); string copySourcePath = associatedInstance.FilesystemViewModel.WorkingDirectory; FilesystemResult result = (FilesystemResult)false; @@ -140,39 +150,47 @@ public static async Task CopyItem(IShellPage associatedInstance) var canFlush = true; if (associatedInstance.SlimContentPage.IsItemSelected) { - foreach (ListedItem listedItem in associatedInstance.SlimContentPage.SelectedItems.ToList()) + try { - if (listedItem is FtpItem ftpItem) + await Task.WhenAll(associatedInstance.SlimContentPage.SelectedItems.ToList().Select(async listedItem => { - canFlush = false; - if (listedItem.PrimaryItemAttribute == StorageItemTypes.File) + if (listedItem is FtpItem ftpItem) { - items.Add(await new FtpStorageFile(ftpItem).ToStorageFileAsync()); + canFlush = false; + if (listedItem.PrimaryItemAttribute == StorageItemTypes.File) + { + items.Add(await new FtpStorageFile(ftpItem).ToStorageFileAsync()); + } + else if (listedItem.PrimaryItemAttribute == StorageItemTypes.Folder) + { + items.Add(new FtpStorageFolder(ftpItem)); + } } - else if (listedItem.PrimaryItemAttribute == StorageItemTypes.Folder) - { - items.Add(new FtpStorageFolder(ftpItem)); - } - } - else if (listedItem.PrimaryItemAttribute == StorageItemTypes.File || listedItem is ZipItem) - { - result = await associatedInstance.FilesystemViewModel.GetFileFromPathAsync(listedItem.ItemPath) - .OnSuccess(t => items.Add(t)); - if (!result) + else if (listedItem.PrimaryItemAttribute == StorageItemTypes.File || listedItem is ZipItem) { - break; + result = await associatedInstance.FilesystemViewModel.GetFileFromPathAsync(listedItem.ItemPath) + .OnSuccess(t => items.Add(t)); + if (!result) + { + throw new IOException($"Failed to process {listedItem.ItemPath}."); + } } - } - else - { - result = await associatedInstance.FilesystemViewModel.GetFolderFromPathAsync(listedItem.ItemPath) - .OnSuccess(t => items.Add(t)); - if (!result) + else { - break; + result = await associatedInstance.FilesystemViewModel.GetFolderFromPathAsync(listedItem.ItemPath) + .OnSuccess(t => items.Add(t)); + if (!result) + { + throw new IOException($"Failed to process {listedItem.ItemPath}."); + } } - } + })); } + catch + { + return; + } + if (result.ErrorCode == FileSystemStatusCode.Unauthorized) { // Try again with fulltrust process @@ -195,7 +213,7 @@ await connection.SendMessageAsync(new ValueSet() var onlyStandard = items.All(x => x is StorageFile || x is StorageFolder || x is SystemStorageFile || x is SystemStorageFolder); if (onlyStandard) { - items = await items.ToStandardStorageItemsAsync(); + items = new ConcurrentBag(await items.ToStandardStorageItemsAsync()); } if (!items.Any()) { diff --git a/Files/Interacts/BaseLayoutCommandImplementationModel.cs b/Files/Interacts/BaseLayoutCommandImplementationModel.cs index 3c36ffbbea2a..2b1699bd1a3d 100644 --- a/Files/Interacts/BaseLayoutCommandImplementationModel.cs +++ b/Files/Interacts/BaseLayoutCommandImplementationModel.cs @@ -199,11 +199,10 @@ await FilesystemHelpers.RestoreFromTrashAsync(StorageItemHelpers.FromPathAndType public virtual async void DeleteItem(RoutedEventArgs e) { - await FilesystemHelpers.DeleteItemsAsync( - SlimContentPage.SelectedItems.Select((item) => StorageItemHelpers.FromPathAndType( + var items = await Task.WhenAll(SlimContentPage.SelectedItems.Select((item) => Task.Run(() => StorageItemHelpers.FromPathAndType( item.ItemPath, - item.PrimaryItemAttribute == StorageItemTypes.File ? FilesystemItemType.File : FilesystemItemType.Directory)).ToList(), - true, false, true); + item.PrimaryItemAttribute == StorageItemTypes.File ? FilesystemItemType.File : FilesystemItemType.Directory)))); + await FilesystemHelpers.DeleteItemsAsync(items, true, false, true); } public virtual void ShowFolderProperties(RoutedEventArgs e) diff --git a/Files/Views/ColumnShellPage.xaml.cs b/Files/Views/ColumnShellPage.xaml.cs index 1a3a21452ca3..ebf9ee14431e 100644 --- a/Files/Views/ColumnShellPage.xaml.cs +++ b/Files/Views/ColumnShellPage.xaml.cs @@ -616,11 +616,10 @@ private async void KeyboardAccelerator_Invoked(KeyboardAccelerator sender, Keybo case (false, true, false, true, VirtualKey.Delete): // shift + delete, PermanentDelete if (ContentPage.IsItemSelected && !NavToolbarViewModel.IsEditModeEnabled && !InstanceViewModel.IsPageTypeSearchResults) { - await FilesystemHelpers.DeleteItemsAsync( - ContentPage.SelectedItems.Select((item) => StorageItemHelpers.FromPathAndType( + var items = await Task.WhenAll(ContentPage.SelectedItems.Select((item) => Task.Run(() => StorageItemHelpers.FromPathAndType( item.ItemPath, - item.PrimaryItemAttribute == StorageItemTypes.File ? FilesystemItemType.File : FilesystemItemType.Directory)).ToList(), - true, true, true); + item.PrimaryItemAttribute == StorageItemTypes.File ? FilesystemItemType.File : FilesystemItemType.Directory)))); + await FilesystemHelpers.DeleteItemsAsync(items, true, true, true); } break; @@ -661,11 +660,10 @@ await FilesystemHelpers.DeleteItemsAsync( case (false, false, false, true, VirtualKey.Delete): // delete, delete item if (ContentPage.IsItemSelected && !ContentPage.IsRenamingItem && !InstanceViewModel.IsPageTypeSearchResults) { - await FilesystemHelpers.DeleteItemsAsync( - ContentPage.SelectedItems.Select((item) => StorageItemHelpers.FromPathAndType( + var items = await Task.WhenAll(ContentPage.SelectedItems.Select((item) => Task.Run(() => StorageItemHelpers.FromPathAndType( item.ItemPath, - item.PrimaryItemAttribute == StorageItemTypes.File ? FilesystemItemType.File : FilesystemItemType.Directory)).ToList(), - true, false, true); + item.PrimaryItemAttribute == StorageItemTypes.File ? FilesystemItemType.File : FilesystemItemType.Directory)))); + await FilesystemHelpers.DeleteItemsAsync(items, true, false, true); } break; diff --git a/Files/Views/ModernShellPage.xaml.cs b/Files/Views/ModernShellPage.xaml.cs index 6ff33e258674..8e72123eefd3 100644 --- a/Files/Views/ModernShellPage.xaml.cs +++ b/Files/Views/ModernShellPage.xaml.cs @@ -661,11 +661,10 @@ private async void KeyboardAccelerator_Invoked(KeyboardAccelerator sender, Keybo case (false, true, false, true, VirtualKey.Delete): // shift + delete, PermanentDelete if (ContentPage.IsItemSelected && !NavToolbarViewModel.IsEditModeEnabled && !InstanceViewModel.IsPageTypeSearchResults) { - await FilesystemHelpers.DeleteItemsAsync( - ContentPage.SelectedItems.Select((item) => StorageItemHelpers.FromPathAndType( + var items = await Task.WhenAll(ContentPage.SelectedItems.Select((item) => Task.Run(() => StorageItemHelpers.FromPathAndType( item.ItemPath, - item.PrimaryItemAttribute == StorageItemTypes.File ? FilesystemItemType.File : FilesystemItemType.Directory)).ToList(), - true, true, true); + item.PrimaryItemAttribute == StorageItemTypes.File ? FilesystemItemType.File : FilesystemItemType.Directory)))); + await FilesystemHelpers.DeleteItemsAsync(items, true, true, true); } break; @@ -706,11 +705,10 @@ await FilesystemHelpers.DeleteItemsAsync( case (false, false, false, true, VirtualKey.Delete): // delete, delete item if (ContentPage.IsItemSelected && !ContentPage.IsRenamingItem && !InstanceViewModel.IsPageTypeSearchResults) { - await FilesystemHelpers.DeleteItemsAsync( - ContentPage.SelectedItems.Select((item) => StorageItemHelpers.FromPathAndType( + var items = await Task.WhenAll(ContentPage.SelectedItems.Select((item) => Task.Run(() => StorageItemHelpers.FromPathAndType( item.ItemPath, - item.PrimaryItemAttribute == StorageItemTypes.File ? FilesystemItemType.File : FilesystemItemType.Directory)).ToList(), - true, false, true); + item.PrimaryItemAttribute == StorageItemTypes.File ? FilesystemItemType.File : FilesystemItemType.Directory)))); + await FilesystemHelpers.DeleteItemsAsync(items, true, false, true); } break; From 9914915362b846545bcd2853dd10a2391d29e118 Mon Sep 17 00:00:00 2001 From: Steve Date: Mon, 6 Sep 2021 00:12:03 +0800 Subject: [PATCH 22/23] Tuning --- Files/ViewModels/ItemViewModel.cs | 29 ++++++++++++++++++++++++++--- 1 file changed, 26 insertions(+), 3 deletions(-) diff --git a/Files/ViewModels/ItemViewModel.cs b/Files/ViewModels/ItemViewModel.cs index 29bef892f0f1..9a733a36dde9 100644 --- a/Files/ViewModels/ItemViewModel.cs +++ b/Files/ViewModels/ItemViewModel.cs @@ -1821,6 +1821,7 @@ private async void ProcessOperationQueue(CancellationToken cancellationToken, bo const uint FILE_ACTION_RENAMED_NEW_NAME = 0x00000005; var sampler = new IntervalSampler(200); + var updateQueue = new Queue(); bool anyEdits = false; try @@ -1831,7 +1832,6 @@ private async void ProcessOperationQueue(CancellationToken cancellationToken, bo { operationEvent.Reset(); itemLoadEvent.Reset(); - var updateList = new HashSet(); while (operationQueue.TryDequeue(out var operation)) { @@ -1847,7 +1847,10 @@ private async void ProcessOperationQueue(CancellationToken cancellationToken, bo break; case FILE_ACTION_MODIFIED: - updateList.Add(operation.FileName); + if (!updateQueue.Contains(operation.FileName)) + { + updateQueue.Enqueue(operation.FileName); + } break; case FILE_ACTION_REMOVED: @@ -1870,7 +1873,27 @@ private async void ProcessOperationQueue(CancellationToken cancellationToken, bo } } - await UpdateFilesOrFoldersAsync(updateList, hasSyncStatus); + var itemsToUpdate = new List(); + for (var i = 0; i < 16 && updateQueue.Count > 0; i++) + { + itemsToUpdate.Add(updateQueue.Dequeue()); + } + + await UpdateFilesOrFoldersAsync(itemsToUpdate, hasSyncStatus); + itemLoadEvent.Set(); + } + + + if (updateQueue.Count > 0) + { + itemLoadEvent.Reset(); + var itemsToUpdate = new List(); + for (var i = 0; i < 16 && updateQueue.Count > 0; i++) + { + itemsToUpdate.Add(updateQueue.Dequeue()); + } + + await UpdateFilesOrFoldersAsync(itemsToUpdate, hasSyncStatus); itemLoadEvent.Set(); } From 22b2a203ffdd100e5178866bdfa200e18fad5b22 Mon Sep 17 00:00:00 2001 From: Steve Date: Mon, 6 Sep 2021 00:14:02 +0800 Subject: [PATCH 23/23] Add UPDATE_BATCH_SIZE --- Files/ViewModels/ItemViewModel.cs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Files/ViewModels/ItemViewModel.cs b/Files/ViewModels/ItemViewModel.cs index 9a733a36dde9..544925b7b3f6 100644 --- a/Files/ViewModels/ItemViewModel.cs +++ b/Files/ViewModels/ItemViewModel.cs @@ -1820,6 +1820,7 @@ private async void ProcessOperationQueue(CancellationToken cancellationToken, bo const uint FILE_ACTION_RENAMED_OLD_NAME = 0x00000004; const uint FILE_ACTION_RENAMED_NEW_NAME = 0x00000005; + const int UPDATE_BATCH_SIZE = 32; var sampler = new IntervalSampler(200); var updateQueue = new Queue(); bool anyEdits = false; @@ -1874,7 +1875,7 @@ private async void ProcessOperationQueue(CancellationToken cancellationToken, bo } var itemsToUpdate = new List(); - for (var i = 0; i < 16 && updateQueue.Count > 0; i++) + for (var i = 0; i < UPDATE_BATCH_SIZE && updateQueue.Count > 0; i++) { itemsToUpdate.Add(updateQueue.Dequeue()); } @@ -1888,7 +1889,7 @@ private async void ProcessOperationQueue(CancellationToken cancellationToken, bo { itemLoadEvent.Reset(); var itemsToUpdate = new List(); - for (var i = 0; i < 16 && updateQueue.Count > 0; i++) + for (var i = 0; i < UPDATE_BATCH_SIZE && updateQueue.Count > 0; i++) { itemsToUpdate.Add(updateQueue.Dequeue()); }